fedimint_dbtool/
dump.rs

1use std::collections::BTreeMap;
2use std::path::PathBuf;
3
4use erased_serde::Serialize;
5use fedimint_client::db::{ClientConfigKey, OperationLogKeyPrefix};
6use fedimint_client::module_init::ClientModuleInitRegistry;
7use fedimint_client_module::oplog::OperationLogEntry;
8use fedimint_core::config::{ClientConfig, CommonModuleInitRegistry};
9use fedimint_core::core::ModuleKind;
10use fedimint_core::db::{
11    Database, DatabaseTransaction, DatabaseVersionKey, IDatabaseTransactionOpsCore,
12    IDatabaseTransactionOpsCoreTyped,
13};
14use fedimint_core::encoding::Encodable;
15use fedimint_core::module::registry::{ModuleDecoderRegistry, ModuleRegistry};
16use fedimint_core::push_db_pair_items;
17use fedimint_gateway_server_db::GatewayDbtxNcExt as _;
18use fedimint_rocksdb::RocksDbReadOnly;
19use fedimint_server::config::ServerConfig;
20use fedimint_server::config::io::read_server_config;
21use fedimint_server::consensus::db as consensus_db;
22use fedimint_server::core::{ServerModuleInitRegistry, ServerModuleInitRegistryExt};
23use fedimint_server::db as server_db;
24use fedimint_server::net::api::announcement::ApiAnnouncementPrefix;
25use futures::StreamExt;
26use strum::IntoEnumIterator;
27
28macro_rules! push_db_pair_items_no_serde {
29    ($dbtx:ident, $prefix_type:expr_2021, $key_type:ty, $value_type:ty, $map:ident, $key_literal:literal) => {
30        let db_items = IDatabaseTransactionOpsCoreTyped::find_by_prefix($dbtx, &$prefix_type)
31            .await
32            .map(|(key, val)| {
33                (
34                    Encodable::consensus_encode_to_hex(&key),
35                    SerdeWrapper::from_encodable(&val),
36                )
37            })
38            .collect::<BTreeMap<_, _>>()
39            .await;
40
41        $map.insert($key_literal.to_string(), Box::new(db_items));
42    };
43}
44
45#[derive(Debug, serde::Serialize)]
46struct SerdeWrapper(#[serde(with = "hex::serde")] Vec<u8>);
47
48impl SerdeWrapper {
49    fn from_encodable<T: Encodable>(e: &T) -> SerdeWrapper {
50        SerdeWrapper(e.consensus_encode_to_vec())
51    }
52}
53
54/// Structure to hold the deserialized structs from the database.
55/// Also includes metadata on which sections of the database to read.
56pub struct DatabaseDump {
57    serialized: BTreeMap<String, Box<dyn Serialize>>,
58    read_only_db: Database,
59    modules: Vec<String>,
60    prefixes: Vec<String>,
61    server_cfg: Option<ServerConfig>,
62    module_inits: ServerModuleInitRegistry,
63    client_cfg: Option<ClientConfig>,
64    client_module_inits: ClientModuleInitRegistry,
65}
66
67impl DatabaseDump {
68    pub async fn new(
69        cfg_dir: PathBuf,
70        data_dir: String,
71        module_inits: ServerModuleInitRegistry,
72        client_module_inits: ClientModuleInitRegistry,
73        modules: Vec<String>,
74        prefixes: Vec<String>,
75    ) -> anyhow::Result<DatabaseDump> {
76        let Ok(read_only_rocks_db) = RocksDbReadOnly::open_read_only(data_dir.clone()).await else {
77            panic!("Error reading RocksDB database. Quitting...");
78        };
79
80        let read_only_db = Database::new(read_only_rocks_db, ModuleRegistry::default());
81
82        let (server_cfg, client_cfg, decoders) = if let Ok(cfg) = read_server_config(&cfg_dir) {
83            // Successfully read the server's config, that means this database is a server
84            // db
85            let decoders = module_inits
86                .available_decoders(cfg.iter_module_instances())
87                .with_fallback();
88            (Some(cfg), None, decoders)
89        } else {
90            // Check if this database is a client database by reading the `ClientConfig`
91            // from the database.
92
93            let mut dbtx = read_only_db.begin_transaction_nc().await;
94            let client_cfg_or = dbtx.get_value(&ClientConfigKey).await;
95
96            match client_cfg_or {
97                Some(client_cfg) => {
98                    // Successfully read the client config, that means this database is a client db
99                    let kinds = client_cfg.modules.iter().map(|(k, v)| (*k, &v.kind));
100                    let decoders = client_module_inits
101                        .available_decoders(kinds)
102                        .with_fallback();
103                    let client_cfg = client_cfg.redecode_raw(&decoders)?;
104                    (None, Some(client_cfg), decoders)
105                }
106                _ => (None, None, ModuleDecoderRegistry::default()),
107            }
108        };
109
110        Ok(DatabaseDump {
111            serialized: BTreeMap::new(),
112            read_only_db: read_only_db.with_decoders(decoders),
113            modules,
114            prefixes,
115            server_cfg,
116            module_inits,
117            client_module_inits,
118            client_cfg,
119        })
120    }
121}
122
123impl DatabaseDump {
124    /// Prints the contents of the `BTreeMap` to a pretty JSON string
125    fn print_database(&self) {
126        let json = serde_json::to_string_pretty(&self.serialized).unwrap();
127        println!("{json}");
128    }
129
130    async fn serialize_module(
131        &mut self,
132        module_id: &u16,
133        kind: &ModuleKind,
134        inits: CommonModuleInitRegistry,
135    ) -> anyhow::Result<()> {
136        if !self.modules.is_empty() && !self.modules.contains(&kind.to_string()) {
137            return Ok(());
138        }
139        let mut dbtx = self.read_only_db.begin_transaction_nc().await;
140        let db_version = dbtx.get_value(&DatabaseVersionKey(*module_id)).await;
141        let mut isolated_dbtx = dbtx.to_ref_with_prefix_module_id(*module_id).0;
142
143        match inits.get(kind) {
144            None => {
145                tracing::warn!(module_id, %kind, "Detected configuration for unsupported module");
146
147                let mut module_serialized = BTreeMap::new();
148                let filtered_prefixes = (0u8..=255).filter(|f| {
149                    self.prefixes.is_empty()
150                        || self.prefixes.contains(&f.to_string().to_lowercase())
151                });
152
153                let isolated_dbtx = &mut isolated_dbtx;
154
155                for prefix in filtered_prefixes {
156                    let db_items = isolated_dbtx
157                        .raw_find_by_prefix(&[prefix])
158                        .await?
159                        .map(|(k, v)| {
160                            (
161                                k.consensus_encode_to_hex(),
162                                Box::new(v.consensus_encode_to_hex()),
163                            )
164                        })
165                        .collect::<BTreeMap<String, Box<_>>>()
166                        .await;
167
168                    module_serialized.extend(db_items);
169                }
170                self.serialized
171                    .insert(format!("{kind}-{module_id}"), Box::new(module_serialized));
172            }
173            Some(init) => {
174                let mut module_serialized = init
175                    .dump_database(&mut isolated_dbtx.to_ref_nc(), self.prefixes.clone())
176                    .await
177                    .collect::<BTreeMap<String, _>>();
178
179                if let Some(db_version) = db_version {
180                    module_serialized.insert("Version".to_string(), Box::new(db_version));
181                } else {
182                    module_serialized
183                        .insert("Version".to_string(), Box::new("Not Specified".to_string()));
184                }
185
186                self.serialized
187                    .insert(format!("{kind}-{module_id}"), Box::new(module_serialized));
188            }
189        }
190
191        Ok(())
192    }
193
194    async fn serialize_gateway(&mut self) -> anyhow::Result<()> {
195        let mut dbtx = self.read_only_db.begin_transaction_nc().await;
196        let gateway_serialized = dbtx.dump_database(self.prefixes.clone()).await;
197        self.serialized
198            .insert("gateway".to_string(), Box::new(gateway_serialized));
199        Ok(())
200    }
201
202    /// Iterates through all the specified ranges in the database and retrieves
203    /// the data for each range. Prints serialized contents at the end.
204    pub async fn dump_database(&mut self) -> anyhow::Result<()> {
205        if let Some(cfg) = self.server_cfg.clone() {
206            if self.modules.is_empty() || self.modules.contains(&"consensus".to_string()) {
207                self.retrieve_consensus_data().await;
208            }
209
210            for (module_id, module_cfg) in &cfg.consensus.modules {
211                let kind = &module_cfg.kind;
212                self.serialize_module(module_id, kind, self.module_inits.to_common())
213                    .await?;
214            }
215
216            self.print_database();
217            return Ok(());
218        }
219
220        if let Some(cfg) = self.client_cfg.clone() {
221            self.serialized
222                .insert("Client Config".into(), Box::new(cfg.to_json()));
223
224            for (module_id, module_cfg) in &cfg.modules {
225                let kind = &module_cfg.kind;
226                let mut modules = Vec::new();
227                if let Some(module) = self.client_module_inits.get(kind) {
228                    modules.push(module.to_dyn_common());
229                }
230
231                let registry = CommonModuleInitRegistry::from(modules);
232                self.serialize_module(module_id, kind, registry).await?;
233            }
234
235            {
236                let mut dbtx = self.read_only_db.begin_transaction_nc().await;
237                Self::write_serialized_client_operation_log(&mut self.serialized, &mut dbtx).await;
238            }
239
240            self.print_database();
241            return Ok(());
242        }
243
244        self.serialize_gateway().await?;
245        self.print_database();
246
247        Ok(())
248    }
249
250    /// Iterates through each of the prefixes within the consensus range and
251    /// retrieves the corresponding data.
252    async fn retrieve_consensus_data(&mut self) {
253        let filtered_prefixes = server_db::DbKeyPrefix::iter().filter(|prefix| {
254            self.prefixes.is_empty() || self.prefixes.contains(&prefix.to_string().to_lowercase())
255        });
256        let mut dbtx = self.read_only_db.begin_transaction_nc().await;
257        let mut consensus: BTreeMap<String, Box<dyn Serialize>> = BTreeMap::new();
258
259        for table in filtered_prefixes {
260            Self::write_serialized_consensus_range(table, &mut dbtx, &mut consensus).await;
261        }
262
263        self.serialized
264            .insert("Consensus".to_string(), Box::new(consensus));
265    }
266
267    async fn write_serialized_consensus_range(
268        table: server_db::DbKeyPrefix,
269        dbtx: &mut DatabaseTransaction<'_>,
270        consensus: &mut BTreeMap<String, Box<dyn Serialize>>,
271    ) {
272        match table {
273            server_db::DbKeyPrefix::AcceptedItem => {
274                push_db_pair_items_no_serde!(
275                    dbtx,
276                    consensus_db::AcceptedItemPrefix,
277                    server_db::AcceptedItemKey,
278                    fedimint_server::consensus::AcceptedItem,
279                    consensus,
280                    "Accepted Items"
281                );
282            }
283            server_db::DbKeyPrefix::AcceptedTransaction => {
284                push_db_pair_items_no_serde!(
285                    dbtx,
286                    consensus_db::AcceptedTransactionKeyPrefix,
287                    server_db::AcceptedTransactionKey,
288                    fedimint_server::consensus::AcceptedTransaction,
289                    consensus,
290                    "Accepted Transactions"
291                );
292            }
293            server_db::DbKeyPrefix::SignedSessionOutcome => {
294                push_db_pair_items_no_serde!(
295                    dbtx,
296                    consensus_db::SignedSessionOutcomePrefix,
297                    server_db::SignedBlockKey,
298                    fedimint_server::consensus::SignedBlock,
299                    consensus,
300                    "Signed Blocks"
301                );
302            }
303            server_db::DbKeyPrefix::AlephUnits => {
304                push_db_pair_items_no_serde!(
305                    dbtx,
306                    consensus_db::AlephUnitsPrefix,
307                    server_db::AlephUnitsKey,
308                    Vec<u8>,
309                    consensus,
310                    "Aleph Units"
311                );
312            }
313            // Module is a global prefix for all module data
314            server_db::DbKeyPrefix::Module
315            | server_db::DbKeyPrefix::ServerInfo
316            | server_db::DbKeyPrefix::DatabaseVersion
317            | server_db::DbKeyPrefix::ClientBackup => {}
318            server_db::DbKeyPrefix::ApiAnnouncements => {
319                push_db_pair_items_no_serde!(
320                    dbtx,
321                    ApiAnnouncementPrefix,
322                    ApiAnnouncementKey,
323                    fedimint_core::net::api_announcement::SignedApiAnnouncement,
324                    consensus,
325                    "API Announcements"
326                );
327            }
328            server_db::DbKeyPrefix::GuardianMetadata => {
329                push_db_pair_items_no_serde!(
330                    dbtx,
331                    fedimint_server::net::api::guardian_metadata::GuardianMetadataPrefix,
332                    fedimint_server::net::api::guardian_metadata::GuardianMetadataKey,
333                    fedimint_core::net::guardian_metadata::SignedGuardianMetadata,
334                    consensus,
335                    "Guardian Metadata"
336                );
337            }
338        }
339    }
340    async fn write_serialized_client_operation_log(
341        serialized: &mut BTreeMap<String, Box<dyn Serialize>>,
342        dbtx: &mut DatabaseTransaction<'_>,
343    ) {
344        push_db_pair_items!(
345            dbtx,
346            OperationLogKeyPrefix,
347            OperationLogKey,
348            OperationLogEntry,
349            serialized,
350            "Operations"
351        );
352    }
353}
fedimint_dbtool/dump.rs

fedimint_dbtool/
dump.rs
