fedimint_dbtool/
dump.rs

1use std::collections::BTreeMap;
2use std::path::PathBuf;
3
4use erased_serde::Serialize;
5use fedimint_client::db::{ClientConfigKey, OperationLogKeyPrefix};
6use fedimint_client::module_init::ClientModuleInitRegistry;
7use fedimint_client_module::oplog::OperationLogEntry;
8use fedimint_core::config::{ClientConfig, CommonModuleInitRegistry};
9use fedimint_core::core::ModuleKind;
10use fedimint_core::db::{
11    Database, DatabaseTransaction, DatabaseVersionKey, IDatabaseTransactionOpsCore,
12    IDatabaseTransactionOpsCoreTyped,
13};
14use fedimint_core::encoding::Encodable;
15use fedimint_core::module::registry::{ModuleDecoderRegistry, ModuleRegistry};
16use fedimint_core::push_db_pair_items;
17use fedimint_gateway_server_db::GatewayDbtxNcExt as _;
18use fedimint_rocksdb::RocksDbReadOnly;
19use fedimint_server::config::ServerConfig;
20use fedimint_server::config::io::read_server_config;
21use fedimint_server::consensus::db as consensus_db;
22use fedimint_server::core::{ServerModuleInitRegistry, ServerModuleInitRegistryExt};
23use fedimint_server::db as server_db;
24use fedimint_server::net::api::announcement::ApiAnnouncementPrefix;
25use futures::StreamExt;
26use strum::IntoEnumIterator;
27
28macro_rules! push_db_pair_items_no_serde {
29    ($dbtx:ident, $prefix_type:expr_2021, $key_type:ty, $value_type:ty, $map:ident, $key_literal:literal) => {
30        let db_items = IDatabaseTransactionOpsCoreTyped::find_by_prefix($dbtx, &$prefix_type)
31            .await
32            .map(|(key, val)| {
33                (
34                    Encodable::consensus_encode_to_hex(&key),
35                    SerdeWrapper::from_encodable(&val),
36                )
37            })
38            .collect::<BTreeMap<_, _>>()
39            .await;
40
41        $map.insert($key_literal.to_string(), Box::new(db_items));
42    };
43}
44
45#[derive(Debug, serde::Serialize)]
46struct SerdeWrapper(#[serde(with = "hex::serde")] Vec<u8>);
47
48impl SerdeWrapper {
49    fn from_encodable<T: Encodable>(e: &T) -> SerdeWrapper {
50        SerdeWrapper(e.consensus_encode_to_vec())
51    }
52}
53
54/// Structure to hold the deserialized structs from the database.
55/// Also includes metadata on which sections of the database to read.
56pub struct DatabaseDump {
57    serialized: BTreeMap<String, Box<dyn Serialize>>,
58    read_only_db: Database,
59    modules: Vec<String>,
60    prefixes: Vec<String>,
61    server_cfg: Option<ServerConfig>,
62    module_inits: ServerModuleInitRegistry,
63    client_cfg: Option<ClientConfig>,
64    client_module_inits: ClientModuleInitRegistry,
65}
66
67impl DatabaseDump {
68    pub async fn new(
69        cfg_dir: PathBuf,
70        data_dir: String,
71        module_inits: ServerModuleInitRegistry,
72        client_module_inits: ClientModuleInitRegistry,
73        modules: Vec<String>,
74        prefixes: Vec<String>,
75    ) -> anyhow::Result<DatabaseDump> {
76        let Ok(read_only_rocks_db) = RocksDbReadOnly::open_read_only(data_dir.clone()).await else {
77            panic!("Error reading RocksDB database. Quitting...");
78        };
79
80        let read_only_db = Database::new(read_only_rocks_db, ModuleRegistry::default());
81
82        let (server_cfg, client_cfg, decoders) = if let Ok(cfg) = read_server_config(&cfg_dir) {
83            // Successfully read the server's config, that means this database is a server
84            // db
85            let decoders = module_inits
86                .available_decoders(cfg.iter_module_instances())
87                .unwrap()
88                .with_fallback();
89            (Some(cfg), None, decoders)
90        } else {
91            // Check if this database is a client database by reading the `ClientConfig`
92            // from the database.
93
94            let mut dbtx = read_only_db.begin_transaction_nc().await;
95            let client_cfg_or = dbtx.get_value(&ClientConfigKey).await;
96
97            match client_cfg_or {
98                Some(client_cfg) => {
99                    // Successfully read the client config, that means this database is a client db
100                    let kinds = client_cfg.modules.iter().map(|(k, v)| (*k, &v.kind));
101                    let decoders = client_module_inits
102                        .available_decoders(kinds)
103                        .unwrap()
104                        .with_fallback();
105                    let client_cfg = client_cfg.redecode_raw(&decoders)?;
106                    (None, Some(client_cfg), decoders)
107                }
108                _ => (None, None, ModuleDecoderRegistry::default()),
109            }
110        };
111
112        Ok(DatabaseDump {
113            serialized: BTreeMap::new(),
114            read_only_db: read_only_db.with_decoders(decoders),
115            modules,
116            prefixes,
117            server_cfg,
118            module_inits,
119            client_module_inits,
120            client_cfg,
121        })
122    }
123}
124
125impl DatabaseDump {
126    /// Prints the contents of the `BTreeMap` to a pretty JSON string
127    fn print_database(&self) {
128        let json = serde_json::to_string_pretty(&self.serialized).unwrap();
129        println!("{json}");
130    }
131
132    async fn serialize_module(
133        &mut self,
134        module_id: &u16,
135        kind: &ModuleKind,
136        inits: CommonModuleInitRegistry,
137    ) -> anyhow::Result<()> {
138        if !self.modules.is_empty() && !self.modules.contains(&kind.to_string()) {
139            return Ok(());
140        }
141        let mut dbtx = self.read_only_db.begin_transaction_nc().await;
142        let db_version = dbtx.get_value(&DatabaseVersionKey(*module_id)).await;
143        let mut isolated_dbtx = dbtx.to_ref_with_prefix_module_id(*module_id).0;
144
145        match inits.get(kind) {
146            None => {
147                tracing::warn!(module_id, %kind, "Detected configuration for unsupported module");
148
149                let mut module_serialized = BTreeMap::new();
150                let filtered_prefixes = (0u8..=255).filter(|f| {
151                    self.prefixes.is_empty()
152                        || self.prefixes.contains(&f.to_string().to_lowercase())
153                });
154
155                let isolated_dbtx = &mut isolated_dbtx;
156
157                for prefix in filtered_prefixes {
158                    let db_items = isolated_dbtx
159                        .raw_find_by_prefix(&[prefix])
160                        .await?
161                        .map(|(k, v)| {
162                            (
163                                k.consensus_encode_to_hex(),
164                                Box::new(v.consensus_encode_to_hex()),
165                            )
166                        })
167                        .collect::<BTreeMap<String, Box<_>>>()
168                        .await;
169
170                    module_serialized.extend(db_items);
171                }
172                self.serialized
173                    .insert(format!("{kind}-{module_id}"), Box::new(module_serialized));
174            }
175            Some(init) => {
176                let mut module_serialized = init
177                    .dump_database(&mut isolated_dbtx.to_ref_nc(), self.prefixes.clone())
178                    .await
179                    .collect::<BTreeMap<String, _>>();
180
181                if let Some(db_version) = db_version {
182                    module_serialized.insert("Version".to_string(), Box::new(db_version));
183                } else {
184                    module_serialized
185                        .insert("Version".to_string(), Box::new("Not Specified".to_string()));
186                }
187
188                self.serialized
189                    .insert(format!("{kind}-{module_id}"), Box::new(module_serialized));
190            }
191        }
192
193        Ok(())
194    }
195
196    async fn serialize_gateway(&mut self) -> anyhow::Result<()> {
197        let mut dbtx = self.read_only_db.begin_transaction_nc().await;
198        let gateway_serialized = dbtx.dump_database(self.prefixes.clone()).await;
199        self.serialized
200            .insert("gateway".to_string(), Box::new(gateway_serialized));
201        Ok(())
202    }
203
204    /// Iterates through all the specified ranges in the database and retrieves
205    /// the data for each range. Prints serialized contents at the end.
206    pub async fn dump_database(&mut self) -> anyhow::Result<()> {
207        if let Some(cfg) = self.server_cfg.clone() {
208            if self.modules.is_empty() || self.modules.contains(&"consensus".to_string()) {
209                self.retrieve_consensus_data().await;
210            }
211
212            for (module_id, module_cfg) in &cfg.consensus.modules {
213                let kind = &module_cfg.kind;
214                self.serialize_module(module_id, kind, self.module_inits.to_common())
215                    .await?;
216            }
217
218            self.print_database();
219            return Ok(());
220        }
221
222        if let Some(cfg) = self.client_cfg.clone() {
223            self.serialized
224                .insert("Client Config".into(), Box::new(cfg.to_json()));
225
226            for (module_id, module_cfg) in &cfg.modules {
227                let kind = &module_cfg.kind;
228                let mut modules = Vec::new();
229                if let Some(module) = self.client_module_inits.get(kind) {
230                    modules.push(module.to_dyn_common());
231                }
232
233                let registry = CommonModuleInitRegistry::from(modules);
234                self.serialize_module(module_id, kind, registry).await?;
235            }
236
237            {
238                let mut dbtx = self.read_only_db.begin_transaction_nc().await;
239                Self::write_serialized_client_operation_log(&mut self.serialized, &mut dbtx).await;
240            }
241
242            self.print_database();
243            return Ok(());
244        }
245
246        self.serialize_gateway().await?;
247        self.print_database();
248
249        Ok(())
250    }
251
252    /// Iterates through each of the prefixes within the consensus range and
253    /// retrieves the corresponding data.
254    async fn retrieve_consensus_data(&mut self) {
255        let filtered_prefixes = server_db::DbKeyPrefix::iter().filter(|prefix| {
256            self.prefixes.is_empty() || self.prefixes.contains(&prefix.to_string().to_lowercase())
257        });
258        let mut dbtx = self.read_only_db.begin_transaction_nc().await;
259        let mut consensus: BTreeMap<String, Box<dyn Serialize>> = BTreeMap::new();
260
261        for table in filtered_prefixes {
262            Self::write_serialized_consensus_range(table, &mut dbtx, &mut consensus).await;
263        }
264
265        self.serialized
266            .insert("Consensus".to_string(), Box::new(consensus));
267    }
268
269    async fn write_serialized_consensus_range(
270        table: server_db::DbKeyPrefix,
271        dbtx: &mut DatabaseTransaction<'_>,
272        consensus: &mut BTreeMap<String, Box<dyn Serialize>>,
273    ) {
274        match table {
275            server_db::DbKeyPrefix::AcceptedItem => {
276                push_db_pair_items_no_serde!(
277                    dbtx,
278                    consensus_db::AcceptedItemPrefix,
279                    server_db::AcceptedItemKey,
280                    fedimint_server::consensus::AcceptedItem,
281                    consensus,
282                    "Accepted Items"
283                );
284            }
285            server_db::DbKeyPrefix::AcceptedTransaction => {
286                push_db_pair_items_no_serde!(
287                    dbtx,
288                    consensus_db::AcceptedTransactionKeyPrefix,
289                    server_db::AcceptedTransactionKey,
290                    fedimint_server::consensus::AcceptedTransaction,
291                    consensus,
292                    "Accepted Transactions"
293                );
294            }
295            server_db::DbKeyPrefix::SignedSessionOutcome => {
296                push_db_pair_items_no_serde!(
297                    dbtx,
298                    consensus_db::SignedSessionOutcomePrefix,
299                    server_db::SignedBlockKey,
300                    fedimint_server::consensus::SignedBlock,
301                    consensus,
302                    "Signed Blocks"
303                );
304            }
305            server_db::DbKeyPrefix::AlephUnits => {
306                push_db_pair_items_no_serde!(
307                    dbtx,
308                    consensus_db::AlephUnitsPrefix,
309                    server_db::AlephUnitsKey,
310                    Vec<u8>,
311                    consensus,
312                    "Aleph Units"
313                );
314            }
315            // Module is a global prefix for all module data
316            server_db::DbKeyPrefix::Module
317            | server_db::DbKeyPrefix::ServerInfo
318            | server_db::DbKeyPrefix::DatabaseVersion
319            | server_db::DbKeyPrefix::ClientBackup => {}
320            server_db::DbKeyPrefix::ApiAnnouncements => {
321                push_db_pair_items_no_serde!(
322                    dbtx,
323                    ApiAnnouncementPrefix,
324                    ApiAnnouncementKey,
325                    fedimint_core::net::api_announcement::SignedApiAnnouncement,
326                    consensus,
327                    "API Announcements"
328                );
329            }
330            server_db::DbKeyPrefix::GuardianMetadata => {
331                push_db_pair_items_no_serde!(
332                    dbtx,
333                    fedimint_server::net::api::guardian_metadata::GuardianMetadataPrefix,
334                    fedimint_server::net::api::guardian_metadata::GuardianMetadataKey,
335                    fedimint_core::net::guardian_metadata::SignedGuardianMetadata,
336                    consensus,
337                    "Guardian Metadata"
338                );
339            }
340        }
341    }
342    async fn write_serialized_client_operation_log(
343        serialized: &mut BTreeMap<String, Box<dyn Serialize>>,
344        dbtx: &mut DatabaseTransaction<'_>,
345    ) {
346        push_db_pair_items!(
347            dbtx,
348            OperationLogKeyPrefix,
349            OperationLogKey,
350            OperationLogEntry,
351            serialized,
352            "Operations"
353        );
354    }
355}
fedimint_dbtool/dump.rs

fedimint_dbtool/
dump.rs
