1#![allow(clippy::disallowed_types)]
2pub(crate) mod jsonrpsee;
5
6use std::sync::LazyLock;
7use std::time::Duration;
8
9use fedimint_core::backup::ClientBackupKeyPrefix;
10use fedimint_core::db::{Database, IDatabaseTransactionOpsCoreTyped};
11use fedimint_core::task::{TaskGroup, sleep};
12use fedimint_metrics::prometheus::{
13 HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec,
14 register_histogram_vec_with_registry, register_int_counter_with_registry,
15 register_int_gauge_vec_with_registry, register_int_gauge_with_registry,
16};
17use fedimint_metrics::{
18 Histogram, REGISTRY, histogram_opts, opts, register_histogram_with_registry,
19 register_int_counter_vec_with_registry,
20};
21use futures::StreamExt as _;
22use tokio::sync::OnceCell;
23
24use crate::consensus::api::backup_statistics_static;
25
26const BACKUP_STATS_REFRESH_INTERVAL: Duration = Duration::from_mins(1);
27
28pub static TX_ELEMS_BUCKETS: LazyLock<Vec<f64>> = LazyLock::new(|| {
29 vec![
30 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0,
31 ]
32});
33pub(crate) static CONSENSUS_TX_PROCESSED_INPUTS: LazyLock<Histogram> = LazyLock::new(|| {
34 register_histogram_with_registry!(
35 histogram_opts!(
36 "consensus_tx_processed_inputs",
37 "Number of inputs processed in a transaction",
38 TX_ELEMS_BUCKETS.clone()
39 ),
40 REGISTRY
41 )
42 .unwrap()
43});
44pub(crate) static CONSENSUS_TX_PROCESSED_OUTPUTS: LazyLock<Histogram> = LazyLock::new(|| {
45 register_histogram_with_registry!(
46 histogram_opts!(
47 "consensus_tx_processed_outputs",
48 "Number of outputs processed in a transaction",
49 TX_ELEMS_BUCKETS.clone()
50 ),
51 REGISTRY
52 )
53 .unwrap()
54});
55pub(crate) static CONSENSUS_ITEMS_PROCESSED_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
56 register_int_counter_vec_with_registry!(
57 opts!(
58 "consensus_items_processed_total",
59 "Number of consensus items processed in the consensus",
60 ),
61 &["peer_id"],
62 REGISTRY
63 )
64 .unwrap()
65});
66pub(crate) static CONSENSUS_ITEM_PROCESSING_DURATION_SECONDS: LazyLock<HistogramVec> =
67 LazyLock::new(|| {
68 register_histogram_vec_with_registry!(
69 histogram_opts!(
70 "consensus_item_processing_duration_seconds",
71 "Duration of processing a consensus item",
72 ),
73 &["peer_id"],
74 REGISTRY
75 )
76 .unwrap()
77 });
78pub(crate) static CONSENSUS_ITEM_PROCESSING_MODULE_AUDIT_DURATION_SECONDS: LazyLock<HistogramVec> =
79 LazyLock::new(|| {
80 register_histogram_vec_with_registry!(
81 histogram_opts!(
82 "consensus_item_processing_module_audit_duration_seconds",
83 "Duration of processing a consensus item",
84 ),
85 &["module_id", "module_kind"],
86 REGISTRY
87 )
88 .unwrap()
89 });
90
91pub(crate) static CONSENSUS_ORDERING_LATENCY_SECONDS: LazyLock<Histogram> = LazyLock::new(|| {
92 register_histogram_with_registry!(
93 histogram_opts!(
94 "consensus_ordering_latency_seconds",
95 "Duration of ordering a batch of consensus items",
96 ),
97 REGISTRY
98 )
99 .unwrap()
100});
101
102pub(crate) static IROH_API_CONNECTIONS_ACTIVE: LazyLock<IntGauge> = LazyLock::new(|| {
103 register_int_gauge_with_registry!(
104 opts!(
105 "iroh_api_connections_active",
106 "Number of currently active iroh API connections",
107 ),
108 REGISTRY
109 )
110 .unwrap()
111});
112
113pub(crate) static IROH_API_CONNECTION_DURATION_SECONDS: LazyLock<Histogram> = LazyLock::new(|| {
114 register_histogram_with_registry!(
115 histogram_opts!(
116 "iroh_api_connection_duration_seconds",
117 "Duration of iroh API connections",
118 ),
119 REGISTRY
120 )
121 .unwrap()
122});
123
124pub(crate) static IROH_API_CONNECTION_IDLE_TIMEOUT_TOTAL: LazyLock<IntCounter> =
125 LazyLock::new(|| {
126 register_int_counter_with_registry!(
127 opts!(
128 "iroh_api_connection_idle_timeout_total",
129 "Number of iroh API connections closed by the server after being idle",
130 ),
131 REGISTRY
132 )
133 .unwrap()
134 });
135
136pub(crate) static IROH_API_REQUEST_DURATION_SECONDS: LazyLock<HistogramVec> = LazyLock::new(|| {
137 register_histogram_vec_with_registry!(
138 histogram_opts!(
139 "iroh_api_request_duration_seconds",
140 "Duration of processing an iroh API request",
141 ),
142 &["method"],
143 REGISTRY
144 )
145 .unwrap()
146});
147
148pub(crate) static IROH_API_REQUEST_RESPONSE_CODE: LazyLock<IntCounterVec> = LazyLock::new(|| {
149 register_int_counter_vec_with_registry!(
150 opts!(
151 "iroh_api_request_response_code_total",
152 "Count of iroh API response codes and types",
153 ),
154 &["method", "code", "type"],
155 REGISTRY
156 )
157 .unwrap()
158});
159
160pub(crate) static JSONRPC_API_REQUEST_DURATION_SECONDS: LazyLock<HistogramVec> =
161 LazyLock::new(|| {
162 register_histogram_vec_with_registry!(
163 histogram_opts!(
164 "jsonrpc_api_request_duration_seconds",
165 "Duration of processing an rpc request",
166 ),
167 &["method"],
168 REGISTRY
169 )
170 .unwrap()
171 });
172pub(crate) static JSONRPC_API_REQUEST_RESPONSE_CODE: LazyLock<IntCounterVec> =
173 LazyLock::new(|| {
174 register_int_counter_vec_with_registry!(
175 opts!(
176 "jsonrpc_api_request_response_code_total",
177 "Count of response counts and types",
178 ),
179 &["method", "code", "type"],
180 REGISTRY
181 )
182 .unwrap()
183 });
184pub(crate) static CONSENSUS_SESSION_COUNT: LazyLock<IntGauge> = LazyLock::new(|| {
185 register_int_gauge_with_registry!(
186 opts!(
187 "consensus_session_count",
188 "Fedimint consensus session count",
189 ),
190 REGISTRY
191 )
192 .unwrap()
193});
194pub(crate) static CONSENSUS_PEER_CONTRIBUTION_SESSION_IDX: LazyLock<IntGaugeVec> =
195 LazyLock::new(|| {
196 register_int_gauge_vec_with_registry!(
197 opts!(
198 "consensus_peer_contribution_session_idx",
199 "Latest contribution session idx by peer_id",
200 ),
201 &["self_id", "peer_id"],
202 REGISTRY
203 )
204 .unwrap()
205 });
206pub(crate) static BACKUP_WRITE_SIZE_BYTES: LazyLock<Histogram> = LazyLock::new(|| {
207 register_histogram_with_registry!(
208 histogram_opts!(
209 "backup_write_size_bytes",
210 "Size of every backup being written",
211 vec![
212 1.0, 10., 100., 1_000., 5_000., 10_000., 50_000., 100_000., 1_000_000.
213 ]
214 ),
215 REGISTRY
216 )
217 .unwrap()
218});
219pub(crate) static STORED_BACKUPS_COUNT: LazyLock<IntGauge> = LazyLock::new(|| {
220 register_int_gauge_with_registry!(
221 opts!("stored_backups_count", "Total amount of backups stored",),
222 REGISTRY
223 )
224 .unwrap()
225});
226
227pub(crate) static BACKUP_COUNTS: LazyLock<IntGaugeVec> = LazyLock::new(|| {
228 register_int_gauge_vec_with_registry!(
229 opts!(
230 "backup_counts",
231 "Backups refreshed at least once in a given timeframe",
232 ),
233 &["timeframe"],
234 REGISTRY
235 )
236 .unwrap()
237});
238
239pub(crate) static TOTAL_BACKUP_SIZE: LazyLock<IntGauge> = LazyLock::new(|| {
240 register_int_gauge_with_registry!(
241 opts!("total_backup_size", "Total size og backups in the DB",),
242 REGISTRY
243 )
244 .unwrap()
245});
246
247static BACKUP_COUNTS_UPDATE_TASK: OnceCell<()> = OnceCell::const_new();
250
251pub(crate) static PEER_CONNECT_COUNT: LazyLock<IntCounterVec> = LazyLock::new(|| {
252 register_int_counter_vec_with_registry!(
253 opts!("peer_connect_total", "Number of times peer (re/)connected",),
254 &["self_id", "peer_id", "direction"],
255 REGISTRY
256 )
257 .unwrap()
258});
259pub(crate) static PEER_DISCONNECT_COUNT: LazyLock<IntCounterVec> = LazyLock::new(|| {
260 register_int_counter_vec_with_registry!(
261 opts!(
262 "peer_disconnect_total",
263 "Number of times peer (re/)connected",
264 ),
265 &["self_id", "peer_id"],
266 REGISTRY
267 )
268 .unwrap()
269});
270pub(crate) static PEER_MESSAGES_COUNT: LazyLock<IntCounterVec> = LazyLock::new(|| {
271 register_int_counter_vec_with_registry!(
272 opts!("peer_messages_total", "Messages with the peer",),
273 &["self_id", "peer_id", "direction"],
274 REGISTRY
275 )
276 .unwrap()
277});
278
279pub(crate) async fn initialize_gauge_metrics(tg: &TaskGroup, db: &Database) {
282 STORED_BACKUPS_COUNT.set(
283 db.begin_transaction_nc()
284 .await
285 .find_by_prefix(&ClientBackupKeyPrefix)
286 .await
287 .count()
288 .await as i64,
289 );
290
291 let db_inner = db.clone();
292 BACKUP_COUNTS_UPDATE_TASK
293 .get_or_init(move || async move {
294 tg.spawn_cancellable("prometheus_backup_stats", async move {
295 loop {
296 let backup_counts =
297 backup_statistics_static(&mut db_inner.begin_transaction_nc().await).await;
298
299 BACKUP_COUNTS.with_label_values(&["1d"]).set(
300 backup_counts
301 .refreshed_1d
302 .try_into()
303 .expect("u64 to i64 overflow"),
304 );
305 BACKUP_COUNTS.with_label_values(&["1w"]).set(
306 backup_counts
307 .refreshed_1w
308 .try_into()
309 .expect("u64 to i64 overflow"),
310 );
311 BACKUP_COUNTS.with_label_values(&["1m"]).set(
312 backup_counts
313 .refreshed_1m
314 .try_into()
315 .expect("u64 to i64 overflow"),
316 );
317 BACKUP_COUNTS.with_label_values(&["3m"]).set(
318 backup_counts
319 .refreshed_3m
320 .try_into()
321 .expect("u64 to i64 overflow"),
322 );
323 BACKUP_COUNTS.with_label_values(&["all_time"]).set(
324 backup_counts
325 .num_backups
326 .try_into()
327 .expect("u64 to i64 overflow"),
328 );
329
330 TOTAL_BACKUP_SIZE.set(
331 backup_counts
332 .total_size
333 .try_into()
334 .expect("u64 to i64 overflow"),
335 );
336
337 sleep(BACKUP_STATS_REFRESH_INTERVAL).await;
338 }
339 });
340 })
341 .await;
342}