1#![deny(clippy::pedantic)]
2#![allow(clippy::cast_possible_truncation)]
3#![allow(clippy::cast_precision_loss)]
4#![allow(clippy::missing_errors_doc)]
5#![allow(clippy::ref_option)]
6#![allow(clippy::too_many_lines)]
7#![allow(clippy::large_futures)]
8
9use std::collections::{BTreeMap, HashMap};
10use std::path::PathBuf;
11use std::str::FromStr;
12use std::time::Duration;
13use std::vec;
14
15use anyhow::{Context, anyhow, bail};
16use clap::{Args, Parser, Subcommand, ValueEnum};
17use common::{
18 gateway_pay_invoice, get_note_summary, ldk_create_invoice, ldk_pay_invoice,
19 ldk_wait_invoice_payment, parse_gateway_id, reissue_notes,
20};
21use devimint::cmd;
22use devimint::util::GatewayLndCli;
23use devimint::version_constants::VERSION_0_10_0_ALPHA;
24use fedimint_api_client::download_from_invite_code;
25use fedimint_client::ClientHandleArc;
26use fedimint_connectors::ConnectorRegistry;
27use fedimint_core::Amount;
28use fedimint_core::core::Account;
29use fedimint_core::endpoint_constants::SESSION_COUNT_ENDPOINT;
30use fedimint_core::invite_code::InviteCode;
31use fedimint_core::module::ApiRequestErased;
32use fedimint_core::runtime::spawn;
33use fedimint_core::util::{BoxFuture, SafeUrl};
34use fedimint_ln_client::{LightningClientModule, LnReceiveState};
35use fedimint_ln_common::LightningGateway;
36use fedimint_mint_client::OOBNotes;
37use futures::StreamExt;
38use lightning_invoice::{Bolt11Invoice, Bolt11InvoiceDescription, Description};
39use serde::{Deserialize, Serialize};
40use tokio::fs::OpenOptions;
41use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufWriter};
42use tokio::sync::mpsc;
43use tracing::{debug, info, warn};
44
45use crate::common::{
46 build_client, do_spend_notes, get_invite_code_cli, remint_denomination, try_get_notes_cli,
47};
48pub mod common;
49#[cfg(test)]
50mod tests;
51
52#[derive(Parser, Clone)]
53#[command(version)]
54struct Opts {
55 #[arg(
56 long,
57 default_value = "10",
58 help = "Number of users. Each user will work in parallel"
59 )]
60 users: u16,
61
62 #[arg(long, help = "Output with the metrics results in JSON format")]
63 metrics_json_output: Option<PathBuf>,
64
65 #[arg(
66 long,
67 help = "If given, will be used to store and retrieve past metrics for comparison purposes"
68 )]
69 archive_dir: Option<PathBuf>,
70
71 #[clap(subcommand)]
72 command: Command,
73}
74
75#[derive(Debug, Clone, Copy, ValueEnum)]
76enum LnInvoiceGeneration {
77 LdkLightningCli,
78}
79
80#[derive(Subcommand, Clone)]
81enum Command {
82 #[command(about = "Keep many websocket connections to a federation for a duration of time")]
83 TestConnect {
84 #[arg(long, help = "Federation invite code")]
85 invite_code: String,
86 #[arg(
87 long,
88 default_value = "60",
89 help = "How much time to keep the connections open, in seconds"
90 )]
91 duration_secs: u64,
92 #[arg(
93 long,
94 default_value = "120",
95 help = "Timeout for connection attempt and for each request, in secnods"
96 )]
97 timeout_secs: u64,
98 #[arg(
99 long,
100 help = "If given, will limit the number of endpoints (guardians) to connect to"
101 )]
102 limit_endpoints: Option<usize>,
103 },
104 #[command(about = "Try to download the client config many times.")]
105 TestDownload {
106 #[arg(long, help = "Federation invite code")]
107 invite_code: String,
108 },
109 #[command(
110 about = "Run a load test where many users in parallel will try to reissue notes and pay invoices through the gateway"
111 )]
112 LoadTest(LoadTestArgs),
113 #[command()]
118 LnCircularLoadTest(LnCircularLoadTestArgs),
119}
120
121#[derive(Args, Clone)]
122struct LoadTestArgs {
123 #[arg(
124 long,
125 help = "Federation invite code. If none given, we assume the client already has a config downloaded in DB"
126 )]
127 invite_code: Option<InviteCode>,
128
129 #[arg(
130 long,
131 help = "Notes for the test. If none and no funds on archive, will call fedimint-cli spend"
132 )]
133 initial_notes: Option<OOBNotes>,
134
135 #[arg(
136 long,
137 help = "Gateway Id. If none, retrieve one according to --generate-invoice-with"
138 )]
139 gateway_id: Option<String>,
140
141 #[arg(
142 long,
143 help = "The method used to generate invoices to be paid through the gateway. If none and no --invoices-file provided, no gateway/LN tests will be run. Note that you can't generate an invoice using the same lightning node used by the gateway (i.e self payment is forbidden)"
144 )]
145 generate_invoice_with: Option<LnInvoiceGeneration>,
146
147 #[arg(
148 long,
149 default_value = "1",
150 help = "How many invoices will be created for each user. Only applicable if --generate-invoice-with is provided"
151 )]
152 invoices_per_user: u16,
153
154 #[arg(
155 long,
156 default_value = "0",
157 help = "How many seconds to sleep between LN payments"
158 )]
159 ln_payment_sleep_secs: u64,
160
161 #[arg(
162 long,
163 help = "A text file with one invoice per line. If --generate-invoice-with is provided, these will be additional invoices to be paid"
164 )]
165 invoices_file: Option<PathBuf>,
166
167 #[arg(
168 long,
169 help = "How many notes to distribute to each user",
170 default_value = "2"
171 )]
172 notes_per_user: u16,
173
174 #[arg(
175 long,
176 help = "Note denomination to use for the test",
177 default_value = "1024"
178 )]
179 note_denomination: Amount,
180
181 #[arg(
182 long,
183 help = "Invoice amount when generating one",
184 default_value = "1000"
185 )]
186 invoice_amount: Amount,
187}
188
189#[derive(Args, Clone)]
190struct LnCircularLoadTestArgs {
191 #[arg(
192 long,
193 help = "Federation invite code. If none given, we assume the client already has a config downloaded in DB"
194 )]
195 invite_code: Option<InviteCode>,
196
197 #[arg(
198 long,
199 help = "Notes for the test. If none and no funds on archive, will call fedimint-cli spend"
200 )]
201 initial_notes: Option<OOBNotes>,
202
203 #[arg(
204 long,
205 default_value = "60",
206 help = "For how many seconds to run the test"
207 )]
208 test_duration_secs: u64,
209
210 #[arg(
211 long,
212 default_value = "0",
213 help = "How many seconds to sleep between LN payments"
214 )]
215 ln_payment_sleep_secs: u64,
216
217 #[arg(
218 long,
219 help = "How many notes to distribute to each user",
220 default_value = "1"
221 )]
222 notes_per_user: u16,
223
224 #[arg(
225 long,
226 help = "Note denomination to use for the test",
227 default_value = "1024"
228 )]
229 note_denomination: Amount,
230
231 #[arg(
232 long,
233 help = "Invoice amount when generating one",
234 default_value = "1000"
235 )]
236 invoice_amount: Amount,
237
238 #[arg(long)]
239 strategy: LnCircularStrategy,
240}
241
242#[derive(Debug, Clone, Copy, ValueEnum)]
243enum LnCircularStrategy {
244 SelfPayment,
246 TwoGateways,
249 PartnerPingPong,
251}
252
253#[derive(Debug, Clone)]
254pub struct MetricEvent {
255 name: String,
256 duration: Duration,
257}
258
259#[derive(Debug, Clone, Serialize, Deserialize)]
260struct EventMetricSummary {
261 name: String,
262 users: u64,
263 n: u64,
264 avg_ms: u128,
265 median_ms: u128,
266 max_ms: u128,
267 min_ms: u128,
268 timestamp_seconds: u64,
269}
270
271#[derive(Debug, Serialize, Deserialize)]
272struct EventMetricComparison {
273 avg_ms_gain: f64,
274 median_ms_gain: f64,
275 max_ms_gain: f64,
276 min_ms_gain: f64,
277 current: EventMetricSummary,
278 previous: EventMetricSummary,
279}
280
281impl std::fmt::Display for EventMetricComparison {
282 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
283 fn to_percent(gain: f64) -> String {
284 if gain >= 1.0 {
285 format!("+{:.2}%", (gain - 1.0) * 100.0)
286 } else {
287 format!("-{:.2}%", (1.0 - gain) * 100.0)
288 }
289 }
290 f.write_str(&format!(
291 "avg: {}, median: {}, max: {}, min: {}",
292 to_percent(self.avg_ms_gain),
293 to_percent(self.median_ms_gain),
294 to_percent(self.max_ms_gain),
295 to_percent(self.min_ms_gain),
296 ))
297 }
298}
299
300#[tokio::main]
301async fn main() -> anyhow::Result<()> {
302 fedimint_logging::TracingSetup::default().init()?;
303 let opts = Opts::parse();
304 validate_total_notes(&opts)?;
305 let (event_sender, event_receiver) = tokio::sync::mpsc::unbounded_channel();
306 let summary_handle = spawn("handle metrics summary", {
307 let opts = opts.clone();
308 async { handle_metrics_summary(opts, event_receiver).await }
309 });
310 let futures = match opts.command.clone() {
311 Command::TestConnect {
312 invite_code,
313 duration_secs,
314 timeout_secs,
315 limit_endpoints,
316 } => {
317 let connectors = ConnectorRegistry::build_from_client_env().bind().await;
318 let invite_code = InviteCode::from_str(&invite_code).context("invalid invite code")?;
319 test_connect_raw_client(
320 &connectors,
321 invite_code,
322 opts.users,
323 Duration::from_secs(duration_secs),
324 Duration::from_secs(timeout_secs),
325 limit_endpoints,
326 event_sender.clone(),
327 )
328 .await?
329 }
330 Command::TestDownload { invite_code } => {
331 let connectors = ConnectorRegistry::build_from_client_env().bind().await;
332 let invite_code = InviteCode::from_str(&invite_code).context("invalid invite code")?;
333 test_download_config(&connectors, &invite_code, opts.users, &event_sender.clone())
334 }
335 Command::LoadTest(args) => {
336 let invite_code = invite_code_or_fallback(args.invite_code).await;
337
338 let gateway_id = if let Some(gateway_id) = args.gateway_id {
339 Some(gateway_id)
340 } else if let Some(generate_invoice_with) = args.generate_invoice_with {
341 Some(get_gateway_id(generate_invoice_with).await?)
342 } else {
343 None
344 };
345 let invoices = if let Some(invoices_file) = args.invoices_file {
346 let display_path = invoices_file.display();
347 let invoices_file = tokio::fs::File::open(&invoices_file)
348 .await
349 .with_context(|| format!("Failed to open {display_path}"))?;
350 let mut lines = tokio::io::BufReader::new(invoices_file).lines();
351 let mut invoices = vec![];
352 while let Some(line) = lines.next_line().await? {
353 let invoice = Bolt11Invoice::from_str(&line)?;
354 invoices.push(invoice);
355 }
356 invoices
357 } else {
358 vec![]
359 };
360 if args.generate_invoice_with.is_none() && invoices.is_empty() {
361 info!(
362 "No --generate-invoice-with given no invoices on --invoices-file, not LN/gateway tests will be run"
363 );
364 }
365 run_load_test(
366 opts.archive_dir,
367 opts.users,
368 invite_code,
369 args.initial_notes,
370 args.generate_invoice_with,
371 args.invoices_per_user,
372 Duration::from_secs(args.ln_payment_sleep_secs),
373 invoices,
374 gateway_id,
375 args.notes_per_user,
376 args.note_denomination,
377 args.invoice_amount,
378 event_sender.clone(),
379 )
380 .await?
381 }
382 Command::LnCircularLoadTest(args) => {
383 let invite_code = invite_code_or_fallback(args.invite_code).await;
384 run_ln_circular_load_test(
385 opts.archive_dir,
386 opts.users,
387 invite_code,
388 args.initial_notes,
389 Duration::from_secs(args.test_duration_secs),
390 Duration::from_secs(args.ln_payment_sleep_secs),
391 args.notes_per_user,
392 args.note_denomination,
393 args.invoice_amount,
394 args.strategy,
395 event_sender.clone(),
396 )
397 .await?
398 }
399 };
400
401 let result = futures::future::join_all(futures).await;
402 drop(event_sender);
403 summary_handle.await??;
404 let len_failures = result.iter().filter(|r| r.is_err()).count();
405 eprintln!("{} results, {len_failures} failures", result.len());
406 for r in result {
407 if let Err(e) = r {
408 warn!("Task failed: {:?}", e);
409 }
410 }
411 if 0 < len_failures {
412 bail!("Finished with failures");
413 }
414 info!("Finished successfully");
415 Ok(())
416}
417
418fn validate_total_notes(opts: &Opts) -> anyhow::Result<()> {
419 match &opts.command {
420 Command::LoadTest(args) => {
421 checked_total(opts.users, args.notes_per_user)?;
422 }
423 Command::LnCircularLoadTest(args) => {
424 checked_total(opts.users, args.notes_per_user)?;
425 }
426 Command::TestConnect { .. } | Command::TestDownload { .. } => {}
427 }
428 Ok(())
429}
430
431fn checked_total(users: u16, notes_per_user: u16) -> anyhow::Result<u16> {
432 users.checked_mul(notes_per_user).ok_or_else(|| {
433 anyhow!(
434 "--users {users} multiplied by --notes-per-user {notes_per_user} exceeds the maximum supported combined note count of {}",
435 u16::MAX
436 )
437 })
438}
439
440async fn invite_code_or_fallback(invite_code: Option<InviteCode>) -> Option<InviteCode> {
441 if let Some(invite_code) = invite_code {
442 Some(invite_code)
443 } else {
444 match get_invite_code_cli(0.into()).await {
446 Ok(invite_code) => Some(invite_code),
447 Err(e) => {
448 info!(
449 "No invite code provided and failed to get one with '{e}' error, will try to proceed without one..."
450 );
451 None
452 }
453 }
454 }
455}
456
457#[allow(clippy::too_many_arguments)]
458async fn run_load_test(
459 archive_dir: Option<PathBuf>,
460 users: u16,
461 invite_code: Option<InviteCode>,
462 initial_notes: Option<OOBNotes>,
463 generate_invoice_with: Option<LnInvoiceGeneration>,
464 generated_invoices_per_user: u16,
465 ln_payment_sleep: Duration,
466 invoices_from_file: Vec<Bolt11Invoice>,
467 gateway_id: Option<String>,
468 notes_per_user: u16,
469 note_denomination: Amount,
470 invoice_amount: Amount,
471 event_sender: mpsc::UnboundedSender<MetricEvent>,
472) -> anyhow::Result<Vec<BoxFuture<'static, anyhow::Result<()>>>> {
473 let minimum_notes = checked_total(users, notes_per_user)?;
474 let db_path = get_db_path(&archive_dir);
475 let (coordinator, invite_code) = get_coordinator_client(&db_path, &invite_code).await?;
476 let minimum_amount_required = note_denomination * u64::from(minimum_notes);
477
478 reissue_initial_notes(initial_notes, &coordinator, &event_sender).await?;
479 get_required_notes(&coordinator, minimum_amount_required, &event_sender).await?;
480 print_coordinator_notes(&coordinator).await?;
481 info!(
482 "Reminting {minimum_notes} notes of denomination {note_denomination} for {users} users, {notes_per_user} notes per user (this may take a while if the number of users/notes is high)"
483 );
484 remint_denomination(&coordinator, note_denomination, minimum_notes).await?;
485 print_coordinator_notes(&coordinator).await?;
486
487 let users_clients = get_users_clients(users, db_path, invite_code).await?;
488
489 let mut users_notes =
490 get_notes_for_users(users, notes_per_user, coordinator, note_denomination).await?;
491 let mut users_invoices = HashMap::new();
492 let mut user = 0;
493 for invoice in invoices_from_file {
495 users_invoices
496 .entry(user)
497 .or_insert_with(Vec::new)
498 .push(invoice);
499 user = (user + 1) % users;
500 }
501
502 info!("Starting user tasks");
503 let futures = users_clients
504 .into_iter()
505 .enumerate()
506 .map(|(u, client)| {
507 let u = u as u16;
508 let oob_notes = users_notes.remove(&u).unwrap();
509 let invoices = users_invoices.remove(&u).unwrap_or_default();
510 let event_sender = event_sender.clone();
511 let f: BoxFuture<_> = Box::pin(do_load_test_user_task(
512 format!("User {u}:"),
513 client,
514 oob_notes,
515 generated_invoices_per_user,
516 ln_payment_sleep,
517 invoice_amount,
518 invoices,
519 generate_invoice_with,
520 event_sender,
521 gateway_id.clone(),
522 ));
523 f
524 })
525 .collect::<Vec<_>>();
526
527 Ok(futures)
528}
529
530async fn get_notes_for_users(
531 users: u16,
532 notes_per_user: u16,
533 coordinator: ClientHandleArc,
534 note_denomination: Amount,
535) -> anyhow::Result<HashMap<u16, Vec<OOBNotes>>> {
536 let mut users_notes = HashMap::new();
537 for u in 0..users {
538 users_notes.insert(u, Vec::with_capacity(notes_per_user.into()));
539 for _ in 0..notes_per_user {
540 let (_, oob_notes) = do_spend_notes(&coordinator, note_denomination).await?;
541 let user_amount = oob_notes.total_amount();
542 info!("Giving {user_amount} to user {u}");
543 users_notes.get_mut(&u).unwrap().push(oob_notes);
544 }
545 }
546 Ok(users_notes)
547}
548
549async fn get_users_clients(
550 n: u16,
551 db_path: Option<PathBuf>,
552 invite_code: Option<InviteCode>,
553) -> anyhow::Result<Vec<ClientHandleArc>> {
554 let mut users_clients = Vec::with_capacity(n.into());
555 for u in 0..n {
556 let (client, _) = get_user_client(u, &db_path, &invite_code).await?;
557 users_clients.push(client);
558 }
559 Ok(users_clients)
560}
561
562async fn get_user_client(
563 user_index: u16,
564 db_path: &Option<PathBuf>,
565 invite_code: &Option<InviteCode>,
566) -> anyhow::Result<(ClientHandleArc, Option<InviteCode>)> {
567 let user_db = db_path
568 .as_ref()
569 .map(|db_path| db_path.join(format!("user_{user_index}.db")));
570 let user_invite_code = if user_db.as_ref().is_some_and(|db| db.exists()) {
571 None
572 } else {
573 invite_code.clone()
574 };
575 let (client, invite_code) = build_client(user_invite_code, user_db.as_ref()).await?;
576 if let Ok(ln_client) = client.get_first_module::<LightningClientModule>() {
578 let _ = ln_client.update_gateway_cache().await;
579 }
580 Ok((client, invite_code))
581}
582
583async fn print_coordinator_notes(coordinator: &ClientHandleArc) -> anyhow::Result<()> {
584 info!("Note summary:");
585 let summary = get_note_summary(coordinator).await?;
586 for (k, v) in summary.iter() {
587 info!("{k}: {v}");
588 }
589 Ok(())
590}
591
592async fn get_required_notes(
593 coordinator: &ClientHandleArc,
594 minimum_amount_required: Amount,
595 event_sender: &mpsc::UnboundedSender<MetricEvent>,
596) -> anyhow::Result<()> {
597 let current_balance = coordinator.get_balance_for_btc(Account::Primary).await?;
598
599 if current_balance < minimum_amount_required {
600 let diff = minimum_amount_required.saturating_sub(current_balance);
601 info!(
602 "Current balance {current_balance} on coordinator not enough, trying to get {diff} more through fedimint-cli"
603 );
604 match try_get_notes_cli(&diff, 5).await {
605 Ok(notes) => {
606 info!("Got {} more notes, reissuing them", notes.total_amount());
607 reissue_notes(coordinator, notes, event_sender).await?;
608 }
609 Err(e) => {
610 info!("Unable to get more notes: '{e}', will try to proceed without them");
611 }
612 }
613 } else {
614 info!(
615 "Current balance of {current_balance} already covers the minimum required of {minimum_amount_required}"
616 );
617 }
618 Ok(())
619}
620
621async fn reissue_initial_notes(
622 initial_notes: Option<OOBNotes>,
623 coordinator: &ClientHandleArc,
624 event_sender: &mpsc::UnboundedSender<MetricEvent>,
625) -> anyhow::Result<()> {
626 if let Some(notes) = initial_notes {
627 let amount = notes.total_amount();
628 info!("Reissuing initial notes, got {amount}");
629 reissue_notes(coordinator, notes, event_sender).await?;
630 }
631 Ok(())
632}
633
634async fn get_coordinator_client(
635 db_path: &Option<PathBuf>,
636 invite_code: &Option<InviteCode>,
637) -> anyhow::Result<(ClientHandleArc, Option<InviteCode>)> {
638 let (client, invite_code) = if let Some(db_path) = db_path {
639 let coordinator_db = db_path.join("coordinator.db");
640 if coordinator_db.exists() {
641 build_client(invite_code.clone(), Some(&coordinator_db)).await?
642 } else {
643 tokio::fs::create_dir_all(db_path).await?;
644 build_client(
645 Some(invite_code.clone().context(
646 "Running on this archive dir for the first time, an invite code is required",
647 )?),
648 Some(&coordinator_db),
649 )
650 .await?
651 }
652 } else {
653 build_client(
654 Some(
655 invite_code
656 .clone()
657 .context("No archive dir given, an invite code is strictly required")?,
658 ),
659 None,
660 )
661 .await?
662 };
663 Ok((client, invite_code))
664}
665
666fn get_db_path(archive_dir: &Option<PathBuf>) -> Option<PathBuf> {
667 archive_dir.as_ref().map(|p| p.join("db"))
668}
669
670async fn get_lightning_gateway(
671 client: &ClientHandleArc,
672 gateway_id: Option<String>,
673) -> Option<LightningGateway> {
674 let gateway_id = parse_gateway_id(gateway_id.or(None)?.as_str()).expect("Invalid gateway id");
675 let ln_module = client
676 .get_first_module::<LightningClientModule>()
677 .expect("Must have ln client module");
678 ln_module.select_gateway(&gateway_id).await
679}
680
681#[allow(clippy::too_many_arguments)]
682async fn do_load_test_user_task(
683 prefix: String,
684 client: ClientHandleArc,
685 oob_notes: Vec<OOBNotes>,
686 generated_invoices_per_user: u16,
687 ln_payment_sleep: Duration,
688 invoice_amount: Amount,
689 additional_invoices: Vec<Bolt11Invoice>,
690 generate_invoice_with: Option<LnInvoiceGeneration>,
691 event_sender: mpsc::UnboundedSender<MetricEvent>,
692 gateway_id: Option<String>,
693) -> anyhow::Result<()> {
694 let ln_gateway = get_lightning_gateway(&client, gateway_id).await;
695 for oob_note in oob_notes {
696 let amount = oob_note.total_amount();
697 reissue_notes(&client, oob_note, &event_sender)
698 .await
699 .map_err(|e| anyhow!("while reissuing initial {amount}: {e}"))?;
700 }
701 let mut generated_invoices_per_user_iterator = (0..generated_invoices_per_user).peekable();
702 while let Some(_) = generated_invoices_per_user_iterator.next() {
703 let total_amount = get_note_summary(&client).await?.total_amount();
704 if total_amount < invoice_amount {
705 warn!("Can't pay invoice, not enough funds: {invoice_amount} > {total_amount}");
706 } else {
707 match generate_invoice_with {
708 Some(LnInvoiceGeneration::LdkLightningCli) => {
709 let invoice = ldk_create_invoice(invoice_amount).await?;
710 gateway_pay_invoice(
711 &prefix,
712 "LND",
713 &client,
714 invoice.clone(),
715 &event_sender,
716 ln_gateway.clone(),
717 )
718 .await?;
719 ldk_wait_invoice_payment(&invoice).await?;
720 }
721 None if additional_invoices.is_empty() => {
722 debug!(
723 "No method given to generate an invoice and no invoices on file, will not test the gateway"
724 );
725 break;
726 }
727 None => {
728 break;
729 }
730 }
731 if generated_invoices_per_user_iterator.peek().is_some() {
732 fedimint_core::task::sleep(ln_payment_sleep).await;
734 }
735 }
736 }
737 let mut additional_invoices = additional_invoices.into_iter().peekable();
738 while let Some(invoice) = additional_invoices.next() {
739 let total_amount = get_note_summary(&client).await?.total_amount();
740 let invoice_amount =
741 Amount::from_msats(invoice.amount_milli_satoshis().unwrap_or_default());
742 if total_amount < invoice_amount {
743 warn!("Can't pay invoice, not enough funds: {invoice_amount} > {total_amount}");
744 } else if invoice_amount == Amount::ZERO {
745 warn!("Can't pay invoice {invoice}, amount is zero");
746 } else {
747 gateway_pay_invoice(
748 &prefix,
749 "unknown",
750 &client,
751 invoice,
752 &event_sender,
753 ln_gateway.clone(),
754 )
755 .await?;
756 if additional_invoices.peek().is_some() {
757 fedimint_core::task::sleep(ln_payment_sleep).await;
759 }
760 }
761 }
762 Ok(())
763}
764
765#[allow(clippy::too_many_arguments)]
766async fn run_ln_circular_load_test(
767 archive_dir: Option<PathBuf>,
768 users: u16,
769 invite_code: Option<InviteCode>,
770 initial_notes: Option<OOBNotes>,
771 test_duration: Duration,
772 ln_payment_sleep: Duration,
773 notes_per_user: u16,
774 note_denomination: Amount,
775 invoice_amount: Amount,
776 strategy: LnCircularStrategy,
777 event_sender: mpsc::UnboundedSender<MetricEvent>,
778) -> anyhow::Result<Vec<BoxFuture<'static, anyhow::Result<()>>>> {
779 let minimum_notes = checked_total(users, notes_per_user)?;
780 let db_path = get_db_path(&archive_dir);
781 let (coordinator, invite_code) = get_coordinator_client(&db_path, &invite_code).await?;
782 let minimum_amount_required = note_denomination * u64::from(minimum_notes);
783
784 reissue_initial_notes(initial_notes, &coordinator, &event_sender).await?;
785 get_required_notes(&coordinator, minimum_amount_required, &event_sender).await?;
786
787 info!(
788 "Reminting {minimum_notes} notes of denomination {note_denomination} for {users} users, {notes_per_user} notes per user (this may take a while if the number of users/notes is high)"
789 );
790 remint_denomination(&coordinator, note_denomination, minimum_notes).await?;
791
792 print_coordinator_notes(&coordinator).await?;
793
794 let users_clients = get_users_clients(users, db_path, invite_code.clone()).await?;
795
796 let mut users_notes =
797 get_notes_for_users(users, notes_per_user, coordinator, note_denomination).await?;
798
799 info!("Starting user tasks");
800 let futures = users_clients
801 .into_iter()
802 .enumerate()
803 .map(|(u, client)| {
804 let u = u as u16;
805 let oob_notes = users_notes.remove(&u).unwrap();
806 let event_sender = event_sender.clone();
807 let f: BoxFuture<_> = Box::pin(do_ln_circular_test_user_task(
808 format!("User {u}:"),
809 client,
810 invite_code.clone(),
811 oob_notes,
812 test_duration,
813 ln_payment_sleep,
814 invoice_amount,
815 strategy,
816 event_sender,
817 ));
818 f
819 })
820 .collect::<Vec<_>>();
821
822 Ok(futures)
823}
824
825#[allow(clippy::too_many_arguments)]
826async fn do_ln_circular_test_user_task(
827 prefix: String,
828 client: ClientHandleArc,
829 invite_code: Option<InviteCode>,
830 oob_notes: Vec<OOBNotes>,
831 test_duration: Duration,
832 ln_payment_sleep: Duration,
833 invoice_amount: Amount,
834 strategy: LnCircularStrategy,
835 event_sender: mpsc::UnboundedSender<MetricEvent>,
836) -> anyhow::Result<()> {
837 for oob_note in oob_notes {
838 let amount = oob_note.total_amount();
839 reissue_notes(&client, oob_note, &event_sender)
840 .await
841 .map_err(|e| anyhow!("while reissuing initial {amount}: {e}"))?;
842 }
843 let initial_time = fedimint_core::time::now();
844 let still_ontime = || async {
845 fedimint_core::time::now()
846 .duration_since(initial_time)
847 .expect("time to work")
848 <= test_duration
849 };
850 let sleep_a_bit = || async {
851 if still_ontime().await {
852 fedimint_core::task::sleep(ln_payment_sleep).await;
853 }
854 };
855 match strategy {
856 LnCircularStrategy::TwoGateways => {
857 let invoice_generation = LnInvoiceGeneration::LdkLightningCli;
858 while still_ontime().await {
859 let gateway_id = get_gateway_id(invoice_generation).await?;
860 let ln_gateway = get_lightning_gateway(&client, Some(gateway_id)).await;
861 run_two_gateways_strategy(
862 &prefix,
863 &invoice_generation,
864 &invoice_amount,
865 &event_sender,
866 &client,
867 ln_gateway,
868 )
869 .await?;
870 sleep_a_bit().await;
871 }
872 }
873 LnCircularStrategy::SelfPayment => {
874 while still_ontime().await {
875 do_self_payment(&prefix, &client, invoice_amount, &event_sender).await?;
876 sleep_a_bit().await;
877 }
878 }
879 LnCircularStrategy::PartnerPingPong => {
880 let (partner, _) = build_client(invite_code, None).await?;
881 while still_ontime().await {
882 do_partner_ping_pong(&prefix, &client, &partner, invoice_amount, &event_sender)
883 .await?;
884 sleep_a_bit().await;
885 }
886 }
887 }
888 Ok(())
889}
890
891const GATEWAY_CREATE_INVOICE: &str = "gateway_create_invoice";
892
893async fn run_two_gateways_strategy(
894 prefix: &str,
895 invoice_generation: &LnInvoiceGeneration,
896 invoice_amount: &Amount,
897 event_sender: &mpsc::UnboundedSender<MetricEvent>,
898 client: &ClientHandleArc,
899 ln_gateway: Option<LightningGateway>,
900) -> Result<(), anyhow::Error> {
901 let create_invoice_time = fedimint_core::time::now();
902 match *invoice_generation {
903 LnInvoiceGeneration::LdkLightningCli => {
904 let invoice = ldk_create_invoice(*invoice_amount).await?;
905 let elapsed = create_invoice_time.elapsed()?;
906 info!("Created invoice using LDK in {elapsed:?}");
907 event_sender.send(MetricEvent {
908 name: GATEWAY_CREATE_INVOICE.into(),
909 duration: elapsed,
910 })?;
911 gateway_pay_invoice(
912 prefix,
913 "LND",
914 client,
915 invoice.clone(),
916 event_sender,
917 ln_gateway.clone(),
918 )
919 .await?;
920 ldk_wait_invoice_payment(&invoice).await?;
921 let (operation_id, invoice) =
922 client_create_invoice(client, *invoice_amount, event_sender, ln_gateway).await?;
923 let pay_invoice_time = fedimint_core::time::now();
924 ldk_pay_invoice(invoice).await?;
925 wait_invoice_payment(
926 prefix,
927 "LND",
928 client,
929 operation_id,
930 event_sender,
931 pay_invoice_time,
932 )
933 .await?;
934 }
935 }
936 Ok(())
937}
938
939async fn do_self_payment(
940 prefix: &str,
941 client: &ClientHandleArc,
942 invoice_amount: Amount,
943 event_sender: &mpsc::UnboundedSender<MetricEvent>,
944) -> anyhow::Result<()> {
945 let (operation_id, invoice) =
946 client_create_invoice(client, invoice_amount, event_sender, None).await?;
947 let pay_invoice_time = fedimint_core::time::now();
948 let lightning_module = client.get_first_module::<LightningClientModule>()?;
949 lightning_module
951 .pay_bolt11_invoice(None, invoice, ())
952 .await?;
953 wait_invoice_payment(
954 prefix,
955 "gateway",
956 client,
957 operation_id,
958 event_sender,
959 pay_invoice_time,
960 )
961 .await?;
962 Ok(())
963}
964
965async fn do_partner_ping_pong(
966 prefix: &str,
967 client: &ClientHandleArc,
968 partner: &ClientHandleArc,
969 invoice_amount: Amount,
970 event_sender: &mpsc::UnboundedSender<MetricEvent>,
971) -> anyhow::Result<()> {
972 let (operation_id, invoice) =
974 client_create_invoice(partner, invoice_amount, event_sender, None).await?;
975 let pay_invoice_time = fedimint_core::time::now();
976 let lightning_module = client.get_first_module::<LightningClientModule>()?;
977 lightning_module
980 .pay_bolt11_invoice(None, invoice, ())
981 .await?;
982 wait_invoice_payment(
983 prefix,
984 "gateway",
985 partner,
986 operation_id,
987 event_sender,
988 pay_invoice_time,
989 )
990 .await?;
991 let (operation_id, invoice) =
993 client_create_invoice(client, invoice_amount, event_sender, None).await?;
994 let pay_invoice_time = fedimint_core::time::now();
995 let partner_lightning_module = partner.get_first_module::<LightningClientModule>()?;
996 partner_lightning_module
999 .pay_bolt11_invoice(None, invoice, ())
1000 .await?;
1001 wait_invoice_payment(
1002 prefix,
1003 "gateway",
1004 client,
1005 operation_id,
1006 event_sender,
1007 pay_invoice_time,
1008 )
1009 .await?;
1010 Ok(())
1011}
1012
1013async fn wait_invoice_payment(
1014 prefix: &str,
1015 gateway_name: &str,
1016 client: &ClientHandleArc,
1017 operation_id: fedimint_core::core::OperationId,
1018 event_sender: &mpsc::UnboundedSender<MetricEvent>,
1019 pay_invoice_time: std::time::SystemTime,
1020) -> anyhow::Result<()> {
1021 let elapsed = pay_invoice_time.elapsed()?;
1022 info!("{prefix} Invoice payment receive started using {gateway_name} in {elapsed:?}");
1023 event_sender.send(MetricEvent {
1024 name: format!("gateway_{gateway_name}_payment_received_started"),
1025 duration: elapsed,
1026 })?;
1027 let lightning_module = client.get_first_module::<LightningClientModule>()?;
1028 let mut updates = lightning_module
1029 .subscribe_ln_receive(operation_id)
1030 .await?
1031 .into_stream();
1032 wait_invoice_payment_updates(
1033 prefix,
1034 gateway_name,
1035 operation_id,
1036 event_sender,
1037 pay_invoice_time,
1038 &mut updates,
1039 )
1040 .await
1041}
1042
1043async fn wait_invoice_payment_updates(
1044 prefix: &str,
1045 gateway_name: &str,
1046 operation_id: fedimint_core::core::OperationId,
1047 event_sender: &mpsc::UnboundedSender<MetricEvent>,
1048 pay_invoice_time: std::time::SystemTime,
1049 updates: &mut (impl futures::Stream<Item = LnReceiveState> + Unpin),
1050) -> anyhow::Result<()> {
1051 while let Some(update) = updates.next().await {
1052 debug!(%prefix, ?update, "Invoice payment update");
1053 match update {
1054 LnReceiveState::Claimed => {
1055 let elapsed: Duration = pay_invoice_time.elapsed()?;
1056 info!("{prefix} Invoice payment received on {gateway_name} in {elapsed:?}");
1057 event_sender.send(MetricEvent {
1058 name: "gateway_payment_received_success".into(),
1059 duration: elapsed,
1060 })?;
1061 event_sender.send(MetricEvent {
1062 name: format!("gateway_{gateway_name}_payment_received_success"),
1063 duration: elapsed,
1064 })?;
1065 return Ok(());
1066 }
1067 LnReceiveState::Canceled { reason } => {
1068 let elapsed: Duration = pay_invoice_time.elapsed()?;
1069 info!(
1070 "{prefix} Invoice payment receive was canceled on {gateway_name}: {reason} in {elapsed:?}"
1071 );
1072 event_sender.send(MetricEvent {
1073 name: "gateway_payment_received_canceled".into(),
1074 duration: elapsed,
1075 })?;
1076 bail!(
1077 "Invoice payment receive for operation {operation_id:?} was canceled on \
1078 {gateway_name}: {reason}"
1079 );
1080 }
1081 _ => {}
1082 }
1083 }
1084 bail!(
1085 "Invoice payment receive stream ended before operation {operation_id:?} was claimed on \
1086 {gateway_name}"
1087 )
1088}
1089
1090async fn client_create_invoice(
1091 client: &ClientHandleArc,
1092 invoice_amount: Amount,
1093 event_sender: &mpsc::UnboundedSender<MetricEvent>,
1094 ln_gateway: Option<LightningGateway>,
1095) -> anyhow::Result<(fedimint_core::core::OperationId, Bolt11Invoice)> {
1096 let create_invoice_time = fedimint_core::time::now();
1097 let lightning_module = client.get_first_module::<LightningClientModule>()?;
1098 let desc = Description::new("test".to_string())?;
1099 let (operation_id, invoice, _) = lightning_module
1100 .create_bolt11_invoice(
1101 invoice_amount,
1102 Bolt11InvoiceDescription::Direct(desc),
1103 None,
1104 (),
1105 ln_gateway,
1106 )
1107 .await?;
1108 let elapsed = create_invoice_time.elapsed()?;
1109 info!("Created invoice using gateway in {elapsed:?}");
1110 event_sender.send(MetricEvent {
1111 name: GATEWAY_CREATE_INVOICE.into(),
1112 duration: elapsed,
1113 })?;
1114 Ok((operation_id, invoice))
1115}
1116
1117fn test_download_config(
1118 endpoints: &ConnectorRegistry,
1119 invite_code: &InviteCode,
1120 users: u16,
1121 event_sender: &mpsc::UnboundedSender<MetricEvent>,
1122) -> Vec<BoxFuture<'static, anyhow::Result<()>>> {
1123 (0..users)
1124 .map(move |_| {
1125 let invite_code = invite_code.clone();
1126 let event_sender = event_sender.clone();
1127 let endpoints = endpoints.clone();
1128 let f: BoxFuture<_> = Box::pin(async move {
1129 let m = fedimint_core::time::now();
1130 let _ = download_from_invite_code(&endpoints, &invite_code).await?;
1131 event_sender.send(MetricEvent {
1132 name: "download_client_config".into(),
1133 duration: m.elapsed()?,
1134 })?;
1135 Ok(())
1136 });
1137 f
1138 })
1139 .collect()
1140}
1141
1142async fn test_connect_raw_client(
1143 endpoints: &ConnectorRegistry,
1144 invite_code: InviteCode,
1145 users: u16,
1146 duration: Duration,
1147 timeout: Duration,
1148 limit_endpoints: Option<usize>,
1149 event_sender: mpsc::UnboundedSender<MetricEvent>,
1150) -> anyhow::Result<Vec<BoxFuture<'static, anyhow::Result<()>>>> {
1151 use jsonrpsee_core::client::ClientT;
1152 use jsonrpsee_ws_client::WsClientBuilder;
1153
1154 let (mut cfg, _) = download_from_invite_code(endpoints, &invite_code).await?;
1155
1156 if let Some(limit_endpoints) = limit_endpoints {
1157 cfg.global.api_endpoints = cfg
1158 .global
1159 .api_endpoints
1160 .into_iter()
1161 .take(limit_endpoints)
1162 .collect();
1163 info!("Limiting endpoints to {:?}", cfg.global.api_endpoints);
1164 }
1165
1166 info!("Connecting to {users} clients");
1167 let clients = (0..users)
1168 .flat_map(|_| {
1169 cfg.global.api_endpoints.values().map(|url| async {
1170 let ws_client = WsClientBuilder::default()
1171 .request_timeout(timeout)
1172 .connection_timeout(timeout)
1173 .build(url_to_string_with_default_port(&url.url))
1174 .await?;
1175 Ok::<_, anyhow::Error>(ws_client)
1176 })
1177 })
1178 .collect::<Vec<_>>();
1179 let clients = futures::future::try_join_all(clients).await?;
1180 info!("Keeping {users} clients connected for {duration:?}");
1181 Ok(clients
1182 .into_iter()
1183 .map(|client| {
1184 let event_sender = event_sender.clone();
1185 let f: BoxFuture<_> = Box::pin(async move {
1186 let initial_time = fedimint_core::time::now();
1187 while initial_time.elapsed()? < duration {
1188 let m = fedimint_core::time::now();
1189 let _epoch: u64 = client
1190 .request::<_, _>(SESSION_COUNT_ENDPOINT, vec![ApiRequestErased::default()])
1191 .await?;
1192 event_sender.send(MetricEvent {
1193 name: SESSION_COUNT_ENDPOINT.into(),
1194 duration: m.elapsed()?,
1195 })?;
1196 fedimint_core::task::sleep(Duration::from_secs(1)).await;
1197 }
1198 Ok(())
1199 });
1200 f
1201 })
1202 .collect())
1203}
1204
1205fn url_to_string_with_default_port(url: &SafeUrl) -> String {
1206 format!(
1207 "{}://{}:{}{}",
1208 url.scheme(),
1209 url.host().expect("Asserted on construction"),
1210 url.port_or_known_default()
1211 .expect("Asserted on construction"),
1212 url.path()
1213 )
1214}
1215
1216async fn handle_metrics_summary(
1217 opts: Opts,
1218 mut event_receiver: mpsc::UnboundedReceiver<MetricEvent>,
1219) -> anyhow::Result<()> {
1220 let timestamp_seconds = fedimint_core::time::duration_since_epoch().as_secs();
1221 let mut metrics_json_output_files = vec![];
1222 let mut previous_metrics = vec![];
1223 let mut comparison_output = None;
1224 if let Some(archive_dir) = opts.archive_dir {
1225 let mut archive_metrics = archive_dir.join("metrics");
1226 archive_metrics.push(opts.users.to_string());
1227 tokio::fs::create_dir_all(&archive_metrics).await?;
1228 let mut archive_comparisons = archive_dir.join("comparisons");
1229 archive_comparisons.push(opts.users.to_string());
1230 tokio::fs::create_dir_all(&archive_comparisons).await?;
1231
1232 let latest_metrics_file = std::fs::read_dir(&archive_metrics)?
1233 .map(|entry| {
1234 let entry = entry.unwrap();
1235 let metadata = entry.metadata().unwrap();
1236 let created = metadata
1237 .created()
1238 .unwrap_or_else(|_| metadata.modified().unwrap());
1239 (entry, created)
1240 })
1241 .max_by_key(|(_entry, created)| created.to_owned())
1242 .map(|(entry, _)| entry.path());
1243 if let Some(latest_metrics_file) = latest_metrics_file {
1244 let display_path = latest_metrics_file.display();
1245 let latest_metrics_file = tokio::fs::File::open(&latest_metrics_file)
1246 .await
1247 .with_context(|| format!("Failed to open {display_path}"))?;
1248 let mut lines = tokio::io::BufReader::new(latest_metrics_file).lines();
1249 while let Some(line) = lines.next_line().await? {
1250 match serde_json::from_str::<EventMetricSummary>(&line) {
1251 Ok(metric) => {
1252 previous_metrics.push(metric);
1253 }
1254 Err(e) => {
1255 warn!("Failed to parse previous metric: {e:?}");
1256 }
1257 }
1258 }
1259 }
1260 let new_metric_output = archive_metrics.join(format!("{timestamp_seconds}.json",));
1261 let new_metric_output = BufWriter::new(
1262 OpenOptions::new()
1263 .write(true)
1264 .create(true)
1265 .truncate(true)
1266 .open(new_metric_output)
1267 .await?,
1268 );
1269 metrics_json_output_files.push(new_metric_output);
1270 if !previous_metrics.is_empty() {
1271 let new_comparison_output =
1272 archive_comparisons.join(format!("{timestamp_seconds}.json",));
1273 comparison_output = Some(BufWriter::new(
1274 OpenOptions::new()
1275 .write(true)
1276 .create(true)
1277 .truncate(true)
1278 .open(new_comparison_output)
1279 .await?,
1280 ));
1281 }
1282 }
1283 if let Some(metrics_json_output) = opts.metrics_json_output {
1284 metrics_json_output_files.push(BufWriter::new(
1285 tokio::fs::OpenOptions::new()
1286 .write(true)
1287 .create(true)
1288 .truncate(true)
1289 .open(metrics_json_output)
1290 .await?,
1291 ));
1292 }
1293 let mut results = BTreeMap::new();
1294 while let Some(event) = event_receiver.recv().await {
1295 let entry = results.entry(event.name).or_insert_with(Vec::new);
1296 entry.push(event.duration);
1297 }
1298 let mut previous_metrics = previous_metrics
1299 .into_iter()
1300 .map(|metric| (metric.name.clone(), metric))
1301 .collect::<HashMap<_, _>>();
1302 for (k, mut v) in results {
1303 v.sort();
1304 let n = v.len();
1305 let max = v.iter().last().unwrap();
1306 let min = v.first().unwrap();
1307 let median = v[n / 2];
1308 let sum: Duration = v.iter().sum();
1309 let avg = sum / n as u32;
1310 let metric_summary = EventMetricSummary {
1311 name: k.clone(),
1312 users: u64::from(opts.users),
1313 n: n as u64,
1314 avg_ms: avg.as_millis(),
1315 median_ms: median.as_millis(),
1316 max_ms: max.as_millis(),
1317 min_ms: min.as_millis(),
1318 timestamp_seconds,
1319 };
1320 let comparison = if let Some(previous_metric) = previous_metrics.remove(&k) {
1321 if previous_metric.n == metric_summary.n {
1322 fn calculate_gain(current: u128, previous: u128) -> f64 {
1323 current as f64 / previous as f64
1324 }
1325 let comparison = EventMetricComparison {
1326 avg_ms_gain: calculate_gain(metric_summary.avg_ms, previous_metric.avg_ms),
1327 median_ms_gain: calculate_gain(
1328 metric_summary.median_ms,
1329 previous_metric.median_ms,
1330 ),
1331 max_ms_gain: calculate_gain(metric_summary.max_ms, previous_metric.max_ms),
1332 min_ms_gain: calculate_gain(metric_summary.min_ms, previous_metric.min_ms),
1333 current: metric_summary.clone(),
1334 previous: previous_metric,
1335 };
1336 if let Some(comparison_output) = &mut comparison_output {
1337 let comparison_json =
1338 serde_json::to_string(&comparison).expect("to be serializable");
1339 comparison_output
1340 .write_all(format!("{comparison_json}\n").as_bytes())
1341 .await
1342 .expect("to write on file");
1343 }
1344 Some(comparison)
1345 } else {
1346 info!(
1347 "Skipping comparison for {k} because previous metric has different n ({} vs {})",
1348 previous_metric.n, metric_summary.n
1349 );
1350 None
1351 }
1352 } else {
1353 None
1354 };
1355 if let Some(comparison) = comparison {
1356 println!(
1357 "{n} {k}: avg {avg:?}, median {median:?}, max {max:?}, min {min:?} (compared to previous: {comparison})"
1358 );
1359 } else {
1360 println!("{n} {k}: avg {avg:?}, median {median:?}, max {max:?}, min {min:?}");
1361 }
1362 let metric_summary_json =
1363 serde_json::to_string(&metric_summary).expect("to be serializable");
1364 for metrics_json_output_file in &mut metrics_json_output_files {
1365 metrics_json_output_file
1366 .write_all(format!("{metric_summary_json}\n").as_bytes())
1367 .await
1368 .expect("to write on file");
1369 }
1370 }
1371 for mut output in metrics_json_output_files {
1372 output.flush().await?;
1373 }
1374 if let Some(mut output) = comparison_output {
1375 output.flush().await?;
1376 }
1377 Ok(())
1378}
1379
1380async fn get_gateway_id(generate_invoice_with: LnInvoiceGeneration) -> anyhow::Result<String> {
1381 let gateway_json = match generate_invoice_with {
1382 LnInvoiceGeneration::LdkLightningCli => {
1383 cmd!(GatewayLndCli, "info").out_json().await
1385 }
1386 }?;
1387
1388 let gatewayd_version = devimint::util::Gatewayd::version_or_default().await;
1389 let gateway_id = if gatewayd_version < *VERSION_0_10_0_ALPHA {
1390 gateway_json["gateway_id"]
1391 .as_str()
1392 .context("gateway_id must be a string")?
1393 .to_owned()
1394 } else {
1395 gateway_json["registrations"]["http"][1]
1396 .as_str()
1397 .context("gateway id must be a string")?
1398 .to_owned()
1399 };
1400
1401 Ok(gateway_id)
1402}