Skip to main content

fedimint_load_test_tool/
main.rs

1#![deny(clippy::pedantic)]
2#![allow(clippy::cast_possible_truncation)]
3#![allow(clippy::cast_precision_loss)]
4#![allow(clippy::missing_errors_doc)]
5#![allow(clippy::ref_option)]
6#![allow(clippy::too_many_lines)]
7#![allow(clippy::large_futures)]
8
9use std::collections::{BTreeMap, HashMap};
10use std::path::PathBuf;
11use std::str::FromStr;
12use std::time::Duration;
13use std::vec;
14
15use anyhow::{Context, anyhow, bail};
16use clap::{Args, Parser, Subcommand, ValueEnum};
17use common::{
18    gateway_pay_invoice, get_note_summary, ldk_create_invoice, ldk_pay_invoice,
19    ldk_wait_invoice_payment, parse_gateway_id, reissue_notes,
20};
21use devimint::cmd;
22use devimint::util::GatewayLndCli;
23use devimint::version_constants::VERSION_0_10_0_ALPHA;
24use fedimint_api_client::download_from_invite_code;
25use fedimint_client::ClientHandleArc;
26use fedimint_connectors::ConnectorRegistry;
27use fedimint_core::Amount;
28use fedimint_core::core::Account;
29use fedimint_core::endpoint_constants::SESSION_COUNT_ENDPOINT;
30use fedimint_core::invite_code::InviteCode;
31use fedimint_core::module::ApiRequestErased;
32use fedimint_core::runtime::spawn;
33use fedimint_core::util::{BoxFuture, SafeUrl};
34use fedimint_ln_client::{LightningClientModule, LnReceiveState};
35use fedimint_ln_common::LightningGateway;
36use fedimint_mint_client::OOBNotes;
37use futures::StreamExt;
38use lightning_invoice::{Bolt11Invoice, Bolt11InvoiceDescription, Description};
39use serde::{Deserialize, Serialize};
40use tokio::fs::OpenOptions;
41use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufWriter};
42use tokio::sync::mpsc;
43use tracing::{debug, info, warn};
44
45use crate::common::{
46    build_client, do_spend_notes, get_invite_code_cli, remint_denomination, try_get_notes_cli,
47};
48pub mod common;
49#[cfg(test)]
50mod tests;
51
52#[derive(Parser, Clone)]
53#[command(version)]
54struct Opts {
55    #[arg(
56        long,
57        default_value = "10",
58        help = "Number of users. Each user will work in parallel"
59    )]
60    users: u16,
61
62    #[arg(long, help = "Output with the metrics results in JSON format")]
63    metrics_json_output: Option<PathBuf>,
64
65    #[arg(
66        long,
67        help = "If given, will be used to store and retrieve past metrics for comparison purposes"
68    )]
69    archive_dir: Option<PathBuf>,
70
71    #[clap(subcommand)]
72    command: Command,
73}
74
75#[derive(Debug, Clone, Copy, ValueEnum)]
76enum LnInvoiceGeneration {
77    LdkLightningCli,
78}
79
80#[derive(Subcommand, Clone)]
81enum Command {
82    #[command(about = "Keep many websocket connections to a federation for a duration of time")]
83    TestConnect {
84        #[arg(long, help = "Federation invite code")]
85        invite_code: String,
86        #[arg(
87            long,
88            default_value = "60",
89            help = "How much time to keep the connections open, in seconds"
90        )]
91        duration_secs: u64,
92        #[arg(
93            long,
94            default_value = "120",
95            help = "Timeout for connection attempt and for each request, in secnods"
96        )]
97        timeout_secs: u64,
98        #[arg(
99            long,
100            help = "If given, will limit the number of endpoints (guardians) to connect to"
101        )]
102        limit_endpoints: Option<usize>,
103    },
104    #[command(about = "Try to download the client config many times.")]
105    TestDownload {
106        #[arg(long, help = "Federation invite code")]
107        invite_code: String,
108    },
109    #[command(
110        about = "Run a load test where many users in parallel will try to reissue notes and pay invoices through the gateway"
111    )]
112    LoadTest(LoadTestArgs),
113    /// Run a load test where many users in parallel will receive then send a
114    /// payment through lightning.
115    /// It's 'circular' because the funds always come back to the same user then
116    /// we can keep making the payments in a loop
117    #[command()]
118    LnCircularLoadTest(LnCircularLoadTestArgs),
119}
120
121#[derive(Args, Clone)]
122struct LoadTestArgs {
123    #[arg(
124        long,
125        help = "Federation invite code. If none given, we assume the client already has a config downloaded in DB"
126    )]
127    invite_code: Option<InviteCode>,
128
129    #[arg(
130        long,
131        help = "Notes for the test. If none and no funds on archive, will call fedimint-cli spend"
132    )]
133    initial_notes: Option<OOBNotes>,
134
135    #[arg(
136        long,
137        help = "Gateway Id. If none, retrieve one according to --generate-invoice-with"
138    )]
139    gateway_id: Option<String>,
140
141    #[arg(
142        long,
143        help = "The method used to generate invoices to be paid through the gateway. If none and no --invoices-file provided, no gateway/LN tests will be run. Note that you can't generate an invoice using the same lightning node used by the gateway (i.e self payment is forbidden)"
144    )]
145    generate_invoice_with: Option<LnInvoiceGeneration>,
146
147    #[arg(
148        long,
149        default_value = "1",
150        help = "How many invoices will be created for each user. Only applicable if --generate-invoice-with is provided"
151    )]
152    invoices_per_user: u16,
153
154    #[arg(
155        long,
156        default_value = "0",
157        help = "How many seconds to sleep between LN payments"
158    )]
159    ln_payment_sleep_secs: u64,
160
161    #[arg(
162        long,
163        help = "A text file with one invoice per line. If --generate-invoice-with is provided, these will be additional invoices to be paid"
164    )]
165    invoices_file: Option<PathBuf>,
166
167    #[arg(
168        long,
169        help = "How many notes to distribute to each user",
170        default_value = "2"
171    )]
172    notes_per_user: u16,
173
174    #[arg(
175        long,
176        help = "Note denomination to use for the test",
177        default_value = "1024"
178    )]
179    note_denomination: Amount,
180
181    #[arg(
182        long,
183        help = "Invoice amount when generating one",
184        default_value = "1000"
185    )]
186    invoice_amount: Amount,
187}
188
189#[derive(Args, Clone)]
190struct LnCircularLoadTestArgs {
191    #[arg(
192        long,
193        help = "Federation invite code. If none given, we assume the client already has a config downloaded in DB"
194    )]
195    invite_code: Option<InviteCode>,
196
197    #[arg(
198        long,
199        help = "Notes for the test. If none and no funds on archive, will call fedimint-cli spend"
200    )]
201    initial_notes: Option<OOBNotes>,
202
203    #[arg(
204        long,
205        default_value = "60",
206        help = "For how many seconds to run the test"
207    )]
208    test_duration_secs: u64,
209
210    #[arg(
211        long,
212        default_value = "0",
213        help = "How many seconds to sleep between LN payments"
214    )]
215    ln_payment_sleep_secs: u64,
216
217    #[arg(
218        long,
219        help = "How many notes to distribute to each user",
220        default_value = "1"
221    )]
222    notes_per_user: u16,
223
224    #[arg(
225        long,
226        help = "Note denomination to use for the test",
227        default_value = "1024"
228    )]
229    note_denomination: Amount,
230
231    #[arg(
232        long,
233        help = "Invoice amount when generating one",
234        default_value = "1000"
235    )]
236    invoice_amount: Amount,
237
238    #[arg(long)]
239    strategy: LnCircularStrategy,
240}
241
242#[derive(Debug, Clone, Copy, ValueEnum)]
243enum LnCircularStrategy {
244    /// The user will pay its own invoice
245    SelfPayment,
246    /// One gateway will pay/receive to/from the other, then they will swap
247    /// places
248    TwoGateways,
249    /// Two clients will pay to each other using the same gateway
250    PartnerPingPong,
251}
252
253#[derive(Debug, Clone)]
254pub struct MetricEvent {
255    name: String,
256    duration: Duration,
257}
258
259#[derive(Debug, Clone, Serialize, Deserialize)]
260struct EventMetricSummary {
261    name: String,
262    users: u64,
263    n: u64,
264    avg_ms: u128,
265    median_ms: u128,
266    max_ms: u128,
267    min_ms: u128,
268    timestamp_seconds: u64,
269}
270
271#[derive(Debug, Serialize, Deserialize)]
272struct EventMetricComparison {
273    avg_ms_gain: f64,
274    median_ms_gain: f64,
275    max_ms_gain: f64,
276    min_ms_gain: f64,
277    current: EventMetricSummary,
278    previous: EventMetricSummary,
279}
280
281impl std::fmt::Display for EventMetricComparison {
282    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
283        fn to_percent(gain: f64) -> String {
284            if gain >= 1.0 {
285                format!("+{:.2}%", (gain - 1.0) * 100.0)
286            } else {
287                format!("-{:.2}%", (1.0 - gain) * 100.0)
288            }
289        }
290        f.write_str(&format!(
291            "avg: {}, median: {}, max: {}, min: {}",
292            to_percent(self.avg_ms_gain),
293            to_percent(self.median_ms_gain),
294            to_percent(self.max_ms_gain),
295            to_percent(self.min_ms_gain),
296        ))
297    }
298}
299
300#[tokio::main]
301async fn main() -> anyhow::Result<()> {
302    fedimint_logging::TracingSetup::default().init()?;
303    let opts = Opts::parse();
304    validate_total_notes(&opts)?;
305    let (event_sender, event_receiver) = tokio::sync::mpsc::unbounded_channel();
306    let summary_handle = spawn("handle metrics summary", {
307        let opts = opts.clone();
308        async { handle_metrics_summary(opts, event_receiver).await }
309    });
310    let futures = match opts.command.clone() {
311        Command::TestConnect {
312            invite_code,
313            duration_secs,
314            timeout_secs,
315            limit_endpoints,
316        } => {
317            let connectors = ConnectorRegistry::build_from_client_env().bind().await;
318            let invite_code = InviteCode::from_str(&invite_code).context("invalid invite code")?;
319            test_connect_raw_client(
320                &connectors,
321                invite_code,
322                opts.users,
323                Duration::from_secs(duration_secs),
324                Duration::from_secs(timeout_secs),
325                limit_endpoints,
326                event_sender.clone(),
327            )
328            .await?
329        }
330        Command::TestDownload { invite_code } => {
331            let connectors = ConnectorRegistry::build_from_client_env().bind().await;
332            let invite_code = InviteCode::from_str(&invite_code).context("invalid invite code")?;
333            test_download_config(&connectors, &invite_code, opts.users, &event_sender.clone())
334        }
335        Command::LoadTest(args) => {
336            let invite_code = invite_code_or_fallback(args.invite_code).await;
337
338            let gateway_id = if let Some(gateway_id) = args.gateway_id {
339                Some(gateway_id)
340            } else if let Some(generate_invoice_with) = args.generate_invoice_with {
341                Some(get_gateway_id(generate_invoice_with).await?)
342            } else {
343                None
344            };
345            let invoices = if let Some(invoices_file) = args.invoices_file {
346                let display_path = invoices_file.display();
347                let invoices_file = tokio::fs::File::open(&invoices_file)
348                    .await
349                    .with_context(|| format!("Failed to open {display_path}"))?;
350                let mut lines = tokio::io::BufReader::new(invoices_file).lines();
351                let mut invoices = vec![];
352                while let Some(line) = lines.next_line().await? {
353                    let invoice = Bolt11Invoice::from_str(&line)?;
354                    invoices.push(invoice);
355                }
356                invoices
357            } else {
358                vec![]
359            };
360            if args.generate_invoice_with.is_none() && invoices.is_empty() {
361                info!(
362                    "No --generate-invoice-with given no invoices on --invoices-file, not LN/gateway tests will be run"
363                );
364            }
365            run_load_test(
366                opts.archive_dir,
367                opts.users,
368                invite_code,
369                args.initial_notes,
370                args.generate_invoice_with,
371                args.invoices_per_user,
372                Duration::from_secs(args.ln_payment_sleep_secs),
373                invoices,
374                gateway_id,
375                args.notes_per_user,
376                args.note_denomination,
377                args.invoice_amount,
378                event_sender.clone(),
379            )
380            .await?
381        }
382        Command::LnCircularLoadTest(args) => {
383            let invite_code = invite_code_or_fallback(args.invite_code).await;
384            run_ln_circular_load_test(
385                opts.archive_dir,
386                opts.users,
387                invite_code,
388                args.initial_notes,
389                Duration::from_secs(args.test_duration_secs),
390                Duration::from_secs(args.ln_payment_sleep_secs),
391                args.notes_per_user,
392                args.note_denomination,
393                args.invoice_amount,
394                args.strategy,
395                event_sender.clone(),
396            )
397            .await?
398        }
399    };
400
401    let result = futures::future::join_all(futures).await;
402    drop(event_sender);
403    summary_handle.await??;
404    let len_failures = result.iter().filter(|r| r.is_err()).count();
405    eprintln!("{} results, {len_failures} failures", result.len());
406    for r in result {
407        if let Err(e) = r {
408            warn!("Task failed: {:?}", e);
409        }
410    }
411    if 0 < len_failures {
412        bail!("Finished with failures");
413    }
414    info!("Finished successfully");
415    Ok(())
416}
417
418fn validate_total_notes(opts: &Opts) -> anyhow::Result<()> {
419    match &opts.command {
420        Command::LoadTest(args) => {
421            checked_total(opts.users, args.notes_per_user)?;
422        }
423        Command::LnCircularLoadTest(args) => {
424            checked_total(opts.users, args.notes_per_user)?;
425        }
426        Command::TestConnect { .. } | Command::TestDownload { .. } => {}
427    }
428    Ok(())
429}
430
431fn checked_total(users: u16, notes_per_user: u16) -> anyhow::Result<u16> {
432    users.checked_mul(notes_per_user).ok_or_else(|| {
433        anyhow!(
434            "--users {users} multiplied by --notes-per-user {notes_per_user} exceeds the maximum supported combined note count of {}",
435            u16::MAX
436        )
437    })
438}
439
440async fn invite_code_or_fallback(invite_code: Option<InviteCode>) -> Option<InviteCode> {
441    if let Some(invite_code) = invite_code {
442        Some(invite_code)
443    } else {
444        // Try to get an invite code through cli in a best effort basis
445        match get_invite_code_cli(0.into()).await {
446            Ok(invite_code) => Some(invite_code),
447            Err(e) => {
448                info!(
449                    "No invite code provided and failed to get one with '{e}' error, will try to proceed without one..."
450                );
451                None
452            }
453        }
454    }
455}
456
457#[allow(clippy::too_many_arguments)]
458async fn run_load_test(
459    archive_dir: Option<PathBuf>,
460    users: u16,
461    invite_code: Option<InviteCode>,
462    initial_notes: Option<OOBNotes>,
463    generate_invoice_with: Option<LnInvoiceGeneration>,
464    generated_invoices_per_user: u16,
465    ln_payment_sleep: Duration,
466    invoices_from_file: Vec<Bolt11Invoice>,
467    gateway_id: Option<String>,
468    notes_per_user: u16,
469    note_denomination: Amount,
470    invoice_amount: Amount,
471    event_sender: mpsc::UnboundedSender<MetricEvent>,
472) -> anyhow::Result<Vec<BoxFuture<'static, anyhow::Result<()>>>> {
473    let minimum_notes = checked_total(users, notes_per_user)?;
474    let db_path = get_db_path(&archive_dir);
475    let (coordinator, invite_code) = get_coordinator_client(&db_path, &invite_code).await?;
476    let minimum_amount_required = note_denomination * u64::from(minimum_notes);
477
478    reissue_initial_notes(initial_notes, &coordinator, &event_sender).await?;
479    get_required_notes(&coordinator, minimum_amount_required, &event_sender).await?;
480    print_coordinator_notes(&coordinator).await?;
481    info!(
482        "Reminting {minimum_notes} notes of denomination {note_denomination} for {users} users, {notes_per_user} notes per user (this may take a while if the number of users/notes is high)"
483    );
484    remint_denomination(&coordinator, note_denomination, minimum_notes).await?;
485    print_coordinator_notes(&coordinator).await?;
486
487    let users_clients = get_users_clients(users, db_path, invite_code).await?;
488
489    let mut users_notes =
490        get_notes_for_users(users, notes_per_user, coordinator, note_denomination).await?;
491    let mut users_invoices = HashMap::new();
492    let mut user = 0;
493    // Distribute invoices to users in a round robin fashion
494    for invoice in invoices_from_file {
495        users_invoices
496            .entry(user)
497            .or_insert_with(Vec::new)
498            .push(invoice);
499        user = (user + 1) % users;
500    }
501
502    info!("Starting user tasks");
503    let futures = users_clients
504        .into_iter()
505        .enumerate()
506        .map(|(u, client)| {
507            let u = u as u16;
508            let oob_notes = users_notes.remove(&u).unwrap();
509            let invoices = users_invoices.remove(&u).unwrap_or_default();
510            let event_sender = event_sender.clone();
511            let f: BoxFuture<_> = Box::pin(do_load_test_user_task(
512                format!("User {u}:"),
513                client,
514                oob_notes,
515                generated_invoices_per_user,
516                ln_payment_sleep,
517                invoice_amount,
518                invoices,
519                generate_invoice_with,
520                event_sender,
521                gateway_id.clone(),
522            ));
523            f
524        })
525        .collect::<Vec<_>>();
526
527    Ok(futures)
528}
529
530async fn get_notes_for_users(
531    users: u16,
532    notes_per_user: u16,
533    coordinator: ClientHandleArc,
534    note_denomination: Amount,
535) -> anyhow::Result<HashMap<u16, Vec<OOBNotes>>> {
536    let mut users_notes = HashMap::new();
537    for u in 0..users {
538        users_notes.insert(u, Vec::with_capacity(notes_per_user.into()));
539        for _ in 0..notes_per_user {
540            let (_, oob_notes) = do_spend_notes(&coordinator, note_denomination).await?;
541            let user_amount = oob_notes.total_amount();
542            info!("Giving {user_amount} to user {u}");
543            users_notes.get_mut(&u).unwrap().push(oob_notes);
544        }
545    }
546    Ok(users_notes)
547}
548
549async fn get_users_clients(
550    n: u16,
551    db_path: Option<PathBuf>,
552    invite_code: Option<InviteCode>,
553) -> anyhow::Result<Vec<ClientHandleArc>> {
554    let mut users_clients = Vec::with_capacity(n.into());
555    for u in 0..n {
556        let (client, _) = get_user_client(u, &db_path, &invite_code).await?;
557        users_clients.push(client);
558    }
559    Ok(users_clients)
560}
561
562async fn get_user_client(
563    user_index: u16,
564    db_path: &Option<PathBuf>,
565    invite_code: &Option<InviteCode>,
566) -> anyhow::Result<(ClientHandleArc, Option<InviteCode>)> {
567    let user_db = db_path
568        .as_ref()
569        .map(|db_path| db_path.join(format!("user_{user_index}.db")));
570    let user_invite_code = if user_db.as_ref().is_some_and(|db| db.exists()) {
571        None
572    } else {
573        invite_code.clone()
574    };
575    let (client, invite_code) = build_client(user_invite_code, user_db.as_ref()).await?;
576    // if lightning module is present, update the gateway cache
577    if let Ok(ln_client) = client.get_first_module::<LightningClientModule>() {
578        let _ = ln_client.update_gateway_cache().await;
579    }
580    Ok((client, invite_code))
581}
582
583async fn print_coordinator_notes(coordinator: &ClientHandleArc) -> anyhow::Result<()> {
584    info!("Note summary:");
585    let summary = get_note_summary(coordinator).await?;
586    for (k, v) in summary.iter() {
587        info!("{k}: {v}");
588    }
589    Ok(())
590}
591
592async fn get_required_notes(
593    coordinator: &ClientHandleArc,
594    minimum_amount_required: Amount,
595    event_sender: &mpsc::UnboundedSender<MetricEvent>,
596) -> anyhow::Result<()> {
597    let current_balance = coordinator.get_balance_for_btc(Account::Primary).await?;
598
599    if current_balance < minimum_amount_required {
600        let diff = minimum_amount_required.saturating_sub(current_balance);
601        info!(
602            "Current balance {current_balance} on coordinator not enough, trying to get {diff} more through fedimint-cli"
603        );
604        match try_get_notes_cli(&diff, 5).await {
605            Ok(notes) => {
606                info!("Got {} more notes, reissuing them", notes.total_amount());
607                reissue_notes(coordinator, notes, event_sender).await?;
608            }
609            Err(e) => {
610                info!("Unable to get more notes: '{e}', will try to proceed without them");
611            }
612        }
613    } else {
614        info!(
615            "Current balance of {current_balance} already covers the minimum required of {minimum_amount_required}"
616        );
617    }
618    Ok(())
619}
620
621async fn reissue_initial_notes(
622    initial_notes: Option<OOBNotes>,
623    coordinator: &ClientHandleArc,
624    event_sender: &mpsc::UnboundedSender<MetricEvent>,
625) -> anyhow::Result<()> {
626    if let Some(notes) = initial_notes {
627        let amount = notes.total_amount();
628        info!("Reissuing initial notes, got {amount}");
629        reissue_notes(coordinator, notes, event_sender).await?;
630    }
631    Ok(())
632}
633
634async fn get_coordinator_client(
635    db_path: &Option<PathBuf>,
636    invite_code: &Option<InviteCode>,
637) -> anyhow::Result<(ClientHandleArc, Option<InviteCode>)> {
638    let (client, invite_code) = if let Some(db_path) = db_path {
639        let coordinator_db = db_path.join("coordinator.db");
640        if coordinator_db.exists() {
641            build_client(invite_code.clone(), Some(&coordinator_db)).await?
642        } else {
643            tokio::fs::create_dir_all(db_path).await?;
644            build_client(
645                Some(invite_code.clone().context(
646                    "Running on this archive dir for the first time, an invite code is required",
647                )?),
648                Some(&coordinator_db),
649            )
650            .await?
651        }
652    } else {
653        build_client(
654            Some(
655                invite_code
656                    .clone()
657                    .context("No archive dir given, an invite code is strictly required")?,
658            ),
659            None,
660        )
661        .await?
662    };
663    Ok((client, invite_code))
664}
665
666fn get_db_path(archive_dir: &Option<PathBuf>) -> Option<PathBuf> {
667    archive_dir.as_ref().map(|p| p.join("db"))
668}
669
670async fn get_lightning_gateway(
671    client: &ClientHandleArc,
672    gateway_id: Option<String>,
673) -> Option<LightningGateway> {
674    let gateway_id = parse_gateway_id(gateway_id.or(None)?.as_str()).expect("Invalid gateway id");
675    let ln_module = client
676        .get_first_module::<LightningClientModule>()
677        .expect("Must have ln client module");
678    ln_module.select_gateway(&gateway_id).await
679}
680
681#[allow(clippy::too_many_arguments)]
682async fn do_load_test_user_task(
683    prefix: String,
684    client: ClientHandleArc,
685    oob_notes: Vec<OOBNotes>,
686    generated_invoices_per_user: u16,
687    ln_payment_sleep: Duration,
688    invoice_amount: Amount,
689    additional_invoices: Vec<Bolt11Invoice>,
690    generate_invoice_with: Option<LnInvoiceGeneration>,
691    event_sender: mpsc::UnboundedSender<MetricEvent>,
692    gateway_id: Option<String>,
693) -> anyhow::Result<()> {
694    let ln_gateway = get_lightning_gateway(&client, gateway_id).await;
695    for oob_note in oob_notes {
696        let amount = oob_note.total_amount();
697        reissue_notes(&client, oob_note, &event_sender)
698            .await
699            .map_err(|e| anyhow!("while reissuing initial {amount}: {e}"))?;
700    }
701    let mut generated_invoices_per_user_iterator = (0..generated_invoices_per_user).peekable();
702    while let Some(_) = generated_invoices_per_user_iterator.next() {
703        let total_amount = get_note_summary(&client).await?.total_amount();
704        if total_amount < invoice_amount {
705            warn!("Can't pay invoice, not enough funds: {invoice_amount} > {total_amount}");
706        } else {
707            match generate_invoice_with {
708                Some(LnInvoiceGeneration::LdkLightningCli) => {
709                    let invoice = ldk_create_invoice(invoice_amount).await?;
710                    gateway_pay_invoice(
711                        &prefix,
712                        "LND",
713                        &client,
714                        invoice.clone(),
715                        &event_sender,
716                        ln_gateway.clone(),
717                    )
718                    .await?;
719                    ldk_wait_invoice_payment(&invoice).await?;
720                }
721                None if additional_invoices.is_empty() => {
722                    debug!(
723                        "No method given to generate an invoice and no invoices on file, will not test the gateway"
724                    );
725                    break;
726                }
727                None => {
728                    break;
729                }
730            }
731            if generated_invoices_per_user_iterator.peek().is_some() {
732                // Only sleep while there are more invoices to pay
733                fedimint_core::task::sleep(ln_payment_sleep).await;
734            }
735        }
736    }
737    let mut additional_invoices = additional_invoices.into_iter().peekable();
738    while let Some(invoice) = additional_invoices.next() {
739        let total_amount = get_note_summary(&client).await?.total_amount();
740        let invoice_amount =
741            Amount::from_msats(invoice.amount_milli_satoshis().unwrap_or_default());
742        if total_amount < invoice_amount {
743            warn!("Can't pay invoice, not enough funds: {invoice_amount} > {total_amount}");
744        } else if invoice_amount == Amount::ZERO {
745            warn!("Can't pay invoice {invoice}, amount is zero");
746        } else {
747            gateway_pay_invoice(
748                &prefix,
749                "unknown",
750                &client,
751                invoice,
752                &event_sender,
753                ln_gateway.clone(),
754            )
755            .await?;
756            if additional_invoices.peek().is_some() {
757                // Only sleep while there are more invoices to pay
758                fedimint_core::task::sleep(ln_payment_sleep).await;
759            }
760        }
761    }
762    Ok(())
763}
764
765#[allow(clippy::too_many_arguments)]
766async fn run_ln_circular_load_test(
767    archive_dir: Option<PathBuf>,
768    users: u16,
769    invite_code: Option<InviteCode>,
770    initial_notes: Option<OOBNotes>,
771    test_duration: Duration,
772    ln_payment_sleep: Duration,
773    notes_per_user: u16,
774    note_denomination: Amount,
775    invoice_amount: Amount,
776    strategy: LnCircularStrategy,
777    event_sender: mpsc::UnboundedSender<MetricEvent>,
778) -> anyhow::Result<Vec<BoxFuture<'static, anyhow::Result<()>>>> {
779    let minimum_notes = checked_total(users, notes_per_user)?;
780    let db_path = get_db_path(&archive_dir);
781    let (coordinator, invite_code) = get_coordinator_client(&db_path, &invite_code).await?;
782    let minimum_amount_required = note_denomination * u64::from(minimum_notes);
783
784    reissue_initial_notes(initial_notes, &coordinator, &event_sender).await?;
785    get_required_notes(&coordinator, minimum_amount_required, &event_sender).await?;
786
787    info!(
788        "Reminting {minimum_notes} notes of denomination {note_denomination} for {users} users, {notes_per_user} notes per user (this may take a while if the number of users/notes is high)"
789    );
790    remint_denomination(&coordinator, note_denomination, minimum_notes).await?;
791
792    print_coordinator_notes(&coordinator).await?;
793
794    let users_clients = get_users_clients(users, db_path, invite_code.clone()).await?;
795
796    let mut users_notes =
797        get_notes_for_users(users, notes_per_user, coordinator, note_denomination).await?;
798
799    info!("Starting user tasks");
800    let futures = users_clients
801        .into_iter()
802        .enumerate()
803        .map(|(u, client)| {
804            let u = u as u16;
805            let oob_notes = users_notes.remove(&u).unwrap();
806            let event_sender = event_sender.clone();
807            let f: BoxFuture<_> = Box::pin(do_ln_circular_test_user_task(
808                format!("User {u}:"),
809                client,
810                invite_code.clone(),
811                oob_notes,
812                test_duration,
813                ln_payment_sleep,
814                invoice_amount,
815                strategy,
816                event_sender,
817            ));
818            f
819        })
820        .collect::<Vec<_>>();
821
822    Ok(futures)
823}
824
825#[allow(clippy::too_many_arguments)]
826async fn do_ln_circular_test_user_task(
827    prefix: String,
828    client: ClientHandleArc,
829    invite_code: Option<InviteCode>,
830    oob_notes: Vec<OOBNotes>,
831    test_duration: Duration,
832    ln_payment_sleep: Duration,
833    invoice_amount: Amount,
834    strategy: LnCircularStrategy,
835    event_sender: mpsc::UnboundedSender<MetricEvent>,
836) -> anyhow::Result<()> {
837    for oob_note in oob_notes {
838        let amount = oob_note.total_amount();
839        reissue_notes(&client, oob_note, &event_sender)
840            .await
841            .map_err(|e| anyhow!("while reissuing initial {amount}: {e}"))?;
842    }
843    let initial_time = fedimint_core::time::now();
844    let still_ontime = || async {
845        fedimint_core::time::now()
846            .duration_since(initial_time)
847            .expect("time to work")
848            <= test_duration
849    };
850    let sleep_a_bit = || async {
851        if still_ontime().await {
852            fedimint_core::task::sleep(ln_payment_sleep).await;
853        }
854    };
855    match strategy {
856        LnCircularStrategy::TwoGateways => {
857            let invoice_generation = LnInvoiceGeneration::LdkLightningCli;
858            while still_ontime().await {
859                let gateway_id = get_gateway_id(invoice_generation).await?;
860                let ln_gateway = get_lightning_gateway(&client, Some(gateway_id)).await;
861                run_two_gateways_strategy(
862                    &prefix,
863                    &invoice_generation,
864                    &invoice_amount,
865                    &event_sender,
866                    &client,
867                    ln_gateway,
868                )
869                .await?;
870                sleep_a_bit().await;
871            }
872        }
873        LnCircularStrategy::SelfPayment => {
874            while still_ontime().await {
875                do_self_payment(&prefix, &client, invoice_amount, &event_sender).await?;
876                sleep_a_bit().await;
877            }
878        }
879        LnCircularStrategy::PartnerPingPong => {
880            let (partner, _) = build_client(invite_code, None).await?;
881            while still_ontime().await {
882                do_partner_ping_pong(&prefix, &client, &partner, invoice_amount, &event_sender)
883                    .await?;
884                sleep_a_bit().await;
885            }
886        }
887    }
888    Ok(())
889}
890
891const GATEWAY_CREATE_INVOICE: &str = "gateway_create_invoice";
892
893async fn run_two_gateways_strategy(
894    prefix: &str,
895    invoice_generation: &LnInvoiceGeneration,
896    invoice_amount: &Amount,
897    event_sender: &mpsc::UnboundedSender<MetricEvent>,
898    client: &ClientHandleArc,
899    ln_gateway: Option<LightningGateway>,
900) -> Result<(), anyhow::Error> {
901    let create_invoice_time = fedimint_core::time::now();
902    match *invoice_generation {
903        LnInvoiceGeneration::LdkLightningCli => {
904            let invoice = ldk_create_invoice(*invoice_amount).await?;
905            let elapsed = create_invoice_time.elapsed()?;
906            info!("Created invoice using LDK in {elapsed:?}");
907            event_sender.send(MetricEvent {
908                name: GATEWAY_CREATE_INVOICE.into(),
909                duration: elapsed,
910            })?;
911            gateway_pay_invoice(
912                prefix,
913                "LND",
914                client,
915                invoice.clone(),
916                event_sender,
917                ln_gateway.clone(),
918            )
919            .await?;
920            ldk_wait_invoice_payment(&invoice).await?;
921            let (operation_id, invoice) =
922                client_create_invoice(client, *invoice_amount, event_sender, ln_gateway).await?;
923            let pay_invoice_time = fedimint_core::time::now();
924            ldk_pay_invoice(invoice).await?;
925            wait_invoice_payment(
926                prefix,
927                "LND",
928                client,
929                operation_id,
930                event_sender,
931                pay_invoice_time,
932            )
933            .await?;
934        }
935    }
936    Ok(())
937}
938
939async fn do_self_payment(
940    prefix: &str,
941    client: &ClientHandleArc,
942    invoice_amount: Amount,
943    event_sender: &mpsc::UnboundedSender<MetricEvent>,
944) -> anyhow::Result<()> {
945    let (operation_id, invoice) =
946        client_create_invoice(client, invoice_amount, event_sender, None).await?;
947    let pay_invoice_time = fedimint_core::time::now();
948    let lightning_module = client.get_first_module::<LightningClientModule>()?;
949    //let gateway = lightning_module.select_active_gateway_opt().await;
950    lightning_module
951        .pay_bolt11_invoice(None, invoice, ())
952        .await?;
953    wait_invoice_payment(
954        prefix,
955        "gateway",
956        client,
957        operation_id,
958        event_sender,
959        pay_invoice_time,
960    )
961    .await?;
962    Ok(())
963}
964
965async fn do_partner_ping_pong(
966    prefix: &str,
967    client: &ClientHandleArc,
968    partner: &ClientHandleArc,
969    invoice_amount: Amount,
970    event_sender: &mpsc::UnboundedSender<MetricEvent>,
971) -> anyhow::Result<()> {
972    // Ping (partner creates invoice, client pays)
973    let (operation_id, invoice) =
974        client_create_invoice(partner, invoice_amount, event_sender, None).await?;
975    let pay_invoice_time = fedimint_core::time::now();
976    let lightning_module = client.get_first_module::<LightningClientModule>()?;
977    // TODO: Select random gateway?
978    //let gateway = lightning_module.select_active_gateway_opt().await;
979    lightning_module
980        .pay_bolt11_invoice(None, invoice, ())
981        .await?;
982    wait_invoice_payment(
983        prefix,
984        "gateway",
985        partner,
986        operation_id,
987        event_sender,
988        pay_invoice_time,
989    )
990    .await?;
991    // Pong (client creates invoice, partner pays)
992    let (operation_id, invoice) =
993        client_create_invoice(client, invoice_amount, event_sender, None).await?;
994    let pay_invoice_time = fedimint_core::time::now();
995    let partner_lightning_module = partner.get_first_module::<LightningClientModule>()?;
996    //let gateway = partner_lightning_module.select_active_gateway_opt().await;
997    // TODO: Select random gateway?
998    partner_lightning_module
999        .pay_bolt11_invoice(None, invoice, ())
1000        .await?;
1001    wait_invoice_payment(
1002        prefix,
1003        "gateway",
1004        client,
1005        operation_id,
1006        event_sender,
1007        pay_invoice_time,
1008    )
1009    .await?;
1010    Ok(())
1011}
1012
1013async fn wait_invoice_payment(
1014    prefix: &str,
1015    gateway_name: &str,
1016    client: &ClientHandleArc,
1017    operation_id: fedimint_core::core::OperationId,
1018    event_sender: &mpsc::UnboundedSender<MetricEvent>,
1019    pay_invoice_time: std::time::SystemTime,
1020) -> anyhow::Result<()> {
1021    let elapsed = pay_invoice_time.elapsed()?;
1022    info!("{prefix} Invoice payment receive started using {gateway_name} in {elapsed:?}");
1023    event_sender.send(MetricEvent {
1024        name: format!("gateway_{gateway_name}_payment_received_started"),
1025        duration: elapsed,
1026    })?;
1027    let lightning_module = client.get_first_module::<LightningClientModule>()?;
1028    let mut updates = lightning_module
1029        .subscribe_ln_receive(operation_id)
1030        .await?
1031        .into_stream();
1032    wait_invoice_payment_updates(
1033        prefix,
1034        gateway_name,
1035        operation_id,
1036        event_sender,
1037        pay_invoice_time,
1038        &mut updates,
1039    )
1040    .await
1041}
1042
1043async fn wait_invoice_payment_updates(
1044    prefix: &str,
1045    gateway_name: &str,
1046    operation_id: fedimint_core::core::OperationId,
1047    event_sender: &mpsc::UnboundedSender<MetricEvent>,
1048    pay_invoice_time: std::time::SystemTime,
1049    updates: &mut (impl futures::Stream<Item = LnReceiveState> + Unpin),
1050) -> anyhow::Result<()> {
1051    while let Some(update) = updates.next().await {
1052        debug!(%prefix, ?update, "Invoice payment update");
1053        match update {
1054            LnReceiveState::Claimed => {
1055                let elapsed: Duration = pay_invoice_time.elapsed()?;
1056                info!("{prefix} Invoice payment received on {gateway_name} in {elapsed:?}");
1057                event_sender.send(MetricEvent {
1058                    name: "gateway_payment_received_success".into(),
1059                    duration: elapsed,
1060                })?;
1061                event_sender.send(MetricEvent {
1062                    name: format!("gateway_{gateway_name}_payment_received_success"),
1063                    duration: elapsed,
1064                })?;
1065                return Ok(());
1066            }
1067            LnReceiveState::Canceled { reason } => {
1068                let elapsed: Duration = pay_invoice_time.elapsed()?;
1069                info!(
1070                    "{prefix} Invoice payment receive was canceled on {gateway_name}: {reason} in {elapsed:?}"
1071                );
1072                event_sender.send(MetricEvent {
1073                    name: "gateway_payment_received_canceled".into(),
1074                    duration: elapsed,
1075                })?;
1076                bail!(
1077                    "Invoice payment receive for operation {operation_id:?} was canceled on \
1078                     {gateway_name}: {reason}"
1079                );
1080            }
1081            _ => {}
1082        }
1083    }
1084    bail!(
1085        "Invoice payment receive stream ended before operation {operation_id:?} was claimed on \
1086         {gateway_name}"
1087    )
1088}
1089
1090async fn client_create_invoice(
1091    client: &ClientHandleArc,
1092    invoice_amount: Amount,
1093    event_sender: &mpsc::UnboundedSender<MetricEvent>,
1094    ln_gateway: Option<LightningGateway>,
1095) -> anyhow::Result<(fedimint_core::core::OperationId, Bolt11Invoice)> {
1096    let create_invoice_time = fedimint_core::time::now();
1097    let lightning_module = client.get_first_module::<LightningClientModule>()?;
1098    let desc = Description::new("test".to_string())?;
1099    let (operation_id, invoice, _) = lightning_module
1100        .create_bolt11_invoice(
1101            invoice_amount,
1102            Bolt11InvoiceDescription::Direct(desc),
1103            None,
1104            (),
1105            ln_gateway,
1106        )
1107        .await?;
1108    let elapsed = create_invoice_time.elapsed()?;
1109    info!("Created invoice using gateway in {elapsed:?}");
1110    event_sender.send(MetricEvent {
1111        name: GATEWAY_CREATE_INVOICE.into(),
1112        duration: elapsed,
1113    })?;
1114    Ok((operation_id, invoice))
1115}
1116
1117fn test_download_config(
1118    endpoints: &ConnectorRegistry,
1119    invite_code: &InviteCode,
1120    users: u16,
1121    event_sender: &mpsc::UnboundedSender<MetricEvent>,
1122) -> Vec<BoxFuture<'static, anyhow::Result<()>>> {
1123    (0..users)
1124        .map(move |_| {
1125            let invite_code = invite_code.clone();
1126            let event_sender = event_sender.clone();
1127            let endpoints = endpoints.clone();
1128            let f: BoxFuture<_> = Box::pin(async move {
1129                let m = fedimint_core::time::now();
1130                let _ = download_from_invite_code(&endpoints, &invite_code).await?;
1131                event_sender.send(MetricEvent {
1132                    name: "download_client_config".into(),
1133                    duration: m.elapsed()?,
1134                })?;
1135                Ok(())
1136            });
1137            f
1138        })
1139        .collect()
1140}
1141
1142async fn test_connect_raw_client(
1143    endpoints: &ConnectorRegistry,
1144    invite_code: InviteCode,
1145    users: u16,
1146    duration: Duration,
1147    timeout: Duration,
1148    limit_endpoints: Option<usize>,
1149    event_sender: mpsc::UnboundedSender<MetricEvent>,
1150) -> anyhow::Result<Vec<BoxFuture<'static, anyhow::Result<()>>>> {
1151    use jsonrpsee_core::client::ClientT;
1152    use jsonrpsee_ws_client::WsClientBuilder;
1153
1154    let (mut cfg, _) = download_from_invite_code(endpoints, &invite_code).await?;
1155
1156    if let Some(limit_endpoints) = limit_endpoints {
1157        cfg.global.api_endpoints = cfg
1158            .global
1159            .api_endpoints
1160            .into_iter()
1161            .take(limit_endpoints)
1162            .collect();
1163        info!("Limiting endpoints to {:?}", cfg.global.api_endpoints);
1164    }
1165
1166    info!("Connecting to {users} clients");
1167    let clients = (0..users)
1168        .flat_map(|_| {
1169            cfg.global.api_endpoints.values().map(|url| async {
1170                let ws_client = WsClientBuilder::default()
1171                    .request_timeout(timeout)
1172                    .connection_timeout(timeout)
1173                    .build(url_to_string_with_default_port(&url.url))
1174                    .await?;
1175                Ok::<_, anyhow::Error>(ws_client)
1176            })
1177        })
1178        .collect::<Vec<_>>();
1179    let clients = futures::future::try_join_all(clients).await?;
1180    info!("Keeping {users} clients connected for {duration:?}");
1181    Ok(clients
1182        .into_iter()
1183        .map(|client| {
1184            let event_sender = event_sender.clone();
1185            let f: BoxFuture<_> = Box::pin(async move {
1186                let initial_time = fedimint_core::time::now();
1187                while initial_time.elapsed()? < duration {
1188                    let m = fedimint_core::time::now();
1189                    let _epoch: u64 = client
1190                        .request::<_, _>(SESSION_COUNT_ENDPOINT, vec![ApiRequestErased::default()])
1191                        .await?;
1192                    event_sender.send(MetricEvent {
1193                        name: SESSION_COUNT_ENDPOINT.into(),
1194                        duration: m.elapsed()?,
1195                    })?;
1196                    fedimint_core::task::sleep(Duration::from_secs(1)).await;
1197                }
1198                Ok(())
1199            });
1200            f
1201        })
1202        .collect())
1203}
1204
1205fn url_to_string_with_default_port(url: &SafeUrl) -> String {
1206    format!(
1207        "{}://{}:{}{}",
1208        url.scheme(),
1209        url.host().expect("Asserted on construction"),
1210        url.port_or_known_default()
1211            .expect("Asserted on construction"),
1212        url.path()
1213    )
1214}
1215
1216async fn handle_metrics_summary(
1217    opts: Opts,
1218    mut event_receiver: mpsc::UnboundedReceiver<MetricEvent>,
1219) -> anyhow::Result<()> {
1220    let timestamp_seconds = fedimint_core::time::duration_since_epoch().as_secs();
1221    let mut metrics_json_output_files = vec![];
1222    let mut previous_metrics = vec![];
1223    let mut comparison_output = None;
1224    if let Some(archive_dir) = opts.archive_dir {
1225        let mut archive_metrics = archive_dir.join("metrics");
1226        archive_metrics.push(opts.users.to_string());
1227        tokio::fs::create_dir_all(&archive_metrics).await?;
1228        let mut archive_comparisons = archive_dir.join("comparisons");
1229        archive_comparisons.push(opts.users.to_string());
1230        tokio::fs::create_dir_all(&archive_comparisons).await?;
1231
1232        let latest_metrics_file = std::fs::read_dir(&archive_metrics)?
1233            .map(|entry| {
1234                let entry = entry.unwrap();
1235                let metadata = entry.metadata().unwrap();
1236                let created = metadata
1237                    .created()
1238                    .unwrap_or_else(|_| metadata.modified().unwrap());
1239                (entry, created)
1240            })
1241            .max_by_key(|(_entry, created)| created.to_owned())
1242            .map(|(entry, _)| entry.path());
1243        if let Some(latest_metrics_file) = latest_metrics_file {
1244            let display_path = latest_metrics_file.display();
1245            let latest_metrics_file = tokio::fs::File::open(&latest_metrics_file)
1246                .await
1247                .with_context(|| format!("Failed to open {display_path}"))?;
1248            let mut lines = tokio::io::BufReader::new(latest_metrics_file).lines();
1249            while let Some(line) = lines.next_line().await? {
1250                match serde_json::from_str::<EventMetricSummary>(&line) {
1251                    Ok(metric) => {
1252                        previous_metrics.push(metric);
1253                    }
1254                    Err(e) => {
1255                        warn!("Failed to parse previous metric: {e:?}");
1256                    }
1257                }
1258            }
1259        }
1260        let new_metric_output = archive_metrics.join(format!("{timestamp_seconds}.json",));
1261        let new_metric_output = BufWriter::new(
1262            OpenOptions::new()
1263                .write(true)
1264                .create(true)
1265                .truncate(true)
1266                .open(new_metric_output)
1267                .await?,
1268        );
1269        metrics_json_output_files.push(new_metric_output);
1270        if !previous_metrics.is_empty() {
1271            let new_comparison_output =
1272                archive_comparisons.join(format!("{timestamp_seconds}.json",));
1273            comparison_output = Some(BufWriter::new(
1274                OpenOptions::new()
1275                    .write(true)
1276                    .create(true)
1277                    .truncate(true)
1278                    .open(new_comparison_output)
1279                    .await?,
1280            ));
1281        }
1282    }
1283    if let Some(metrics_json_output) = opts.metrics_json_output {
1284        metrics_json_output_files.push(BufWriter::new(
1285            tokio::fs::OpenOptions::new()
1286                .write(true)
1287                .create(true)
1288                .truncate(true)
1289                .open(metrics_json_output)
1290                .await?,
1291        ));
1292    }
1293    let mut results = BTreeMap::new();
1294    while let Some(event) = event_receiver.recv().await {
1295        let entry = results.entry(event.name).or_insert_with(Vec::new);
1296        entry.push(event.duration);
1297    }
1298    let mut previous_metrics = previous_metrics
1299        .into_iter()
1300        .map(|metric| (metric.name.clone(), metric))
1301        .collect::<HashMap<_, _>>();
1302    for (k, mut v) in results {
1303        v.sort();
1304        let n = v.len();
1305        let max = v.iter().last().unwrap();
1306        let min = v.first().unwrap();
1307        let median = v[n / 2];
1308        let sum: Duration = v.iter().sum();
1309        let avg = sum / n as u32;
1310        let metric_summary = EventMetricSummary {
1311            name: k.clone(),
1312            users: u64::from(opts.users),
1313            n: n as u64,
1314            avg_ms: avg.as_millis(),
1315            median_ms: median.as_millis(),
1316            max_ms: max.as_millis(),
1317            min_ms: min.as_millis(),
1318            timestamp_seconds,
1319        };
1320        let comparison = if let Some(previous_metric) = previous_metrics.remove(&k) {
1321            if previous_metric.n == metric_summary.n {
1322                fn calculate_gain(current: u128, previous: u128) -> f64 {
1323                    current as f64 / previous as f64
1324                }
1325                let comparison = EventMetricComparison {
1326                    avg_ms_gain: calculate_gain(metric_summary.avg_ms, previous_metric.avg_ms),
1327                    median_ms_gain: calculate_gain(
1328                        metric_summary.median_ms,
1329                        previous_metric.median_ms,
1330                    ),
1331                    max_ms_gain: calculate_gain(metric_summary.max_ms, previous_metric.max_ms),
1332                    min_ms_gain: calculate_gain(metric_summary.min_ms, previous_metric.min_ms),
1333                    current: metric_summary.clone(),
1334                    previous: previous_metric,
1335                };
1336                if let Some(comparison_output) = &mut comparison_output {
1337                    let comparison_json =
1338                        serde_json::to_string(&comparison).expect("to be serializable");
1339                    comparison_output
1340                        .write_all(format!("{comparison_json}\n").as_bytes())
1341                        .await
1342                        .expect("to write on file");
1343                }
1344                Some(comparison)
1345            } else {
1346                info!(
1347                    "Skipping comparison for {k} because previous metric has different n ({} vs {})",
1348                    previous_metric.n, metric_summary.n
1349                );
1350                None
1351            }
1352        } else {
1353            None
1354        };
1355        if let Some(comparison) = comparison {
1356            println!(
1357                "{n} {k}: avg {avg:?}, median {median:?}, max {max:?}, min {min:?} (compared to previous: {comparison})"
1358            );
1359        } else {
1360            println!("{n} {k}: avg {avg:?}, median {median:?}, max {max:?}, min {min:?}");
1361        }
1362        let metric_summary_json =
1363            serde_json::to_string(&metric_summary).expect("to be serializable");
1364        for metrics_json_output_file in &mut metrics_json_output_files {
1365            metrics_json_output_file
1366                .write_all(format!("{metric_summary_json}\n").as_bytes())
1367                .await
1368                .expect("to write on file");
1369        }
1370    }
1371    for mut output in metrics_json_output_files {
1372        output.flush().await?;
1373    }
1374    if let Some(mut output) = comparison_output {
1375        output.flush().await?;
1376    }
1377    Ok(())
1378}
1379
1380async fn get_gateway_id(generate_invoice_with: LnInvoiceGeneration) -> anyhow::Result<String> {
1381    let gateway_json = match generate_invoice_with {
1382        LnInvoiceGeneration::LdkLightningCli => {
1383            // If we are paying a lnd invoice, we use the LDK node
1384            cmd!(GatewayLndCli, "info").out_json().await
1385        }
1386    }?;
1387
1388    let gatewayd_version = devimint::util::Gatewayd::version_or_default().await;
1389    let gateway_id = if gatewayd_version < *VERSION_0_10_0_ALPHA {
1390        gateway_json["gateway_id"]
1391            .as_str()
1392            .context("gateway_id must be a string")?
1393            .to_owned()
1394    } else {
1395        gateway_json["registrations"]["http"][1]
1396            .as_str()
1397            .context("gateway id must be a string")?
1398            .to_owned()
1399    };
1400
1401    Ok(gateway_id)
1402}