Skip to main content

lnv2_module_tests/
tests.rs

1use std::collections::BTreeMap;
2use std::path::PathBuf;
3use std::str::FromStr;
4use std::time::Duration;
5
6use anyhow::{Context as _, bail, ensure};
7use bitcoin::hashes::{Hash, sha256};
8use clap::{Parser, Subcommand};
9use devimint::devfed::{DevFed, DevJitFed};
10use devimint::envs::FM_CLIENT_DIR_ENV;
11use devimint::federation::{Client, Federation};
12use devimint::util::{ProcessManager, almost_equal, poll_simple};
13use devimint::version_constants::{
14    VERSION_0_10_0_ALPHA, VERSION_0_11_0_ALPHA, VERSION_0_12_0_ALPHA, VERSION_0_12_2_ALPHA,
15    VERSION_0_14_0_ALPHA,
16};
17use devimint::{Gatewayd, cmd, util};
18use fedimint_core::core::{Account, OperationId};
19use fedimint_core::encoding::Encodable;
20use fedimint_core::secp256k1::rand::rngs::OsRng;
21use fedimint_core::secp256k1::{SECP256K1, SecretKey};
22use fedimint_core::task::{self};
23use fedimint_core::util::{backoff_util, retry, write_overwrite_async};
24use fedimint_lnurl::{LnurlResponse, VerifyResponse, parse_lnurl};
25use fedimint_lnv2_client::FinalSendOperationState;
26use lightning_invoice::{Bolt11Invoice, Bolt11InvoiceDescriptionRef};
27use serde::Deserialize;
28use serde_json::Value;
29use tokio::try_join;
30use tracing::info;
31
32#[path = "common.rs"]
33mod common;
34
35async fn module_is_present(client: &Client, kind: &str) -> anyhow::Result<bool> {
36    let modules = cmd!(client, "module").out_json().await?;
37
38    let modules = modules["list"].as_array().expect("module list is an array");
39
40    Ok(modules.iter().any(|m| m["kind"].as_str() == Some(kind)))
41}
42
43async fn assert_module_sanity(client: &Client) -> anyhow::Result<()> {
44    if !devimint::util::is_backwards_compatibility_test() {
45        ensure!(
46            !module_is_present(client, "ln").await?,
47            "ln module should not be present"
48        );
49    }
50
51    Ok(())
52}
53
54#[derive(Parser)]
55#[command(name = "lnv2-module-tests")]
56#[command(about = "LNv2 module integration tests", long_about = None)]
57struct Cli {
58    #[command(subcommand)]
59    command: Option<Commands>,
60}
61
62#[derive(Subcommand)]
63enum Commands {
64    /// Run gateway registration tests
65    GatewayRegistration,
66    /// Run payment tests
67    Payments,
68    /// Run LNURL pay tests
69    LnurlPay,
70    /// Test LNURL receives after recovery from seed
71    LnurlRecovery,
72    /// Two clients racing to pay the same invoice settle exactly once
73    DuplicatePayment,
74    /// The gateway refuses to fund a receive while the federation cannot
75    /// reach consensus
76    FederationOutage,
77    /// Run direct HTLC tests between two clients without a gateway
78    DirectHtlc,
79    /// Funds bound to a non-primary account stay there through Lightning,
80    /// on-chain and recovery from seed; needs mintv2 and walletv2
81    Accounts,
82}
83
84#[tokio::main]
85async fn main() -> anyhow::Result<()> {
86    let cli = Cli::parse();
87
88    // The duplicate-payment test needs a second federation; every other test
89    // uses a single one.
90    let num_feds = if matches!(cli.command, None | Some(Commands::DuplicatePayment)) {
91        2
92    } else {
93        1
94    };
95
96    devimint::run_devfed_test()
97        .num_feds(num_feds)
98        .call(|dev_fed, process_mgr| async move {
99            if !devimint::util::supports_lnv2() {
100                info!("lnv2 is disabled, skipping");
101                return Ok(());
102            }
103
104            match &cli.command {
105                Some(Commands::GatewayRegistration) => {
106                    test_gateway_registration(&dev_fed).await?;
107                }
108                Some(Commands::Payments) => {
109                    test_payments(&dev_fed).await?;
110                }
111                Some(Commands::LnurlPay) => {
112                    pegin_gateways(&dev_fed).await?;
113                    test_lnurl_pay(&dev_fed).await?;
114                }
115                Some(Commands::LnurlRecovery) => {
116                    pegin_gateways(&dev_fed).await?;
117                    test_lnurl_recovery(&dev_fed).await?;
118                }
119                Some(Commands::DuplicatePayment) => {
120                    pegin_gateways(&dev_fed).await?;
121                    test_duplicate_payment(&dev_fed, &process_mgr).await?;
122                }
123                Some(Commands::DirectHtlc) => {
124                    test_direct_htlc(&dev_fed).await?;
125                }
126                Some(Commands::Accounts) => {
127                    test_accounts(&dev_fed).await?;
128                }
129                Some(Commands::FederationOutage) => {
130                    pegin_gateways(&dev_fed).await?;
131                    let dev_fed = dev_fed.to_dev_fed(&process_mgr).await?;
132                    test_federation_outage(dev_fed, &process_mgr).await?;
133                }
134                None => {
135                    // Run all tests if no subcommand is specified
136                    test_gateway_registration(&dev_fed).await?;
137                    // Before the payments test funds the gateways, as it
138                    // relies on an unfunded gateway to refund a payment.
139                    test_accounts(&dev_fed).await?;
140                    test_payments(&dev_fed).await?;
141                    test_duplicate_payment(&dev_fed, &process_mgr).await?;
142                    test_lnurl_pay(&dev_fed).await?;
143                    test_lnurl_recovery(&dev_fed).await?;
144                    test_direct_htlc(&dev_fed).await?;
145                    // Last, since it takes ownership of the federation to
146                    // stop and restart guardians.
147                    pegin_gateways(&dev_fed).await?;
148                    let dev_fed = dev_fed.to_dev_fed(&process_mgr).await?;
149                    test_federation_outage(dev_fed, &process_mgr).await?;
150                }
151            }
152
153            info!("Testing LNV2 is complete!");
154
155            Ok(())
156        })
157        .await
158}
159
160async fn pegin_gateways(dev_fed: &DevJitFed) -> anyhow::Result<()> {
161    info!("Pegging-in gateways...");
162
163    let federation = dev_fed.fed().await?;
164
165    let gw_lnd = dev_fed.gw_lnd().await?;
166    let gw_ldk = dev_fed.gw_ldk().await?;
167
168    federation
169        .pegin_gateways(1_000_000, vec![gw_lnd, gw_ldk])
170        .await?;
171
172    Ok(())
173}
174
175async fn test_gateway_registration(dev_fed: &DevJitFed) -> anyhow::Result<()> {
176    let client = dev_fed
177        .fed()
178        .await?
179        .new_joined_client("lnv2-test-gateway-registration-client")
180        .await?;
181
182    assert_module_sanity(&client).await?;
183
184    let gw_lnd = dev_fed.gw_lnd().await?;
185    let gw_ldk = dev_fed.gw_ldk_connected().await?;
186
187    let gateways = [gw_lnd.addr.clone(), gw_ldk.addr.clone()];
188
189    info!("Removing the gateways devimint added on startup...");
190
191    let peers = (0..dev_fed.fed().await?.members.len()).collect::<Vec<usize>>();
192    remove_all_gateways(&client, &peers).await?;
193
194    info!("Testing registration of gateways...");
195
196    for gateway in &gateways {
197        for peer in 0..dev_fed.fed().await?.members.len() {
198            assert!(add_gateway(&client, peer, gateway).await?);
199        }
200    }
201
202    assert_eq!(
203        cmd!(client, "module", "lnv2", "gateways", "list")
204            .out_json()
205            .await?
206            .as_array()
207            .expect("JSON Value is not an array")
208            .len(),
209        2
210    );
211
212    assert_eq!(
213        cmd!(client, "module", "lnv2", "gateways", "list", "--peer", "0")
214            .out_json()
215            .await?
216            .as_array()
217            .expect("JSON Value is not an array")
218            .len(),
219        2
220    );
221
222    info!("Testing selection of gateways...");
223
224    assert!(
225        gateways.contains(
226            &cmd!(client, "module", "lnv2", "gateways", "select")
227                .out_json()
228                .await?
229                .as_str()
230                .expect("JSON Value is not a string")
231                .to_string()
232        )
233    );
234
235    cmd!(client, "module", "lnv2", "gateways", "map")
236        .out_json()
237        .await?;
238
239    for _ in 0..10 {
240        for gateway in &gateways {
241            let invoice = common::receive(&client, gateway, 1_000_000).await?.0;
242
243            assert_eq!(
244                cmd!(
245                    client,
246                    "module",
247                    "lnv2",
248                    "gateways",
249                    "select",
250                    "--invoice",
251                    invoice.to_string()
252                )
253                .out_json()
254                .await?
255                .as_str()
256                .expect("JSON Value is not a string"),
257                gateway
258            )
259        }
260    }
261
262    info!("Testing deregistration of gateways...");
263
264    for gateway in &gateways {
265        for peer in 0..dev_fed.fed().await?.members.len() {
266            assert!(remove_gateway(&client, peer, gateway).await?);
267        }
268    }
269
270    assert!(
271        cmd!(client, "module", "lnv2", "gateways", "list")
272            .out_json()
273            .await?
274            .as_array()
275            .expect("JSON Value is not an array")
276            .is_empty(),
277    );
278
279    assert!(
280        cmd!(client, "module", "lnv2", "gateways", "list", "--peer", "0")
281            .out_json()
282            .await?
283            .as_array()
284            .expect("JSON Value is not an array")
285            .is_empty()
286    );
287
288    Ok(())
289}
290
291async fn test_payments(dev_fed: &DevJitFed) -> anyhow::Result<()> {
292    let federation = dev_fed.fed().await?;
293
294    let client = federation
295        .new_joined_client("lnv2-test-payments-client")
296        .await?;
297
298    assert_module_sanity(&client).await?;
299
300    federation.pegin_client(10_000, &client).await?;
301
302    almost_equal(client.balance().await?, 10_000 * 1000, 500_000).unwrap();
303
304    let gw_lnd = dev_fed.gw_lnd().await?;
305    let gw_ldk = dev_fed.gw_ldk().await?;
306    let lnd = dev_fed.lnd().await?;
307
308    // Gateways before 0.12 forward *every* pending HOLD invoice on their
309    // Lightning node to the gateway's payment handler and cancel the ones
310    // they cannot match to a registered LNv2 incoming contract. That includes
311    // the HOLD invoice this test creates directly on the LND node the LND
312    // gateway shares, so the gateway's cancel races this test's settle and
313    // fails it with `invoice already canceled` whenever the gateway wins.
314    // Only exercise HOLD invoices against gateways that filter out HOLD
315    // invoices they did not create themselves.
316    let test_hold_invoice = gw_lnd.gatewayd_version >= *VERSION_0_12_0_ALPHA;
317
318    let hold_invoice = if test_hold_invoice {
319        Some(lnd.create_hold_invoice(60000).await?)
320    } else {
321        info!(
322            gatewayd_version = %gw_lnd.gatewayd_version,
323            "Skipping HOLD invoice payment: gateway cancels HOLD invoices it did not create"
324        );
325
326        None
327    };
328
329    let gateway_pairs = [(gw_lnd, gw_ldk), (gw_ldk, gw_lnd)];
330
331    let gateway_matrix = [
332        (gw_lnd, gw_lnd),
333        (gw_lnd, gw_ldk),
334        (gw_ldk, gw_lnd),
335        (gw_ldk, gw_ldk),
336    ];
337
338    info!("Testing refund of circular payments...");
339
340    for (gw_send, gw_receive) in gateway_matrix {
341        info!(
342            "Testing refund of payment: client -> {} -> {} -> client",
343            gw_send.ln.ln_type(),
344            gw_receive.ln.ln_type()
345        );
346
347        let invoice = common::receive(&client, &gw_receive.addr, 1_000_000)
348            .await?
349            .0;
350
351        let state = common::send(&client, &gw_send.addr, &invoice.to_string()).await?;
352        assert!(matches!(state, FinalSendOperationState::Refunded));
353    }
354
355    pegin_gateways(dev_fed).await?;
356
357    info!("Testing circular payments...");
358
359    for (gw_send, gw_receive) in gateway_matrix {
360        info!(
361            "Testing payment: client -> {} -> {} -> client",
362            gw_send.ln.ln_type(),
363            gw_receive.ln.ln_type()
364        );
365
366        let (invoice, receive_op) = common::receive(&client, &gw_receive.addr, 1_000_000).await?;
367
368        let state = common::send(&client, &gw_send.addr, &invoice.to_string()).await?;
369        assert!(matches!(state, FinalSendOperationState::Success(_)));
370
371        common::await_receive_claimed(&client, receive_op).await?;
372    }
373
374    info!("Testing payments from client to gateways...");
375
376    for (gw_send, gw_receive) in gateway_pairs {
377        info!(
378            "Testing payment: client -> {} -> {}",
379            gw_send.ln.ln_type(),
380            gw_receive.ln.ln_type()
381        );
382
383        let invoice = gw_receive.client().create_invoice(1_000_000).await?;
384
385        let state = common::send(&client, &gw_send.addr, &invoice.to_string()).await?;
386        assert!(matches!(state, FinalSendOperationState::Success(_)));
387    }
388
389    info!("Testing payments from gateways to client...");
390
391    for (gw_send, gw_receive) in gateway_pairs {
392        info!(
393            "Testing payment: {} -> {} -> client",
394            gw_send.ln.ln_type(),
395            gw_receive.ln.ln_type()
396        );
397
398        let (invoice, receive_op) = common::receive(&client, &gw_receive.addr, 1_000_000).await?;
399
400        gw_send.client().pay_invoice(invoice).await?;
401
402        common::await_receive_claimed(&client, receive_op).await?;
403    }
404
405    retry(
406        "Waiting for the full balance to become available to the client".to_string(),
407        backoff_util::background_backoff(),
408        || async {
409            ensure!(client.balance().await? >= 9000 * 1000);
410
411            Ok(())
412        },
413    )
414    .await?;
415
416    if let Some((hold_preimage, hold_invoice, hold_payment_hash)) = hold_invoice {
417        info!("Testing Client can pay LND HOLD invoice via LDK Gateway...");
418
419        let (state, _) = try_join!(
420            common::send(&client, &gw_ldk.addr, &hold_invoice),
421            lnd.settle_hold_invoice(hold_preimage, hold_payment_hash),
422        )?;
423        assert!(matches!(state, FinalSendOperationState::Success(_)));
424    }
425
426    info!("Testing LNv2 lightning fees...");
427
428    let fed_id = federation.calculate_federation_id();
429
430    gw_lnd
431        .client()
432        .set_federation_routing_fee(fed_id.clone(), 0, 0)
433        .await?;
434
435    gw_lnd
436        .client()
437        .set_federation_transaction_fee(fed_id.clone(), 0, 0)
438        .await?;
439
440    // Gateway pays: 1_000 msat LNv2 federation base fee. Gateway receives:
441    // 1_000_000 payment.
442    test_fees(fed_id, &client, gw_lnd, gw_ldk, 1_000_000 - 1_000).await?;
443
444    let online_peers: Vec<usize> = federation.members.keys().copied().collect();
445
446    test_iroh_payment(&client, gw_lnd, gw_ldk, &online_peers).await?;
447
448    info!("Testing payment summary...");
449
450    let lnd_payment_summary = gw_lnd.client().payment_summary().await?;
451
452    assert_eq!(lnd_payment_summary.outgoing.total_success, 5);
453    assert_eq!(lnd_payment_summary.outgoing.total_failure, 2);
454    assert_eq!(lnd_payment_summary.incoming.total_success, 4);
455    assert_eq!(lnd_payment_summary.incoming.total_failure, 0);
456
457    assert!(lnd_payment_summary.outgoing.median_latency.is_some());
458    assert!(lnd_payment_summary.outgoing.average_latency.is_some());
459    assert!(lnd_payment_summary.incoming.median_latency.is_some());
460    assert!(lnd_payment_summary.incoming.average_latency.is_some());
461
462    let ldk_payment_summary = gw_ldk.client().payment_summary().await?;
463
464    assert_eq!(
465        ldk_payment_summary.outgoing.total_success,
466        if test_hold_invoice { 4 } else { 3 }
467    );
468    assert_eq!(ldk_payment_summary.outgoing.total_failure, 2);
469    assert_eq!(ldk_payment_summary.incoming.total_success, 4);
470    assert_eq!(ldk_payment_summary.incoming.total_failure, 0);
471
472    assert!(ldk_payment_summary.outgoing.median_latency.is_some());
473    assert!(ldk_payment_summary.outgoing.average_latency.is_some());
474    assert!(ldk_payment_summary.incoming.median_latency.is_some());
475    assert!(ldk_payment_summary.incoming.average_latency.is_some());
476
477    Ok(())
478}
479
480/// Creates a fresh client and joins it to the federation identified by
481/// `invite_code`, unlike `Federation::new_joined_client` which always joins
482/// whichever federation last wrote the shared invite-code file.
483async fn join_client(name: &str, invite_code: &str) -> anyhow::Result<Client> {
484    let client = Client::create(name).await?;
485    client.join_federation(invite_code.to_string()).await?;
486    Ok(client)
487}
488
489/// Three independent clients — two in the primary federation and one in a
490/// second federation, all served by the same gateway — race to pay the *same*
491/// invoice. The gateway pays the invoice only once, so it may claim only one of
492/// the three outgoing contracts regardless of which federation funded them.
493/// Exactly one payment settles; how the other two end depends on the gateway
494/// version, see [`assert_duplicates_refunded`] and
495/// [`assert_duplicates_refused`].
496async fn test_duplicate_payment(
497    dev_fed: &DevJitFed,
498    process_mgr: &ProcessManager,
499) -> anyhow::Result<()> {
500    info!(
501        "Testing three clients across two federations paying the same invoice settle exactly once..."
502    );
503
504    // The LDK gateway is used to send: its `pay` is idempotent per payment hash,
505    // so all contracts share a single Lightning payment and preimage.
506    let gw_send = dev_fed.gw_ldk().await?;
507    let gw_receive = dev_fed.gw_lnd().await?;
508
509    // The cross-federation dedup lives in the gateway's global database, added in
510    // 0.12. Older gateways would claim both contracts, so skip against them.
511    if gw_send.gatewayd_version < *VERSION_0_12_0_ALPHA {
512        info!(
513            gatewayd_version = %gw_send.gatewayd_version,
514            "Skipping: gateway predates cross-federation outgoing payment dedup"
515        );
516        return Ok(());
517    }
518
519    let first_federation = dev_fed.fed().await?;
520
521    // `Federation::invite_code` reads a single shared invite-code file, so
522    // capture the primary federation's invite before spawning the second one
523    // overwrites it. Clients must be joined with the right invite explicitly;
524    // `new_joined_client` would join whichever federation wrote that file last.
525    let first_invite = first_federation.invite_code()?;
526
527    // A second federation, also served by the sending gateway, covers the
528    // cross-federation case: a contract funded in one federation must still
529    // block claiming a contract for the same invoice funded in another.
530    let second_federation = Federation::new(
531        process_mgr,
532        dev_fed.bitcoind().await?.clone(),
533        false,
534        false,
535        false,
536        1,
537        "lnv2-duplicate-payment".to_string(),
538    )
539    .await?;
540    let second_invite = second_federation.invite_code()?;
541
542    assert_ne!(
543        first_federation.calculate_federation_id(),
544        second_federation.calculate_federation_id(),
545        "the second federation must be distinct from the first"
546    );
547
548    // Spawning the second federation overwrote the shared invite-code file
549    // with its own invite. Restore the primary federation's invite now that
550    // both invites are captured: `Federation::invite_code` re-reads that file,
551    // so a later `new_joined_client` on the primary federation would otherwise
552    // join the second one, which is torn down when this test returns.
553    let client_dir: PathBuf = std::env::var(FM_CLIENT_DIR_ENV)?.parse()?;
554    write_overwrite_async(client_dir.join("invite-code"), &first_invite).await?;
555
556    gw_send.client().connect_fed(second_invite.clone()).await?;
557    second_federation
558        .pegin_gateways(1_000_000, vec![gw_send])
559        .await?;
560
561    let client_a = join_client("lnv2-duplicate-payment-a", &first_invite).await?;
562    let client_b = join_client("lnv2-duplicate-payment-b", &first_invite).await?;
563    let client_c = join_client("lnv2-duplicate-payment-c", &second_invite).await?;
564
565    first_federation.pegin_client(10_000, &client_a).await?;
566    first_federation.pegin_client(10_000, &client_b).await?;
567    second_federation.pegin_client(10_000, &client_c).await?;
568
569    // Control: the second-federation client can pay a *different* invoice through
570    // the gateway on its own. This proves the gateway serves the second
571    // federation, so any refund of the shared invoice below is caused by the
572    // duplicate payment rather than a second-federation setup problem.
573    let second_federation_id = second_federation.calculate_federation_id();
574    let second_balance_before_control = gw_send
575        .client()
576        .ecash_balance(second_federation_id.clone())
577        .await?;
578    let control_invoice = gw_receive
579        .client()
580        .create_invoice(1_000_000)
581        .await?
582        .to_string();
583    let control = common::send(&client_c, &gw_send.addr, &control_invoice).await?;
584    assert!(
585        matches!(control, FinalSendOperationState::Success(_)),
586        "second-federation control payment must succeed, got {control:?}"
587    );
588
589    // The sender sees the preimage once the gateway's claim is accepted, which
590    // can be before the claimed ecash reaches the gateway. Wait for it, so the
591    // gateway's balances only move below because of the shared invoice.
592    poll_simple(
593        "Waiting for the gateway to claim the control payment",
594        || async {
595            let balance = gw_send
596                .client()
597                .ecash_balance(second_federation_id.clone())
598                .await?;
599            ensure!(
600                balance > second_balance_before_control,
601                "gateway has not claimed the control payment yet"
602            );
603            Ok(())
604        },
605    )
606    .await?;
607
608    let invoice = gw_receive
609        .client()
610        .create_invoice(1_000_000)
611        .await?
612        .to_string();
613
614    // Before 0.12.2 the gateway claimed the payment image only after paying,
615    // and forfeited every later contract for it, so those were refunded at once.
616    // Since then it claims the image when it accepts a send and refuses the
617    // others outright, so their senders are only refunded at contract expiry.
618    if gw_send.gatewayd_version < *VERSION_0_12_2_ALPHA {
619        return assert_duplicates_refunded(&client_a, &client_b, &client_c, gw_send, &invoice)
620            .await;
621    }
622
623    assert_duplicates_refused(
624        [&client_a, &client_b, &client_c],
625        gw_send,
626        [
627            first_federation.calculate_federation_id(),
628            second_federation_id,
629        ],
630        &invoice,
631    )
632    .await
633}
634
635/// How long a send refused by the gateway is watched to stay unresolved.
636const REFUSED_SEND_WATCH: Duration = Duration::from_secs(30);
637
638/// The duplicate-payment assertions for gateways that claim the payment image
639/// only after paying and forfeit the duplicates: one settles, two are refunded.
640async fn assert_duplicates_refunded(
641    client_a: &Client,
642    client_b: &Client,
643    client_c: &Client,
644    gw_send: &Gatewayd,
645    invoice: &str,
646) -> anyhow::Result<()> {
647    // The two clients in the first federation race for the invoice; the
648    // per-federation dedup settles exactly one of them and refunds the other.
649    let (state_a, state_b) = try_join!(
650        common::send(client_a, &gw_send.addr, invoice),
651        common::send(client_b, &gw_send.addr, invoice),
652    )?;
653
654    // The second-federation client then pays the same, now-settled invoice. The
655    // gateway already holds the preimage, so without cross-federation dedup it
656    // claims this contract too, being reimbursed twice for one Lightning payment.
657    let state_c = common::send(client_c, &gw_send.addr, invoice).await?;
658
659    let states = [state_a, state_b, state_c];
660
661    info!("shared-invoice send states: {states:?}");
662
663    let successes = states
664        .iter()
665        .filter(|state| matches!(state, FinalSendOperationState::Success(_)))
666        .count();
667
668    let refunds = states
669        .iter()
670        .filter(|state| matches!(state, FinalSendOperationState::Refunded))
671        .count();
672
673    assert_eq!(
674        (successes, refunds),
675        (1, 2),
676        "exactly one payment must settle and the other two be refunded, got {states:?}"
677    );
678
679    Ok(())
680}
681
682/// The duplicate-payment assertions for gateways that claim the payment image
683/// at intake: one send settles, the gateway refuses the other two without
684/// paying or forfeiting anything, so they stay unresolved until their contracts
685/// expire, and the gateway is reimbursed exactly once.
686async fn assert_duplicates_refused(
687    [client_a, client_b, client_c]: [&Client; 3],
688    gw_send: &Gatewayd,
689    [first_federation_id, second_federation_id]: [String; 2],
690    invoice: &str,
691) -> anyhow::Result<()> {
692    let first_balance_before = gw_send
693        .client()
694        .ecash_balance(first_federation_id.clone())
695        .await?;
696    let second_balance_before = gw_send
697        .client()
698        .ecash_balance(second_federation_id.clone())
699        .await?;
700
701    // The two clients in the first federation race for the invoice. Whichever
702    // the gateway accepts first settles; the other is refused and stays funded.
703    let (op_a, op_b) = try_join!(
704        common::start_send(client_a, &gw_send.addr, invoice),
705        common::start_send(client_b, &gw_send.addr, invoice),
706    )?;
707
708    let (state_a, state_b) = tokio::join!(
709        await_send_within(client_a, op_a, REFUSED_SEND_WATCH),
710        await_send_within(client_b, op_b, REFUSED_SEND_WATCH),
711    );
712
713    // The second-federation client then pays the same, now-settled invoice. The
714    // claim is global to the gateway, so it is refused as well.
715    let op_c = common::start_send(client_c, &gw_send.addr, invoice).await?;
716    let state_c = await_send_within(client_c, op_c, REFUSED_SEND_WATCH).await;
717
718    let states = [state_a?, state_b?, state_c?];
719
720    info!("shared-invoice send states (None = unresolved): {states:?}");
721
722    let successes = states
723        .iter()
724        .filter(|state| matches!(state, Some(FinalSendOperationState::Success(_))))
725        .count();
726
727    let unresolved = states.iter().filter(|state| state.is_none()).count();
728
729    assert_eq!(
730        (successes, unresolved),
731        (1, 2),
732        "exactly one payment must settle and the other two stay unresolved, got {states:?}"
733    );
734
735    // A contract costs at least the invoice amount, so reimbursement for a
736    // second one would at least double the gain.
737    let invoice_msats = Bolt11Invoice::from_str(invoice)?
738        .amount_milli_satoshis()
739        .expect("the invoice has an amount");
740
741    let first_gain = gw_send
742        .client()
743        .ecash_balance(first_federation_id)
744        .await?
745        .checked_sub(first_balance_before)
746        .expect("the gateway's balance in the first federation must not shrink");
747
748    assert!(
749        (invoice_msats..2 * invoice_msats).contains(&first_gain),
750        "the gateway must be reimbursed for exactly one contract, gained {first_gain} msat"
751    );
752
753    assert_eq!(
754        gw_send.client().ecash_balance(second_federation_id).await?,
755        second_balance_before,
756        "the gateway must not claim the second federation's contract"
757    );
758
759    Ok(())
760}
761
762/// Waits up to `watch` for a send to reach its final state, returning `None`
763/// if it is still unresolved by then.
764async fn await_send_within(
765    client: &Client,
766    operation_id: OperationId,
767    watch: Duration,
768) -> anyhow::Result<Option<FinalSendOperationState>> {
769    match tokio::time::timeout(watch, common::await_send(client, operation_id)).await {
770        Ok(state) => state.map(Some),
771        Err(_) => Ok(None),
772    }
773}
774
775/// Stops enough guardians that no threshold of them can answer, checks that
776/// a receive is refused without the gateway funding anything, then checks
777/// that receives work again once the guardians are back.
778async fn test_federation_outage(
779    dev_fed: DevFed,
780    process_mgr: &ProcessManager,
781) -> anyhow::Result<()> {
782    info!("Testing that the gateway refuses to fund a receive during a federation outage...");
783
784    let DevFed {
785        mut fed,
786        gw_lnd,
787        gw_ldk,
788        ..
789    } = dev_fed;
790
791    if gw_lnd.gatewayd_version < *VERSION_0_12_2_ALPHA {
792        info!(
793            gatewayd_version = %gw_lnd.gatewayd_version,
794            "Skipping: gateway predates the federation liveness probe"
795        );
796        return Ok(());
797    }
798
799    let client = fed
800        .new_joined_client("lnv2-federation-outage-client")
801        .await?;
802    fed.await_all_peers().await?;
803
804    let federation_id = fed.calculate_federation_id();
805    let gateway = gw_lnd.client().address();
806
807    // Register the invoice while the federation is healthy so that only the
808    // funding decision is exercised below.
809    let (invoice, _) = common::receive(&client, &gateway, 100_000).await?;
810    let balance_before = gw_lnd.client().ecash_balance(federation_id.clone()).await?;
811
812    // Stop the smallest number of guardians that leaves fewer than a
813    // threshold of them running.
814    let fed_size = process_mgr.globals.FM_FED_SIZE;
815    let max_evil = (fed_size - 1) / 3;
816    let stopped: Vec<usize> = (fed_size - max_evil - 1..fed_size).collect();
817    for peer in &stopped {
818        fed.terminate_server(*peer).await?;
819    }
820
821    info!(
822        ?stopped,
823        "Paying invoice while the federation cannot reach consensus"
824    );
825    // The payment runs concurrently: a gateway that funds anyway holds the
826    // HTLC until the guardians are back, so awaiting it here would block the
827    // restart below.
828    let payment = fedimint_core::runtime::spawn("lnv2-outage-payment", {
829        let gw_ldk = gw_ldk.client();
830        async move { gw_ldk.pay_invoice(invoice).await }
831    });
832
833    // Bring the guardians back only once the gateway has acted on the HTLC:
834    // a refusal fails the payment, while funding consumes ecash at submission
835    // even though the transaction cannot land yet.
836    poll_simple("Waiting for the gateway to act on the HTLC", || async {
837        if payment.is_finished() {
838            return Ok(());
839        }
840        let balance = gw_lnd.client().ecash_balance(federation_id.clone()).await?;
841        if balance < balance_before {
842            return Ok(());
843        }
844        bail!("gateway has not acted on the HTLC yet")
845    })
846    .await?;
847
848    for peer in &stopped {
849        fed.start_server(process_mgr, *peer).await?;
850    }
851    fed.await_all_peers().await?;
852
853    // Funding queued during the outage would land now that the guardians are
854    // back, and the payment would complete against it. The probe exists so
855    // that it does not.
856    let outcome = payment.await?;
857    info!(?outcome, "Payment outcome once the guardians are back");
858    ensure!(
859        outcome.is_err(),
860        "payment must fail: the gateway funded an incoming contract during the outage"
861    );
862    ensure!(
863        gw_lnd.client().ecash_balance(federation_id.clone()).await? == balance_before,
864        "gateway must not fund an incoming contract while the federation cannot reach consensus"
865    );
866
867    // The gateway's own connections to the restarted guardians come back on
868    // their own schedule, and the probe refuses receives until they do, so
869    // retry with a fresh invoice each time; a refused invoice is cancelled.
870    info!("Paying a fresh invoice after the guardians are back");
871    poll_simple("Waiting for receives to succeed again", || async {
872        let (invoice, _) = common::receive(&client, &gateway, 100_000).await?;
873        gw_ldk.client().pay_invoice(invoice).await
874    })
875    .await?;
876    ensure!(
877        gw_lnd.client().ecash_balance(federation_id).await? < balance_before,
878        "gateway must fund the incoming contract once the federation is back"
879    );
880
881    info!("Federation outage test complete");
882    Ok(())
883}
884
885async fn test_fees(
886    fed_id: String,
887    client: &Client,
888    gw_lnd: &Gatewayd,
889    gw_ldk: &Gatewayd,
890    expected_addition: u64,
891) -> anyhow::Result<()> {
892    let gw_lnd_ecash_prev = gw_lnd.client().ecash_balance(fed_id.clone()).await?;
893
894    let (invoice, receive_op) = common::receive(client, &gw_ldk.addr, 1_000_000).await?;
895
896    let state = common::send(client, &gw_lnd.addr, &invoice.to_string()).await?;
897    assert!(matches!(state, FinalSendOperationState::Success(_)));
898
899    common::await_receive_claimed(client, receive_op).await?;
900
901    // The sending gateway claims its outgoing contract in the background after
902    // returning the preimage to the sender, so its ecash balance is credited
903    // asynchronously. Wait for the claim to settle before asserting the fee.
904    while almost_equal(
905        gw_lnd_ecash_prev + expected_addition,
906        gw_lnd.client().ecash_balance(fed_id.clone()).await?,
907        5000,
908    )
909    .is_err()
910    {
911        info!("Waiting for the sending gateway's outgoing claim to settle...");
912        cmd!(client, "dev", "wait", "1").out_json().await?;
913    }
914
915    Ok(())
916}
917
918async fn add_gateway(client: &Client, peer: usize, gateway: &String) -> anyhow::Result<bool> {
919    cmd!(
920        client,
921        "--our-id",
922        peer.to_string(),
923        "--password",
924        "pass",
925        "module",
926        "lnv2",
927        "gateways",
928        "add",
929        gateway
930    )
931    .out_json()
932    .await?
933    .as_bool()
934    .ok_or(anyhow::anyhow!("JSON Value is not a boolean"))
935}
936
937async fn remove_gateway(client: &Client, peer: usize, gateway: &String) -> anyhow::Result<bool> {
938    cmd!(
939        client,
940        "--our-id",
941        peer.to_string(),
942        "--password",
943        "pass",
944        "module",
945        "lnv2",
946        "gateways",
947        "remove",
948        gateway
949    )
950    .out_json()
951    .await?
952    .as_bool()
953    .ok_or(anyhow::anyhow!("JSON Value is not a boolean"))
954}
955
956/// Remove all currently registered gateways, including the ones devimint
957/// adds on startup, from the given peers.
958async fn remove_all_gateways(client: &Client, peers: &[usize]) -> anyhow::Result<()> {
959    let gateways = cmd!(client, "module", "lnv2", "gateways", "list")
960        .out_json()
961        .await?
962        .as_array()
963        .expect("JSON Value is not an array")
964        .iter()
965        .map(|gateway| {
966            gateway
967                .as_str()
968                .expect("JSON Value is not a string")
969                .to_string()
970        })
971        .collect::<Vec<String>>();
972
973    for gateway in &gateways {
974        for &peer in peers {
975            assert!(remove_gateway(client, peer, gateway).await?);
976        }
977    }
978
979    Ok(())
980}
981
982// Keep this below the 310s `fm-run-test` timeout so LNURL flakes fail with
983// useful balance diagnostics instead of a generic test timeout.
984const LNURL_BALANCE_WAIT_ATTEMPTS: u64 = 120;
985
986async fn wait_for_lnurl_balance(
987    client: &Client,
988    client_name: &str,
989    expected_msats: u64,
990) -> anyhow::Result<()> {
991    let mut last_balance_msats = client.balance().await?;
992
993    for attempt in 1..=LNURL_BALANCE_WAIT_ATTEMPTS {
994        if last_balance_msats < expected_msats {
995            info!(
996                client_name,
997                balance_msats = last_balance_msats,
998                expected_msats,
999                attempt,
1000                "Waiting for {client_name} to receive funds via LNURL"
1001            );
1002            cmd!(client, "dev", "wait", "1").out_json().await?;
1003            last_balance_msats = client.balance().await?;
1004        } else {
1005            info!(
1006                client_name,
1007                balance_msats = last_balance_msats,
1008                expected_msats,
1009                attempt,
1010                "{client_name} successfully received funds via LNURL"
1011            );
1012            return Ok(());
1013        }
1014    }
1015
1016    if last_balance_msats < expected_msats {
1017        anyhow::bail!(
1018            "timed out waiting for {client_name} to receive funds via LNURL after {} attempts; last balance: {last_balance_msats} msats; expected balance: {expected_msats} msats",
1019            LNURL_BALANCE_WAIT_ATTEMPTS
1020        );
1021    }
1022
1023    info!(
1024        client_name,
1025        balance_msats = last_balance_msats,
1026        expected_msats,
1027        attempt = LNURL_BALANCE_WAIT_ATTEMPTS,
1028        "{client_name} successfully received funds via LNURL"
1029    );
1030
1031    Ok(())
1032}
1033
1034async fn test_lnurl_pay(dev_fed: &DevJitFed) -> anyhow::Result<()> {
1035    if util::FedimintCli::version_or_default().await < *VERSION_0_11_0_ALPHA {
1036        return Ok(());
1037    }
1038
1039    if util::FedimintdCmd::version_or_default().await < *VERSION_0_11_0_ALPHA {
1040        return Ok(());
1041    }
1042
1043    if util::Gatewayd::version_or_default().await < *VERSION_0_11_0_ALPHA {
1044        return Ok(());
1045    }
1046
1047    let federation = dev_fed.fed().await?;
1048
1049    let gw_lnd = dev_fed.gw_lnd().await?;
1050    let gw_ldk = dev_fed.gw_ldk().await?;
1051
1052    let gateway_pairs = [(gw_lnd, gw_ldk), (gw_ldk, gw_lnd)];
1053
1054    let recurringd = dev_fed.recurringdv2().await?.api_url().to_string();
1055
1056    let client_a = federation
1057        .new_joined_client("lnv2-lnurl-test-client-a")
1058        .await?;
1059
1060    assert_module_sanity(&client_a).await?;
1061
1062    let client_b = federation
1063        .new_joined_client("lnv2-lnurl-test-client-b")
1064        .await?;
1065
1066    assert_module_sanity(&client_b).await?;
1067
1068    for (gw_send, gw_receive) in gateway_pairs {
1069        info!(
1070            "Testing lnurl payments: {} -> {} -> client",
1071            gw_send.ln.ln_type(),
1072            gw_receive.ln.ln_type()
1073        );
1074
1075        let lnurl_a = generate_lnurl(&client_a, &recurringd, &gw_receive.addr).await?;
1076        let lnurl_b = generate_lnurl(&client_b, &recurringd, &gw_receive.addr).await?;
1077
1078        let (invoice_a, verify_url_a) = fetch_invoice(lnurl_a.clone(), 500_000).await?;
1079        let (invoice_b, verify_url_b) = fetch_invoice(lnurl_b.clone(), 500_000).await?;
1080
1081        let verify_task_a = task::spawn("verify_task_a", verify_payment_wait(verify_url_a.clone()));
1082        let verify_task_b = task::spawn("verify_task_b", verify_payment_wait(verify_url_b.clone()));
1083
1084        let response_a = verify_payment(&verify_url_a).await?;
1085        let response_b = verify_payment(&verify_url_b).await?;
1086
1087        assert!(!response_a.settled);
1088        assert!(!response_b.settled);
1089
1090        assert!(response_a.preimage.is_none());
1091        assert!(response_b.preimage.is_none());
1092
1093        gw_send.client().pay_invoice(invoice_a.clone()).await?;
1094        gw_send.client().pay_invoice(invoice_b.clone()).await?;
1095
1096        let response_a = verify_payment(&verify_url_a).await?;
1097        let response_b = verify_payment(&verify_url_b).await?;
1098
1099        assert!(response_a.settled);
1100        assert!(response_b.settled);
1101
1102        verify_preimage(&response_a, &invoice_a);
1103        verify_preimage(&response_b, &invoice_b);
1104
1105        assert_eq!(verify_task_a.await??, response_a);
1106        assert_eq!(verify_task_b.await??, response_b);
1107    }
1108
1109    wait_for_lnurl_balance(&client_a, "client A", 950 * 1000).await?;
1110    wait_for_lnurl_balance(&client_b, "client B", 950 * 1000).await?;
1111
1112    Ok(())
1113}
1114
1115/// Tests LNURL receives after recovery from seed.
1116///
1117/// LNv2 uses `NoModuleBackup`, so recovery restores funds via the mint module
1118/// but not the operation log or LNv2 state. Verifies:
1119/// 1. Balance is fully recovered.
1120/// 2. Payments to a pre-recovery LNURL are still claimed by the restored
1121///    client.
1122async fn test_lnurl_recovery(dev_fed: &DevJitFed) -> anyhow::Result<()> {
1123    // Before v0.10.0 the CLI registered LNURLs via a POST to the recurringd
1124    // server.  That endpoint no longer exists, so old CLIs cannot generate
1125    // LNURLs against the current recurringdv2.
1126    if util::FedimintCli::version_or_default().await < *VERSION_0_10_0_ALPHA {
1127        return Ok(());
1128    }
1129
1130    let federation = dev_fed.fed().await?;
1131    let gw_lnd = dev_fed.gw_lnd().await?;
1132    let gw_ldk = dev_fed.gw_ldk().await?;
1133    let recurringd = dev_fed.recurringdv2().await?.api_url().to_string();
1134
1135    const LNURL_AMOUNT_MSAT: u64 = 500_000;
1136    const LNURL_BALANCE_TOLERANCE_MSAT: u64 = 100_000;
1137
1138    // ── Phase 1: Pre-recovery LNURL receives ──────────────────────────
1139
1140    info!("Phase 1: Creating client and receiving via LNURL before recovery");
1141
1142    let client = federation
1143        .new_joined_client("lnv2-lnurl-recovery-original")
1144        .await?;
1145
1146    let lnurl = generate_lnurl(&client, &recurringd, &gw_ldk.addr).await?;
1147
1148    for i in 0..3 {
1149        info!("Paying LNURL invoice {}/3", i + 1);
1150        let (invoice, _verify_url) = fetch_invoice(lnurl.clone(), LNURL_AMOUNT_MSAT).await?;
1151        gw_lnd.client().pay_invoice(invoice).await?;
1152    }
1153
1154    while almost_equal(
1155        client.balance().await?,
1156        3 * LNURL_AMOUNT_MSAT,
1157        LNURL_BALANCE_TOLERANCE_MSAT,
1158    )
1159    .is_err()
1160    {
1161        info!("Waiting for pre-recovery LNURL payments to settle...");
1162        cmd!(client, "dev", "wait", "1").out_json().await?;
1163    }
1164
1165    let pre_recovery_balance = client.balance().await?;
1166    info!("Pre-recovery balance: {pre_recovery_balance} msats");
1167
1168    let mnemonic = cmd!(client, "print-secret").out_json().await?["secret"]
1169        .as_str()
1170        .expect("secret is a string")
1171        .to_owned();
1172
1173    // ── Phase 2: Recovery ─────────────────────────────────────────────
1174
1175    info!("Phase 2: Recovering client from seed");
1176
1177    let restored = Client::create("lnv2-lnurl-recovery-restored").await?;
1178    cmd!(
1179        restored,
1180        "restore",
1181        "--invite-code",
1182        federation.invite_code()?,
1183        "--mnemonic",
1184        &mnemonic
1185    )
1186    .run()
1187    .await?;
1188
1189    while restored.balance().await? < pre_recovery_balance {
1190        info!("Waiting for recovery to complete...");
1191        cmd!(restored, "dev", "wait", "1").out_json().await?;
1192    }
1193
1194    let post_recovery_balance = restored.balance().await?;
1195    info!("Post-recovery balance: {post_recovery_balance} msats");
1196
1197    // ── Phase 3: Post-recovery LNURL receives ─────────────────────────
1198
1199    info!("Phase 3: Paying to the original LNURL; restored client must claim");
1200
1201    // Reuse the Phase 1 `lnurl` on purpose: this is what a third party would
1202    // have saved pre-recovery, and it must still pay the restored client.
1203    for i in 0..2 {
1204        info!("Paying pre-recovery LNURL invoice {}/2", i + 1);
1205        let (invoice, _verify_url) = fetch_invoice(lnurl.clone(), LNURL_AMOUNT_MSAT).await?;
1206        gw_lnd.client().pay_invoice(invoice).await?;
1207    }
1208
1209    while almost_equal(
1210        restored.balance().await?,
1211        post_recovery_balance + 2 * LNURL_AMOUNT_MSAT,
1212        LNURL_BALANCE_TOLERANCE_MSAT,
1213    )
1214    .is_err()
1215    {
1216        info!("Waiting for post-recovery LNURL payments to settle...");
1217        cmd!(restored, "dev", "wait", "1").out_json().await?;
1218    }
1219
1220    let final_balance = restored.balance().await?;
1221    info!("Final balance: {final_balance} msats");
1222
1223    let operations = cmd!(restored, "list-operations", "--limit", "100")
1224        .out_json()
1225        .await?;
1226    let lnv2_ops: Vec<_> = operations["operations"]
1227        .as_array()
1228        .expect("operations is an array")
1229        .iter()
1230        .filter(|op| op["operation_kind"].as_str() == Some("lnv2"))
1231        .collect();
1232    assert!(
1233        lnv2_ops.len() >= 2,
1234        "Expected at least 2 LNv2 operations after post-recovery receives, found {}",
1235        lnv2_ops.len()
1236    );
1237
1238    info!(
1239        "LNURL recovery test passed: {} new operations, balance {final_balance} msats",
1240        lnv2_ops.len()
1241    );
1242
1243    Ok(())
1244}
1245
1246async fn generate_lnurl(
1247    client: &Client,
1248    recurringd_base_url: &str,
1249    gw_ldk_addr: &str,
1250) -> anyhow::Result<String> {
1251    cmd!(
1252        client,
1253        "module",
1254        "lnv2",
1255        "lnurl",
1256        "generate",
1257        recurringd_base_url,
1258        "--gateway",
1259        gw_ldk_addr,
1260    )
1261    .out_json()
1262    .await
1263    .map(|s| s.as_str().unwrap().to_owned())
1264}
1265
1266fn verify_preimage(response: &VerifyResponse, invoice: &Bolt11Invoice) {
1267    let preimage = response.preimage.expect("Payment should be settled");
1268
1269    let payment_hash = preimage.consensus_hash::<sha256::Hash>();
1270
1271    assert_eq!(payment_hash, *invoice.payment_hash());
1272}
1273
1274async fn verify_payment(verify_url: &str) -> anyhow::Result<VerifyResponse> {
1275    reqwest::get(verify_url)
1276        .await?
1277        .json::<LnurlResponse<VerifyResponse>>()
1278        .await?
1279        .into_result()
1280        .map_err(anyhow::Error::msg)
1281}
1282
1283async fn verify_payment_wait(verify_url: String) -> anyhow::Result<VerifyResponse> {
1284    reqwest::get(format!("{verify_url}?wait"))
1285        .await?
1286        .json::<LnurlResponse<VerifyResponse>>()
1287        .await?
1288        .into_result()
1289        .map_err(anyhow::Error::msg)
1290}
1291
1292#[derive(Deserialize, Clone)]
1293struct LnUrlPayResponse {
1294    callback: String,
1295    metadata: String,
1296}
1297
1298#[derive(Deserialize, Clone)]
1299struct LnUrlPayInvoiceResponse {
1300    pr: Bolt11Invoice,
1301    // LUD-06 requires this field, so parsing fails if the service omits it
1302    routes: Vec<serde_json::Value>,
1303    verify: String,
1304}
1305
1306async fn fetch_invoice(lnurl: String, amount_msat: u64) -> anyhow::Result<(Bolt11Invoice, String)> {
1307    let url = parse_lnurl(&lnurl).ok_or_else(|| anyhow::anyhow!("Invalid LNURL"))?;
1308
1309    let response = reqwest::get(url).await?.json::<LnUrlPayResponse>().await?;
1310
1311    let callback_url = format!("{}?amount={}", response.callback, amount_msat);
1312
1313    let invoice_response = reqwest::get(callback_url)
1314        .await?
1315        .json::<LnUrlPayInvoiceResponse>()
1316        .await?;
1317
1318    ensure!(
1319        invoice_response.pr.amount_milli_satoshis() == Some(amount_msat),
1320        "Invoice amount is not set"
1321    );
1322
1323    ensure!(
1324        invoice_response.routes.is_empty(),
1325        "LUD-06 requires routes to be an empty array"
1326    );
1327
1328    let metadata_hash = sha256::Hash::hash(response.metadata.as_bytes());
1329
1330    ensure!(
1331        matches!(
1332            invoice_response.pr.description(),
1333            Bolt11InvoiceDescriptionRef::Hash(hash) if hash.0 == metadata_hash
1334        ),
1335        "Invoice does not commit to the LNURL metadata via its description hash (LUD-06)"
1336    );
1337
1338    Ok((invoice_response.pr, invoice_response.verify))
1339}
1340
1341async fn test_iroh_payment(
1342    client: &Client,
1343    gw_lnd: &Gatewayd,
1344    gw_ldk: &Gatewayd,
1345    online_peers: &[usize],
1346) -> anyhow::Result<()> {
1347    info!("Testing iroh payment...");
1348    // The send below relies on automatic gateway selection picking the iroh
1349    // gateway, so the gateways devimint added on startup have to go first.
1350    remove_all_gateways(client, online_peers).await?;
1351    for &peer in online_peers {
1352        add_gateway(client, peer, &format!("iroh://{}", gw_lnd.node_id)).await?;
1353    }
1354
1355    // If the client is below v0.10.0, also add the HTTP address so that the client
1356    // can fallback to using that, since the iroh gateway will fail.
1357    if util::FedimintCli::version_or_default().await < *VERSION_0_10_0_ALPHA
1358        || gw_lnd.gatewayd_version < *VERSION_0_10_0_ALPHA
1359    {
1360        for &peer in online_peers {
1361            add_gateway(client, peer, &gw_lnd.addr).await?;
1362        }
1363    }
1364
1365    let invoice = gw_ldk.client().create_invoice(5_000_000).await?;
1366
1367    let send_op = serde_json::from_value::<OperationId>(
1368        cmd!(client, "module", "lnv2", "send", invoice,)
1369            .out_json()
1370            .await?,
1371    )?;
1372
1373    let send_state = common::await_send(client, send_op).await?;
1374    assert!(
1375        matches!(send_state, FinalSendOperationState::Success(_)),
1376        "unexpected send state: {send_state:?}"
1377    );
1378
1379    let (invoice, receive_op) = serde_json::from_value::<(Bolt11Invoice, OperationId)>(
1380        cmd!(client, "module", "lnv2", "receive", "5000000",)
1381            .out_json()
1382            .await?,
1383    )?;
1384
1385    gw_ldk.client().pay_invoice(invoice).await?;
1386    common::await_receive_claimed(client, receive_op).await?;
1387
1388    if util::FedimintCli::version_or_default().await < *VERSION_0_10_0_ALPHA
1389        || gw_lnd.gatewayd_version < *VERSION_0_10_0_ALPHA
1390    {
1391        for &peer in online_peers {
1392            remove_gateway(client, peer, &gw_lnd.addr).await?;
1393        }
1394    }
1395
1396    for &peer in online_peers {
1397        remove_gateway(client, peer, &format!("iroh://{}", gw_lnd.node_id)).await?;
1398    }
1399
1400    Ok(())
1401}
1402
1403/// Exercise the direct HTLC CLI between two clients of the same federation,
1404/// with the full out-of-band handoff as strings: the claimer's key, the
1405/// funder's funding txid and contract JSON, and the forfeit signature. The
1406/// claimer's secret key and preimage are passed through files and stdin.
1407async fn test_direct_htlc(dev_fed: &DevJitFed) -> anyhow::Result<()> {
1408    if util::FedimintCli::version_or_default().await < *VERSION_0_14_0_ALPHA {
1409        info!("fedimint-cli does not support direct HTLCs, skipping");
1410        return Ok(());
1411    }
1412
1413    let federation = dev_fed.fed().await?;
1414
1415    let funder = federation
1416        .new_joined_client("lnv2-test-direct-htlc-funder")
1417        .await?;
1418
1419    let claimer = federation
1420        .new_joined_client("lnv2-test-direct-htlc-claimer")
1421        .await?;
1422
1423    assert_module_sanity(&funder).await?;
1424
1425    federation.pegin_client(10_000, &funder).await?;
1426
1427    let secrets_dir = PathBuf::from(std::env::var(FM_CLIENT_DIR_ENV)?).join("direct-htlc");
1428
1429    tokio::fs::create_dir_all(&secrets_dir).await?;
1430
1431    let claim_sk_file = secrets_dir.join("claim-sk");
1432    let preimage_file = secrets_dir.join("preimage");
1433
1434    info!("Generating the claim keypair...");
1435
1436    let claim_keypair = cmd!(
1437        claimer,
1438        "module",
1439        "lnv2",
1440        "htlc",
1441        "new-claim-keypair",
1442        "--secret-key-file",
1443        claim_sk_file.display()
1444    )
1445    .out_json()
1446    .await?;
1447
1448    ensure!(
1449        claim_keypair.get("secret_key").is_none(),
1450        "The claim secret key must not be printed"
1451    );
1452
1453    let claim_pk = json_string(&claim_keypair["public_key"])?;
1454
1455    {
1456        use std::os::unix::fs::PermissionsExt as _;
1457
1458        let mode = tokio::fs::metadata(&claim_sk_file)
1459            .await?
1460            .permissions()
1461            .mode();
1462
1463        ensure!(mode & 0o777 == 0o600, "Secret key file has mode {mode:o}");
1464    }
1465
1466    ensure!(
1467        cmd!(
1468            claimer,
1469            "module",
1470            "lnv2",
1471            "htlc",
1472            "new-claim-keypair",
1473            "--secret-key-file",
1474            claim_sk_file.display()
1475        )
1476        .run()
1477        .await
1478        .is_err(),
1479        "An existing secret key file must not be overwritten"
1480    );
1481
1482    info!("Testing claim of a point-locked HTLC...");
1483
1484    // The preimage of a point lock is the secret key of the payment point.
1485    let preimage = SecretKey::new(&mut OsRng);
1486    let preimage_hex = preimage.display_secret().to_string();
1487    let payment_point = preimage.public_key(SECP256K1).to_string();
1488
1489    write_overwrite_async(&preimage_file, &preimage_hex).await?;
1490
1491    let (funding_txid, contract) =
1492        create_htlc(&funder, &claim_pk, 100, "--payment-point", &payment_point).await?;
1493
1494    let remaining_blocks = cmd!(
1495        claimer,
1496        "module",
1497        "lnv2",
1498        "htlc",
1499        "await-funded",
1500        funding_txid,
1501        contract
1502    )
1503    .out_json()
1504    .await?;
1505
1506    ensure!(
1507        remaining_blocks.as_u64().is_some_and(|blocks| blocks > 0),
1508        "Expected the remaining blocks until expiration, got {remaining_blocks}"
1509    );
1510
1511    let mut claim = cmd!(
1512        claimer,
1513        "module",
1514        "lnv2",
1515        "htlc",
1516        "claim",
1517        funding_txid,
1518        contract,
1519        "--claim-sk-file",
1520        claim_sk_file.display(),
1521        "--preimage-file",
1522        "-"
1523    );
1524
1525    claim.cmd.stdin(std::fs::File::open(&preimage_file)?);
1526
1527    let claim_operation = claim.out_json().await?;
1528
1529    await_htlc_settled(&claimer, &claim_operation).await?;
1530
1531    ensure!(claimer.balance().await? > 0);
1532
1533    // The funder learns the preimage from the federation.
1534    assert_eq!(
1535        cmd!(
1536            funder,
1537            "module",
1538            "lnv2",
1539            "htlc",
1540            "await-resolution",
1541            funding_txid,
1542            contract
1543        )
1544        .out_json()
1545        .await?,
1546        preimage_hex
1547    );
1548
1549    info!("Testing cooperative cancellation of a hash-locked HTLC...");
1550
1551    let payment_hash = [42_u8; 32].consensus_hash::<sha256::Hash>().to_string();
1552
1553    let (funding_txid, contract) =
1554        create_htlc(&funder, &claim_pk, 100, "--payment-hash", &payment_hash).await?;
1555
1556    let balance_before_cancel = funder.balance().await?;
1557
1558    let mut forfeit = cmd!(
1559        claimer,
1560        "module",
1561        "lnv2",
1562        "htlc",
1563        "forfeit",
1564        contract,
1565        "--claim-sk-file",
1566        "-"
1567    );
1568
1569    forfeit.cmd.stdin(std::fs::File::open(&claim_sk_file)?);
1570
1571    let forfeit_signature = json_string(&forfeit.out_json().await?)?;
1572
1573    let cancel_operation = cmd!(
1574        funder,
1575        "module",
1576        "lnv2",
1577        "htlc",
1578        "cancel",
1579        funding_txid,
1580        contract,
1581        forfeit_signature
1582    )
1583    .out_json()
1584    .await?;
1585
1586    await_htlc_settled(&funder, &cancel_operation).await?;
1587
1588    ensure!(funder.balance().await? > balance_before_cancel);
1589
1590    info!("Testing refund of an expired HTLC...");
1591
1592    let (funding_txid, contract) =
1593        create_htlc(&funder, &claim_pk, 5, "--payment-hash", &payment_hash).await?;
1594
1595    let balance_before_refund = funder.balance().await?;
1596
1597    dev_fed.bitcoind().await?.mine_blocks(10).await?;
1598
1599    // The funder is notified that the contract expired unclaimed...
1600    assert_eq!(
1601        cmd!(
1602            funder,
1603            "module",
1604            "lnv2",
1605            "htlc",
1606            "await-resolution",
1607            funding_txid,
1608            contract
1609        )
1610        .out_json()
1611        .await?,
1612        Value::Null
1613    );
1614
1615    // ...and can refund it once the consensus block count has caught up.
1616    let refund_operation = retry(
1617        "Refunding the expired HTLC",
1618        backoff_util::aggressive_backoff_long(),
1619        || async {
1620            cmd!(
1621                funder,
1622                "module",
1623                "lnv2",
1624                "htlc",
1625                "refund",
1626                funding_txid,
1627                contract
1628            )
1629            .out_json()
1630            .await
1631        },
1632    )
1633    .await?;
1634
1635    await_htlc_settled(&funder, &refund_operation).await?;
1636
1637    ensure!(funder.balance().await? > balance_before_refund);
1638
1639    Ok(())
1640}
1641
1642/// Fund an HTLC of one thousand sats locked to `claim_pk`, wait for the
1643/// funding transaction to be accepted and return the funding txid and the
1644/// contract JSON as the claimer receives them out of band.
1645async fn create_htlc(
1646    funder: &Client,
1647    claim_pk: &str,
1648    expiration_delta: u64,
1649    lock_flag: &str,
1650    lock_value: &str,
1651) -> anyhow::Result<(String, String)> {
1652    let created = cmd!(
1653        funder,
1654        "module",
1655        "lnv2",
1656        "htlc",
1657        "create",
1658        "1000000",
1659        claim_pk,
1660        expiration_delta,
1661        lock_flag,
1662        lock_value
1663    )
1664    .out_json()
1665    .await?;
1666
1667    // The contract is always the first output of its funding transaction.
1668    assert_eq!(created["outpoint"]["out_idx"], 0);
1669
1670    await_htlc_settled(funder, &created["operation_id"]).await?;
1671
1672    Ok((
1673        json_string(&created["outpoint"]["txid"])?,
1674        created["contract"].to_string(),
1675    ))
1676}
1677
1678async fn await_htlc_settled(client: &Client, operation_id: &Value) -> anyhow::Result<()> {
1679    assert_eq!(
1680        cmd!(
1681            client,
1682            "module",
1683            "lnv2",
1684            "htlc",
1685            "await-settled",
1686            json_string(operation_id)?
1687        )
1688        .out_json()
1689        .await?,
1690        "settled"
1691    );
1692
1693    Ok(())
1694}
1695
1696fn json_string(value: &Value) -> anyhow::Result<String> {
1697    value
1698        .as_str()
1699        .map(str::to_owned)
1700        .with_context(|| format!("Expected a JSON string, got {value}"))
1701}
1702
1703/// The balance of every account, so a step can show it moved only the
1704/// accounts it was meant to.
1705async fn account_balances(client: &Client) -> anyhow::Result<BTreeMap<Account, u64>> {
1706    let mut balances = BTreeMap::new();
1707
1708    for account in Account::ALL {
1709        balances.insert(account, client.balance_for(account).await?);
1710    }
1711
1712    Ok(balances)
1713}
1714
1715/// The fees of the steps below are not quoted up front, so a balance that
1716/// moved is checked against its expected value with this much slack. The
1717/// balances that must not move are checked exactly.
1718const FEE_TOLERANCE_MSAT: u64 = 500_000;
1719
1720fn ensure_close(actual: u64, expected: u64, what: &str) -> anyhow::Result<()> {
1721    almost_equal(actual, expected, FEE_TOLERANCE_MSAT)
1722        .map_err(|error| anyhow::anyhow!("{what}: {error}"))
1723}
1724
1725/// Every account outside `touched` holds exactly what it held before.
1726fn ensure_untouched(
1727    before: &BTreeMap<Account, u64>,
1728    after: &BTreeMap<Account, u64>,
1729    touched: &[Account],
1730) -> anyhow::Result<()> {
1731    for account in Account::ALL {
1732        if touched.contains(&account) {
1733            continue;
1734        }
1735
1736        ensure!(
1737            before[&account] == after[&account],
1738            "{account} moved from {} to {} msats without being involved",
1739            before[&account],
1740            after[&account]
1741        );
1742    }
1743
1744    Ok(())
1745}
1746
1747async fn lnv2_receive_into(
1748    client: &Client,
1749    gateway: &str,
1750    amount: u64,
1751    account: Account,
1752) -> anyhow::Result<(Bolt11Invoice, OperationId)> {
1753    Ok(serde_json::from_value(
1754        cmd!(
1755            client,
1756            "module",
1757            "lnv2",
1758            "receive",
1759            amount,
1760            "--gateway",
1761            gateway,
1762            "--account",
1763            account.to_string()
1764        )
1765        .out_json()
1766        .await?,
1767    )?)
1768}
1769
1770async fn lnv2_send_from(
1771    client: &Client,
1772    gateway: &str,
1773    invoice: &str,
1774    account: Account,
1775) -> anyhow::Result<FinalSendOperationState> {
1776    let send_op = serde_json::from_value::<OperationId>(
1777        cmd!(
1778            client,
1779            "module",
1780            "lnv2",
1781            "send",
1782            invoice,
1783            "--gateway",
1784            gateway,
1785            "--account",
1786            account.to_string()
1787        )
1788        .out_json()
1789        .await?,
1790    )?;
1791
1792    common::await_send(client, send_op).await
1793}
1794
1795/// Every module runs against the real mintv2 here, so money bound to a
1796/// non-primary account has to stay in it across a Lightning refund and claim,
1797/// an on-chain deposit and withdrawal, and a recovery from seed. The gateways
1798/// must not be funded when this starts: the refund relies on it.
1799async fn test_accounts(dev_fed: &DevJitFed) -> anyhow::Result<()> {
1800    if util::FedimintCli::version_or_default().await < *VERSION_0_14_0_ALPHA {
1801        info!("fedimint-cli predates accounts, skipping");
1802        return Ok(());
1803    }
1804
1805    let federation = dev_fed.fed().await?;
1806    let bitcoind = dev_fed.bitcoind().await?;
1807    let gw_lnd = dev_fed.gw_lnd().await?;
1808    let gw_ldk = dev_fed.gw_ldk().await?;
1809
1810    let client = federation
1811        .new_joined_client("lnv2-test-accounts-client")
1812        .await?;
1813
1814    if !module_is_present(&client, "mintv2").await?
1815        || !module_is_present(&client, "walletv2").await?
1816    {
1817        info!("mintv2 and walletv2 are not both enabled, skipping");
1818        return Ok(());
1819    }
1820
1821    const SECONDARY_FUNDING_MSAT: u64 = 10_000_000;
1822    const LIGHTNING_AMOUNT_MSAT: u64 = 1_000_000;
1823    const DEPOSIT_SAT: u64 = 50_000;
1824    const WITHDRAW_SAT: u64 = 20_000;
1825
1826    federation.pegin_client(20_000, &client).await?;
1827
1828    info!("Moving funds from primary into secondary...");
1829
1830    let before = account_balances(&client).await?;
1831
1832    let ecash = cmd!(client, "module", "mintv2", "send", SECONDARY_FUNDING_MSAT)
1833        .out_json()
1834        .await?;
1835
1836    let ecash = ecash.as_str().context("ecash is a string")?;
1837
1838    let state = cmd!(
1839        client,
1840        "module",
1841        "mintv2",
1842        "receive",
1843        ecash,
1844        "--account",
1845        "secondary"
1846    )
1847    .out_json()
1848    .await?;
1849
1850    ensure!(state == "Success", "receive failed: {state}");
1851
1852    cmd!(client, "dev", "wait-complete").run().await?;
1853
1854    let after = account_balances(&client).await?;
1855
1856    ensure_untouched(&before, &after, &[Account::Primary, Account::Secondary])?;
1857    ensure_close(
1858        after[&Account::Primary] + SECONDARY_FUNDING_MSAT,
1859        before[&Account::Primary],
1860        "primary did not pay for the funding of secondary",
1861    )?;
1862    ensure_close(
1863        after[&Account::Secondary],
1864        SECONDARY_FUNDING_MSAT,
1865        "secondary did not receive its funding",
1866    )?;
1867
1868    info!("Testing a refunded payment lands back in the paying account...");
1869
1870    let before = after;
1871
1872    // The invoice belongs to a throwaway client: its receive never settles
1873    // and would otherwise keep `wait-complete` on the client below from
1874    // returning until the invoice expires.
1875    let sink = federation
1876        .new_joined_client("lnv2-test-accounts-sink")
1877        .await?;
1878
1879    let invoice = common::receive(&sink, &gw_ldk.addr, LIGHTNING_AMOUNT_MSAT)
1880        .await?
1881        .0;
1882
1883    let state = lnv2_send_from(
1884        &client,
1885        &gw_lnd.addr,
1886        &invoice.to_string(),
1887        Account::Secondary,
1888    )
1889    .await?;
1890
1891    ensure!(
1892        matches!(state, FinalSendOperationState::Refunded),
1893        "payment through an unfunded gateway must be refunded, got {state:?}"
1894    );
1895
1896    cmd!(client, "dev", "wait-complete").run().await?;
1897
1898    let after = account_balances(&client).await?;
1899
1900    ensure_untouched(&before, &after, &[Account::Secondary])?;
1901    ensure_close(
1902        after[&Account::Secondary],
1903        before[&Account::Secondary],
1904        "the refund did not return to secondary",
1905    )?;
1906
1907    pegin_gateways(dev_fed).await?;
1908
1909    info!("Testing a payment between two accounts of one client...");
1910
1911    let before = after;
1912
1913    let (invoice, receive_op) = lnv2_receive_into(
1914        &client,
1915        &gw_ldk.addr,
1916        LIGHTNING_AMOUNT_MSAT,
1917        Account::Tertiary,
1918    )
1919    .await?;
1920
1921    let state = lnv2_send_from(
1922        &client,
1923        &gw_lnd.addr,
1924        &invoice.to_string(),
1925        Account::Secondary,
1926    )
1927    .await?;
1928
1929    ensure!(
1930        matches!(state, FinalSendOperationState::Success(_)),
1931        "payment through a funded gateway must succeed, got {state:?}"
1932    );
1933
1934    common::await_receive_claimed(&client, receive_op).await?;
1935
1936    cmd!(client, "dev", "wait-complete").run().await?;
1937
1938    let after = account_balances(&client).await?;
1939
1940    ensure_untouched(&before, &after, &[Account::Secondary, Account::Tertiary])?;
1941    ensure_close(
1942        after[&Account::Secondary] + LIGHTNING_AMOUNT_MSAT,
1943        before[&Account::Secondary],
1944        "secondary did not pay the invoice",
1945    )?;
1946    ensure_close(
1947        after[&Account::Tertiary],
1948        LIGHTNING_AMOUNT_MSAT,
1949        "tertiary did not claim the invoice",
1950    )?;
1951
1952    info!("Testing an on-chain deposit into secondary...");
1953
1954    let before = after;
1955
1956    // The position is read before the address is handed out so the receive
1957    // cannot be missed.
1958    let position = cmd!(client, "dev", "next-event-log-id").out_json().await?;
1959
1960    let address = cmd!(
1961        client,
1962        "module",
1963        "walletv2",
1964        "receive",
1965        "--account",
1966        "secondary"
1967    )
1968    .out_json()
1969    .await?;
1970
1971    let address = address.as_str().context("address is a string")?;
1972
1973    bitcoind.send_to(address.to_owned(), DEPOSIT_SAT).await?;
1974    bitcoind.mine_blocks(21).await?;
1975
1976    let output = cmd!(
1977        client,
1978        "module",
1979        "walletv2",
1980        "await-receive",
1981        position.to_string(),
1982        "--account",
1983        "secondary"
1984    )
1985    .out_json()
1986    .await?;
1987
1988    ensure!(output[0] == "Success", "deposit was not claimed: {output}");
1989
1990    cmd!(client, "dev", "wait-complete").run().await?;
1991
1992    let after = account_balances(&client).await?;
1993
1994    ensure_untouched(&before, &after, &[Account::Secondary])?;
1995    ensure!(
1996        after[&Account::Secondary] > before[&Account::Secondary],
1997        "the deposit did not reach secondary"
1998    );
1999
2000    info!("Testing an on-chain withdrawal from secondary...");
2001
2002    let before = after;
2003
2004    let withdraw_address = bitcoind.get_new_address().await?;
2005
2006    let output = cmd!(
2007        client,
2008        "module",
2009        "walletv2",
2010        "send",
2011        withdraw_address.to_string(),
2012        format!("{WITHDRAW_SAT} sat"),
2013        "--account",
2014        "secondary"
2015    )
2016    .out_json()
2017    .await?;
2018
2019    let txid = output["Success"]
2020        .as_str()
2021        .with_context(|| format!("withdrawal did not succeed: {output}"))?;
2022
2023    bitcoind
2024        .poll_get_transaction(bitcoin::Txid::from_str(txid)?)
2025        .await?;
2026
2027    cmd!(client, "dev", "wait-complete").run().await?;
2028
2029    let after = account_balances(&client).await?;
2030
2031    ensure_untouched(&before, &after, &[Account::Secondary])?;
2032    ensure!(
2033        after[&Account::Secondary] + WITHDRAW_SAT * 1000 <= before[&Account::Secondary],
2034        "the withdrawal did not leave secondary"
2035    );
2036
2037    info!("Recovering every account from the seed...");
2038
2039    let expected = after;
2040
2041    ensure!(
2042        expected[&Account::Primary] > 0
2043            && expected[&Account::Secondary] > 0
2044            && expected[&Account::Tertiary] > 0,
2045        "the recovery needs three funded accounts to tell apart, got {expected:?}"
2046    );
2047
2048    let mnemonic = cmd!(client, "print-secret").out_json().await?["secret"]
2049        .as_str()
2050        .context("secret is a string")?
2051        .to_owned();
2052
2053    let restored = Client::create("lnv2-test-accounts-restored").await?;
2054
2055    cmd!(
2056        restored,
2057        "restore",
2058        "--invite-code",
2059        federation.invite_code()?,
2060        "--mnemonic",
2061        &mnemonic
2062    )
2063    .run()
2064    .await?;
2065
2066    // Wait on the total so notes restored into the wrong account still end
2067    // the wait and fail the per-account check below instead of hanging.
2068    let expected_total = expected.values().sum::<u64>();
2069
2070    while account_balances(&restored).await?.values().sum::<u64>() < expected_total {
2071        info!("Waiting for the recovery to complete...");
2072        cmd!(restored, "dev", "wait", "1").out_json().await?;
2073    }
2074
2075    let recovered = account_balances(&restored).await?;
2076
2077    ensure!(
2078        recovered == expected,
2079        "the recovered balances differ from the original ones: {recovered:?} vs {expected:?}"
2080    );
2081
2082    info!("Spending the recovered secondary balance...");
2083
2084    let ecash = cmd!(
2085        restored,
2086        "module",
2087        "mintv2",
2088        "send-max",
2089        "--account",
2090        "secondary"
2091    )
2092    .out_json()
2093    .await?;
2094
2095    let ecash = ecash
2096        .as_str()
2097        .context("send-max must hand out the recovered secondary notes")?;
2098
2099    let state = cmd!(restored, "module", "mintv2", "receive", ecash)
2100        .out_json()
2101        .await?;
2102
2103    ensure!(state == "Success", "receive failed: {state}");
2104
2105    cmd!(restored, "dev", "wait-complete").run().await?;
2106
2107    let after = account_balances(&restored).await?;
2108
2109    ensure_untouched(&expected, &after, &[Account::Primary, Account::Secondary])?;
2110    ensure!(
2111        after[&Account::Secondary] == 0,
2112        "send-max left {} msats in secondary",
2113        after[&Account::Secondary]
2114    );
2115    ensure_close(
2116        after[&Account::Primary],
2117        expected[&Account::Primary] + expected[&Account::Secondary],
2118        "primary did not receive the recovered secondary notes",
2119    )?;
2120
2121    info!("Accounts test passed");
2122
2123    Ok(())
2124}