fedimint_core/util/
backoff_util.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
use std::time::Duration;

pub use backon::{Backoff, FibonacciBackoff};
use backon::{BackoffBuilder, FibonacciBuilder};

use crate::envs::is_running_in_test_env;
/// Backoff strategy for background tasks.
///
/// Starts at 1s and increases to 60s, never giving up.
pub fn background_backoff() -> FibonacciBackoff {
    custom_backoff(Duration::from_secs(1), Duration::from_secs(60), None)
}

/// A backoff strategy for relatively quick foreground operations.
///
/// Starts at 200ms and increases to 5s. Will retry 10 times before giving up,
/// with a maximum total delay between 20.8s and 22.8s depending on jitter.
pub fn aggressive_backoff() -> FibonacciBackoff {
    // Not accounting for jitter, the delays are:
    // 0.2, 0.2, 0.4, 0.6, 1.0, 1.6, 2.6, 4.2, 5.0, 5.0.
    //
    // Jitter adds a random value between 0 and `min_delay` to each delay.
    // Total jitter is between 0 and (10 * 0.2) = 2.0.
    //
    // Maximum possible delay including jitter is 22.8 seconds.
    custom_backoff(Duration::from_millis(200), Duration::from_secs(5), Some(10))
}

pub fn aggressive_backoff_long() -> FibonacciBackoff {
    custom_backoff(Duration::from_millis(200), Duration::from_secs(5), Some(15))
}

#[cfg(test)]
pub fn immediate_backoff(max_retries_or: Option<usize>) -> FibonacciBackoff {
    custom_backoff(Duration::ZERO, Duration::ZERO, max_retries_or)
}

pub fn custom_backoff(
    min_delay: Duration,
    max_delay: Duration,
    max_retries_or: Option<usize>,
) -> FibonacciBackoff {
    FibonacciBuilder::default()
        .with_jitter()
        .with_min_delay(min_delay)
        .with_max_delay(max_delay)
        .with_max_times(max_retries_or.unwrap_or(usize::MAX))
        .build()
}

/// Retry every max 10s for up to one hour, with a more aggressive fibonacci
/// backoff in the beginning to reduce expected latency.
///
/// Starts at 200ms increasing to 10s. Retries 360 times before giving up, with
/// a maximum total delay between 3527.6s (58m 47.6s) and 3599.6s (59m 59.6s)
/// depending on jitter.
pub fn fibonacci_max_one_hour() -> FibonacciBackoff {
    // Not accounting for jitter, the delays are:
    // 0.2, 0.2, 0.4, 0.6, 1.0, 1.6, 2.6, 4.2, 6.8, 10.0...
    //
    // Jitter adds a random value between 0 and `min_delay` to each delay.
    // Total jitter is between 0 and (360 * 0.2) = 72.0.
    //
    // Maximum possible delay including jitter is 3599.6s seconds.
    custom_backoff(
        Duration::from_millis(200),
        Duration::from_secs(10),
        Some(360),
    )
}

pub fn api_networking_backoff() -> fedimint_core::util::backoff_util::FibonacciBackoff {
    fedimint_core::util::backoff_util::custom_backoff(
        Duration::from_millis(250),
        if is_running_in_test_env() {
            // In our testing env we often run under very high load, and with tight limits,
            // so we want to force the retries to be relatively fast
            Duration::from_secs(5)
        } else {
            Duration::from_secs(600)
        },
        None,
    )
}