headless_lms_models/library/credit_registration/
backoff.rs

1//! How long the pipeline waits before trying again. Generic scheduling math; which error codes are
2//! even retryable is [`super::classification`].
3
4use headless_lms_utils::backoff::{exponential_backoff_secs, window_expired};
5
6use crate::prelude::*;
7
8pub const SUBMIT_BASE_BACKOFF_SECS: i64 = 60;
9pub const SUBMIT_MAX_BACKOFF_SECS: i64 = 6 * 60 * 60;
10/// After this long in failure a row stops being retried and becomes a support case.
11pub const SUBMIT_MAX_RETRY_AGE_SECS: i64 = 7 * 24 * 60 * 60;
12/// Sisu needs a few minutes before a submitted attainment shows up, so the first poll waits.
13pub const VERIFY_FIRST_DELAY_SECS: i64 = 120;
14pub const VERIFY_BASE_BACKOFF_SECS: i64 = 300;
15pub const VERIFY_MAX_BACKOFF_SECS: i64 = 6 * 60 * 60;
16/// After this, polling drops to daily and a human looks. Never a failure: the attainment may exist,
17/// and calling it failed would invite a second submission.
18pub const VERIFY_MAX_AGE_SECS: i64 = 14 * 24 * 60 * 60;
19pub const VERIFY_GIVE_UP_POLL_SECS: i64 = 24 * 60 * 60;
20pub const JITTER_MAX_SECS: i64 = 30;
21
22/// How long before the pipeline looks for an enrolment again on its own; a student recheck or
23/// enrolment discovery can wake the row sooner.
24pub const NO_USABLE_ENROLMENT_RECHECK_SECS: i64 = 24 * 60 * 60;
25
26/// How long between the checks that look for an attainment we may or may not have created.
27pub const UNCERTAIN_RECHECK_SECS: i64 = 15 * 60;
28/// After this many fruitless checks a human is asked to look in Sisu. The row still never resubmits.
29pub const UNCERTAIN_MAX_CHECKS: i32 = 3;
30
31/// A row still `submitting` this long belongs to a worker that died mid-call. Must stay comfortably
32/// longer than the client's request timeout, so a live request is never condemned.
33pub const SUBMITTING_RECOVERY_GRACE_SECS: i64 = 120;
34
35pub fn submit_backoff_secs(retry_count: i32) -> i64 {
36    exponential_backoff_secs(
37        SUBMIT_BASE_BACKOFF_SECS,
38        SUBMIT_MAX_BACKOFF_SECS,
39        retry_count,
40    )
41}
42
43/// The import phase schedules the first poll, so one prior attempt still means the base delay.
44pub fn verify_backoff_secs(attempt_count: i32) -> i64 {
45    exponential_backoff_secs(
46        VERIFY_BASE_BACKOFF_SECS,
47        VERIFY_MAX_BACKOFF_SECS,
48        attempt_count.saturating_sub(1),
49    )
50}
51
52/// Spreads a batch that failed together, so it does not come back as one thundering herd.
53pub fn next_attempt_at(now: DateTime<Utc>, delay_secs: i64) -> DateTime<Utc> {
54    headless_lms_utils::backoff::next_attempt_at(now, delay_secs, JITTER_MAX_SECS)
55}
56
57pub fn submit_window_expired(first_failed_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> bool {
58    window_expired(first_failed_at, now, SUBMIT_MAX_RETRY_AGE_SECS)
59}
60
61pub fn verify_window_expired(submitted_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> bool {
62    window_expired(submitted_at, now, VERIFY_MAX_AGE_SECS)
63}
64
65#[cfg(test)]
66mod tests {
67    use super::*;
68
69    #[test]
70    fn backoff_doubles_and_then_stops_growing() {
71        assert_eq!(submit_backoff_secs(0), SUBMIT_BASE_BACKOFF_SECS);
72        assert_eq!(submit_backoff_secs(1), SUBMIT_BASE_BACKOFF_SECS * 2);
73        assert_eq!(submit_backoff_secs(3), SUBMIT_BASE_BACKOFF_SECS * 8);
74        assert_eq!(submit_backoff_secs(30), SUBMIT_MAX_BACKOFF_SECS);
75        assert_eq!(submit_backoff_secs(i32::MAX), SUBMIT_MAX_BACKOFF_SECS);
76    }
77
78    #[test]
79    fn verify_backoff_starts_at_the_base_after_the_first_poll() {
80        assert_eq!(verify_backoff_secs(1), VERIFY_BASE_BACKOFF_SECS);
81        assert_eq!(verify_backoff_secs(2), VERIFY_BASE_BACKOFF_SECS * 2);
82        assert_eq!(verify_backoff_secs(100), VERIFY_MAX_BACKOFF_SECS);
83    }
84
85    #[test]
86    fn the_retry_window_runs_from_the_first_failure() {
87        let now = Utc::now();
88        assert!(!submit_window_expired(None, now));
89        assert!(!submit_window_expired(
90            Some(now - chrono::Duration::days(6)),
91            now
92        ));
93        assert!(submit_window_expired(
94            Some(now - chrono::Duration::days(8)),
95            now
96        ));
97    }
98
99    #[test]
100    fn the_verify_window_runs_from_the_submission() {
101        let now = Utc::now();
102        assert!(!verify_window_expired(None, now));
103        assert!(!verify_window_expired(
104            Some(now - chrono::Duration::days(13)),
105            now
106        ));
107        assert!(verify_window_expired(
108            Some(now - chrono::Duration::days(15)),
109            now
110        ));
111    }
112
113    #[test]
114    fn jitter_never_shortens_a_backoff() {
115        let now = Utc::now();
116        for _ in 0..50 {
117            let scheduled = next_attempt_at(now, 60);
118            assert!((scheduled - now).num_seconds() >= 60);
119            assert!((scheduled - now).num_seconds() <= 60 + JITTER_MAX_SECS);
120        }
121    }
122}
headless_lms_models/library/credit_registration/backoff.rs

headless_lms_models/library/credit_registration/
backoff.rs
