headless_lms_models/library/credit_registration/
backoff.rs

1//! How long the pipeline waits before trying again. Generic scheduling math; which error codes are
2//! even retryable is [`super::classification`].
3
4use chrono::TimeDelta;
5use headless_lms_utils::backoff::{exponential_backoff_secs, window_expired};
6
7use crate::prelude::*;
8use crate::suotar_api_calls::SuotarEndpoint;
9
10pub const SUBMIT_BASE_BACKOFF: TimeDelta = TimeDelta::minutes(1);
11pub const SUBMIT_MAX_BACKOFF: TimeDelta = TimeDelta::hours(6);
12/// After this long in failure a row stops being retried and becomes a support case.
13pub const SUBMIT_MAX_RETRY_AGE: TimeDelta = TimeDelta::days(7);
14/// When the first verify poll comes after the submission: about when the assessment item
15/// attainment usually shows.
16const VERIFY_WINDOW_START: TimeDelta = TimeDelta::hours(2);
17const VERIFY_DENSE_WINDOW_END: TimeDelta = TimeDelta::minutes(630);
18const VERIFY_WINDOW_END: TimeDelta = TimeDelta::hours(29);
19pub const VERIFY_WINDOW_INTERVAL: TimeDelta = TimeDelta::minutes(30);
20const VERIFY_LATE_WINDOW_INTERVAL: TimeDelta = TimeDelta::hours(1);
21pub const VERIFY_BASE_BACKOFF: TimeDelta = TimeDelta::hours(2);
22pub const VERIFY_MAX_BACKOFF: TimeDelta = TimeDelta::hours(6);
23/// After this, polling drops to daily and a human looks. Never a failure: the attainment may exist,
24/// and calling it failed would invite a second submission.
25pub const VERIFY_MAX_AGE: TimeDelta = TimeDelta::days(14);
26pub const VERIFY_GIVE_UP_POLL: TimeDelta = TimeDelta::days(1);
27pub const JITTER_MAX: TimeDelta = TimeDelta::seconds(30);
28
29/// The first wait before looking for an attainment we may or may not have created; it doubles
30/// after every fruitless look.
31pub const UNCERTAIN_RECHECK: TimeDelta = TimeDelta::minutes(15);
32pub const UNCERTAIN_MAX_RECHECK: TimeDelta = TimeDelta::hours(6);
33/// How long after the submission a human is asked to look in Sisu, well past the hour an
34/// attainment may take to show up. The row still never resubmits.
35pub const UNCERTAIN_ADMIN_AFTER: TimeDelta = TimeDelta::days(1);
36/// Suotar's `PENDING_WINDOW_MS`: how long it holds a submission it has not yet seen in Sisu as
37/// pending. Within it, a second import of the same completion can slip past Suotar's duplicate check.
38pub const SUOTAR_PENDING_WINDOW: TimeDelta = TimeDelta::hours(24);
39
40/// How long verify may see only the assessment item attainment before a human looks.
41pub const PARTIAL_REGISTRATION_ADMIN_AFTER: TimeDelta = TimeDelta::days(3);
42/// How many times Suotar may lose a submission (`notRegistered`) before a human looks. Each one
43/// still resubmits.
44pub const NOT_REGISTERED_REIMPORT_ADMIN_THRESHOLD: i32 = 3;
45
46/// A row still `submitting` this long belongs to a worker that died mid-call. Above the import
47/// timeout, so a live request is never condemned to `submission_uncertain`.
48pub const SUBMITTING_RECOVERY_GRACE: TimeDelta = TimeDelta::seconds(
49    SuotarEndpoint::ImportAttainments
50        .request_timeout()
51        .as_secs() as i64
52        + 15 * 60,
53);
54/// A row still `resolving_enrolment` this long belongs to a worker that died mid-call, and goes back
55/// to `ready_to_submit`; a parked row's claimed check expires after as long. Above the resolve
56/// timeout, and restarted before each resent half of a split batch, so a live answer still lands.
57pub const RESOLVING_RECOVERY_GRACE: TimeDelta = TimeDelta::seconds(
58    SuotarEndpoint::ResolveEnrolments
59        .request_timeout()
60        .as_secs() as i64
61        + 10 * 60,
62);
63
64fn doubling(base: TimeDelta, max: TimeDelta, count: i32) -> TimeDelta {
65    TimeDelta::seconds(exponential_backoff_secs(
66        base.num_seconds(),
67        max.num_seconds(),
68        count,
69    ))
70}
71
72/// The wait before the next submit after `retry_count` failed ones.
73pub fn submit_backoff(retry_count: i32) -> TimeDelta {
74    doubling(SUBMIT_BASE_BACKOFF, SUBMIT_MAX_BACKOFF, retry_count)
75}
76
77/// The wait until the next verify poll while the assessment item attainment is not seen yet. Timed
78/// from the submission rather than by counting polls, so a delayed row still polls when it is due.
79pub fn verify_delay(submitted_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> TimeDelta {
80    let Some(submitted_at) = submitted_at else {
81        return VERIFY_WINDOW_INTERVAL;
82    };
83    let age = now - submitted_at;
84    if age < VERIFY_WINDOW_START {
85        VERIFY_WINDOW_START - age
86    } else if age < VERIFY_DENSE_WINDOW_END {
87        VERIFY_WINDOW_INTERVAL
88    } else if age < VERIFY_WINDOW_END {
89        VERIFY_LATE_WINDOW_INTERVAL
90    } else {
91        // Grows with the time past the window, so polls thin out until the cap.
92        (age - VERIFY_WINDOW_END).clamp(VERIFY_BASE_BACKOFF, VERIFY_MAX_BACKOFF)
93    }
94}
95
96/// `lookup_count` counts the look just made, so the wait after the first one is already doubled.
97pub fn uncertain_recheck_delay(lookup_count: i32) -> TimeDelta {
98    doubling(UNCERTAIN_RECHECK, UNCERTAIN_MAX_RECHECK, lookup_count)
99}
100
101/// Spreads a batch that failed together, so it does not come back as one thundering herd.
102pub fn next_attempt_at(now: DateTime<Utc>, delay: TimeDelta) -> DateTime<Utc> {
103    headless_lms_utils::backoff::next_attempt_at(now, delay.num_seconds(), JITTER_MAX.num_seconds())
104}
105
106pub fn submit_window_expired(first_failed_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> bool {
107    window_expired(first_failed_at, now, SUBMIT_MAX_RETRY_AGE.num_seconds())
108}
109
110pub fn verify_window_expired(submitted_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> bool {
111    window_expired(submitted_at, now, VERIFY_MAX_AGE.num_seconds())
112}
113
114/// Whether an uncertain submission has waited long enough to be handed to an admin; never without
115/// a submission time.
116pub fn uncertain_needs_admin(submitted_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> bool {
117    window_expired(submitted_at, now, UNCERTAIN_ADMIN_AFTER.num_seconds())
118}
119
120#[cfg(test)]
121mod tests {
122    use super::*;
123
124    #[test]
125    fn backoff_doubles_and_then_stops_growing() {
126        assert_eq!(submit_backoff(0), SUBMIT_BASE_BACKOFF);
127        assert_eq!(submit_backoff(1), SUBMIT_BASE_BACKOFF * 2);
128        assert_eq!(submit_backoff(3), SUBMIT_BASE_BACKOFF * 8);
129        assert_eq!(submit_backoff(30), SUBMIT_MAX_BACKOFF);
130        assert_eq!(submit_backoff(i32::MAX), SUBMIT_MAX_BACKOFF);
131    }
132
133    #[test]
134    fn verify_polls_follow_the_landing_window_from_the_submission() {
135        let now = Utc::now();
136        let delay = |age: TimeDelta| verify_delay(Some(now - age), now);
137        assert_eq!(delay(TimeDelta::zero()), VERIFY_WINDOW_START);
138        assert_eq!(delay(TimeDelta::minutes(90)), TimeDelta::minutes(30));
139        assert_eq!(delay(TimeDelta::hours(2)), VERIFY_WINDOW_INTERVAL);
140        assert_eq!(delay(TimeDelta::hours(10)), VERIFY_WINDOW_INTERVAL);
141        assert_eq!(delay(TimeDelta::hours(11)), VERIFY_LATE_WINDOW_INTERVAL);
142        assert_eq!(delay(TimeDelta::hours(28)), VERIFY_LATE_WINDOW_INTERVAL);
143        assert_eq!(delay(TimeDelta::hours(29)), VERIFY_BASE_BACKOFF);
144        assert_eq!(delay(TimeDelta::hours(33)), VERIFY_BASE_BACKOFF * 2);
145        assert_eq!(delay(TimeDelta::days(5)), VERIFY_MAX_BACKOFF);
146        assert_eq!(verify_delay(None, now), VERIFY_WINDOW_INTERVAL);
147    }
148
149    #[test]
150    fn the_retry_window_runs_from_the_first_failure() {
151        let now = Utc::now();
152        assert!(!submit_window_expired(None, now));
153        assert!(!submit_window_expired(Some(now - TimeDelta::days(6)), now));
154        assert!(submit_window_expired(Some(now - TimeDelta::days(8)), now));
155    }
156
157    #[test]
158    fn the_verify_window_runs_from_the_submission() {
159        let now = Utc::now();
160        assert!(!verify_window_expired(None, now));
161        assert!(!verify_window_expired(Some(now - TimeDelta::days(13)), now));
162        assert!(verify_window_expired(Some(now - TimeDelta::days(15)), now));
163    }
164
165    #[test]
166    fn jitter_never_shortens_a_backoff() {
167        let now = Utc::now();
168        for _ in 0..50 {
169            let scheduled = next_attempt_at(now, TimeDelta::minutes(1));
170            assert!(scheduled - now >= TimeDelta::minutes(1));
171            assert!(scheduled - now <= TimeDelta::minutes(1) + JITTER_MAX);
172        }
173    }
174}
headless_lms_models/library/credit_registration/backoff.rs

headless_lms_models/library/credit_registration/
backoff.rs
