headless_lms_models/library/credit_registration/
backoff.rs

1//! How long the pipeline waits before trying again. Generic scheduling math; which error codes are
2//! even retryable is [`super::classification`].
3
4use chrono::TimeDelta;
5use headless_lms_utils::backoff::{exponential_backoff_secs, window_expired};
6
7use crate::prelude::*;
8use crate::suotar_api_calls::SuotarEndpoint;
9
10pub const SUBMIT_BASE_BACKOFF: TimeDelta = TimeDelta::minutes(1);
11pub const SUBMIT_MAX_BACKOFF: TimeDelta = TimeDelta::hours(6);
12/// After this long in failure a row stops being retried and becomes a support case.
13pub const SUBMIT_MAX_RETRY_AGE: TimeDelta = TimeDelta::days(7);
14/// Registrations land between about 5 and 29 hours after the submission, most of them in the first
15/// half: polls start shortly before, are close together through the dense part, and back off after.
16const VERIFY_WINDOW_START: TimeDelta = TimeDelta::minutes(270);
17const VERIFY_DENSE_WINDOW_END: TimeDelta = TimeDelta::minutes(630);
18const VERIFY_WINDOW_END: TimeDelta = TimeDelta::hours(29);
19pub const VERIFY_WINDOW_INTERVAL: TimeDelta = TimeDelta::minutes(30);
20const VERIFY_LATE_WINDOW_INTERVAL: TimeDelta = TimeDelta::hours(1);
21pub const VERIFY_BASE_BACKOFF: TimeDelta = TimeDelta::hours(2);
22pub const VERIFY_MAX_BACKOFF: TimeDelta = TimeDelta::hours(6);
23/// After this, polling drops to daily and a human looks. Never a failure: the attainment may exist,
24/// and calling it failed would invite a second submission.
25pub const VERIFY_MAX_AGE: TimeDelta = TimeDelta::days(14);
26pub const VERIFY_GIVE_UP_POLL: TimeDelta = TimeDelta::days(1);
27pub const JITTER_MAX: TimeDelta = TimeDelta::seconds(30);
28
29/// The first wait before looking for an attainment we may or may not have created; it doubles
30/// after every fruitless look.
31pub const UNCERTAIN_RECHECK: TimeDelta = TimeDelta::minutes(15);
32pub const UNCERTAIN_MAX_RECHECK: TimeDelta = TimeDelta::hours(6);
33/// How long after the submission a human is asked to look in Sisu, well past the hour an
34/// attainment may take to show up. The row still never resubmits.
35pub const UNCERTAIN_ADMIN_AFTER: TimeDelta = TimeDelta::days(1);
36/// Suotar's `PENDING_WINDOW_MS`: how long it holds a submission it has not yet seen in Sisu as
37/// pending. Within it, a second import of the same completion can slip past Suotar's duplicate check.
38pub const SUOTAR_PENDING_WINDOW: TimeDelta = TimeDelta::hours(24);
39
40/// How long verify may see only the assessment item attainment before a human looks.
41pub const PARTIAL_REGISTRATION_ADMIN_AFTER: TimeDelta = TimeDelta::days(3);
42/// How many times Suotar may lose a submission (`notRegistered`) before a human looks. Each one
43/// still resubmits.
44pub const NOT_REGISTERED_REIMPORT_ADMIN_THRESHOLD: i32 = 3;
45
46/// A row still `submitting` this long belongs to a worker that died mid-call. Above the import
47/// timeout, so a live request is never condemned to `submission_uncertain`.
48pub const SUBMITTING_RECOVERY_GRACE: TimeDelta = TimeDelta::seconds(
49    SuotarEndpoint::ImportAttainments
50        .request_timeout()
51        .as_secs() as i64
52        + 15 * 60,
53);
54/// A row still `resolving_enrolment` this long belongs to a worker that died mid-call, and goes back
55/// to `ready_to_submit`; a parked row's claimed check expires after as long. Above the resolve
56/// timeout, and restarted before each resent half of a split batch, so a live answer still lands.
57pub const RESOLVING_RECOVERY_GRACE: TimeDelta = TimeDelta::seconds(
58    SuotarEndpoint::ResolveEnrolments
59        .request_timeout()
60        .as_secs() as i64
61        + 10 * 60,
62);
63
64fn doubling(base: TimeDelta, max: TimeDelta, count: i32) -> TimeDelta {
65    TimeDelta::seconds(exponential_backoff_secs(
66        base.num_seconds(),
67        max.num_seconds(),
68        count,
69    ))
70}
71
72/// The wait before the next submit after `retry_count` failed ones.
73pub fn submit_backoff(retry_count: i32) -> TimeDelta {
74    doubling(SUBMIT_BASE_BACKOFF, SUBMIT_MAX_BACKOFF, retry_count)
75}
76
77/// The wait from `now` until the next verify poll of a row submitted at `submitted_at`. Timed from
78/// the submission rather than by counting polls, so a resumed or delayed row still polls at the
79/// times registrations land.
80pub fn verify_delay(submitted_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> TimeDelta {
81    let Some(submitted_at) = submitted_at else {
82        return VERIFY_WINDOW_INTERVAL;
83    };
84    let age = now - submitted_at;
85    if age < VERIFY_WINDOW_START {
86        VERIFY_WINDOW_START - age
87    } else if age < VERIFY_DENSE_WINDOW_END {
88        VERIFY_WINDOW_INTERVAL
89    } else if age < VERIFY_WINDOW_END {
90        VERIFY_LATE_WINDOW_INTERVAL
91    } else {
92        // Grows with the time past the window, so polls thin out until the cap.
93        (age - VERIFY_WINDOW_END).clamp(VERIFY_BASE_BACKOFF, VERIFY_MAX_BACKOFF)
94    }
95}
96
97/// `lookup_count` counts the look just made, so the wait after the first one is already doubled.
98pub fn uncertain_recheck_delay(lookup_count: i32) -> TimeDelta {
99    doubling(UNCERTAIN_RECHECK, UNCERTAIN_MAX_RECHECK, lookup_count)
100}
101
102/// Spreads a batch that failed together, so it does not come back as one thundering herd.
103pub fn next_attempt_at(now: DateTime<Utc>, delay: TimeDelta) -> DateTime<Utc> {
104    headless_lms_utils::backoff::next_attempt_at(now, delay.num_seconds(), JITTER_MAX.num_seconds())
105}
106
107pub fn submit_window_expired(first_failed_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> bool {
108    window_expired(first_failed_at, now, SUBMIT_MAX_RETRY_AGE.num_seconds())
109}
110
111pub fn verify_window_expired(submitted_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> bool {
112    window_expired(submitted_at, now, VERIFY_MAX_AGE.num_seconds())
113}
114
115/// Whether an uncertain submission has waited long enough to be handed to an admin; never without
116/// a submission time.
117pub fn uncertain_needs_admin(submitted_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> bool {
118    window_expired(submitted_at, now, UNCERTAIN_ADMIN_AFTER.num_seconds())
119}
120
121#[cfg(test)]
122mod tests {
123    use super::*;
124
125    #[test]
126    fn backoff_doubles_and_then_stops_growing() {
127        assert_eq!(submit_backoff(0), SUBMIT_BASE_BACKOFF);
128        assert_eq!(submit_backoff(1), SUBMIT_BASE_BACKOFF * 2);
129        assert_eq!(submit_backoff(3), SUBMIT_BASE_BACKOFF * 8);
130        assert_eq!(submit_backoff(30), SUBMIT_MAX_BACKOFF);
131        assert_eq!(submit_backoff(i32::MAX), SUBMIT_MAX_BACKOFF);
132    }
133
134    #[test]
135    fn verify_polls_follow_the_landing_window_from_the_submission() {
136        let now = Utc::now();
137        let delay = |age: TimeDelta| verify_delay(Some(now - age), now);
138        assert_eq!(delay(TimeDelta::zero()), VERIFY_WINDOW_START);
139        assert_eq!(delay(TimeDelta::hours(3)), TimeDelta::minutes(90));
140        assert_eq!(delay(TimeDelta::minutes(270)), VERIFY_WINDOW_INTERVAL);
141        assert_eq!(delay(TimeDelta::hours(10)), VERIFY_WINDOW_INTERVAL);
142        assert_eq!(delay(TimeDelta::hours(11)), VERIFY_LATE_WINDOW_INTERVAL);
143        assert_eq!(delay(TimeDelta::hours(28)), VERIFY_LATE_WINDOW_INTERVAL);
144        assert_eq!(delay(TimeDelta::hours(29)), VERIFY_BASE_BACKOFF);
145        assert_eq!(delay(TimeDelta::hours(33)), VERIFY_BASE_BACKOFF * 2);
146        assert_eq!(delay(TimeDelta::days(5)), VERIFY_MAX_BACKOFF);
147        assert_eq!(verify_delay(None, now), VERIFY_WINDOW_INTERVAL);
148    }
149
150    #[test]
151    fn the_retry_window_runs_from_the_first_failure() {
152        let now = Utc::now();
153        assert!(!submit_window_expired(None, now));
154        assert!(!submit_window_expired(Some(now - TimeDelta::days(6)), now));
155        assert!(submit_window_expired(Some(now - TimeDelta::days(8)), now));
156    }
157
158    #[test]
159    fn the_verify_window_runs_from_the_submission() {
160        let now = Utc::now();
161        assert!(!verify_window_expired(None, now));
162        assert!(!verify_window_expired(Some(now - TimeDelta::days(13)), now));
163        assert!(verify_window_expired(Some(now - TimeDelta::days(15)), now));
164    }
165
166    #[test]
167    fn jitter_never_shortens_a_backoff() {
168        let now = Utc::now();
169        for _ in 0..50 {
170            let scheduled = next_attempt_at(now, TimeDelta::minutes(1));
171            assert!(scheduled - now >= TimeDelta::minutes(1));
172            assert!(scheduled - now <= TimeDelta::minutes(1) + JITTER_MAX);
173        }
174    }
175}
headless_lms_models/library/credit_registration/backoff.rs

headless_lms_models/library/credit_registration/
backoff.rs
