headless_lms_server/domain/credit_registration/
health.rs

1//! The alert rules the credit registration dashboard renders.
2//!
3//! An alert carries identifiers and numbers, never prose: the study registry's own error text is
4//! written for an integrator and is not translated, so the frontend renders one key per alert id
5//! with these values interpolated. The thresholds travel with the alerts, not hardcoded twice.
6
7use headless_lms_models::credit_registrations::{
8    CreditRegistrationState, StuckRegistrationCount, StuckThresholds,
9};
10use headless_lms_models::library::credit_registration::materialize::get_unmaterialised_eligible_completions;
11use headless_lms_models::{ModelResult, prelude::*};
12use headless_lms_models::{
13    course_module_suotar_configurations, course_module_suotar_realisations,
14    credit_registration_account_linking_emails, credit_registration_events,
15    credit_registration_phase_state, credit_registrations, suotar_api_calls,
16};
17use utoipa::ToSchema;
18
19use crate::domain::system_health::HealthStatus;
20
21/// Within this much of the past, one rejected credential is enough.
22const CREDENTIAL_REJECTION_WINDOW_SECS: i64 = 60 * 60;
23const UNREACHABLE_WINDOW_SECS: i64 = 15 * 60;
24/// Below this the run is a bad minute rather than an outage.
25const UNREACHABLE_CONSECUTIVE_FAILURES: i64 = 3;
26const SISU_OUTAGE_WINDOW_SECS: i64 = 15 * 60;
27/// Below this many items the share below is one bad batch, not a signal.
28const SISU_OUTAGE_MIN_ITEMS: i64 = 10;
29const SISU_OUTAGE_FAILURE_SHARE_PERCENT: i64 = 30;
30const STUCK_THRESHOLDS: StuckThresholds = StuckThresholds {
31    stuck_ready_to_submit_secs: 2 * 60 * 60,
32    stuck_submitting_secs: 15 * 60,
33    stuck_awaiting_verification_secs: 24 * 60 * 60,
34    stuck_failed_retryable_secs: 3 * 24 * 60 * 60,
35};
36
37const _: () = assert!(
38    STUCK_THRESHOLDS.stuck_failed_retryable_secs
39        < headless_lms_models::library::credit_registration::backoff::SUBMIT_MAX_RETRY_AGE_SECS,
40    "a row must be considered stuck before backoff gives up retrying it"
41);
42const _: () = assert!(
43    STUCK_THRESHOLDS.stuck_submitting_secs
44        > headless_lms_models::library::credit_registration::backoff::SUBMITTING_RECOVERY_GRACE_SECS,
45    "the stuck threshold must outlast the grace period that lets a submit recover on its own"
46);
47/// Above this many stuck rows the backlog stops being something to look at tomorrow.
48const STUCK_CRITICAL_COUNT: i64 = 50;
49const LINKING_MAIL_WINDOW_SECS: i64 = 7 * 24 * 60 * 60;
50/// A phase is late once this many of its own intervals have passed without a heartbeat.
51/// `pub(crate)` because the dashboard's phase rows apply the same threshold server-side.
52pub(crate) const PHASE_HEARTBEAT_INTERVAL_MULTIPLIER: i32 = 2;
53/// Failures in a row before a phase counts as broken rather than unlucky.
54pub(crate) const PHASE_CONSECUTIVE_FAILURE_LIMIT: i32 = 5;
55/// A phase that owns a nonempty queue and has not succeeded within this many of its own intervals
56/// is running without getting anywhere, which no failure count catches.
57const PHASE_SUCCESS_INTERVAL_MULTIPLIER: i32 = 10;
58/// The window every "in the last day" rule shares.
59const TERMINAL_WINDOW_SECS: i64 = 24 * 60 * 60;
60const PERMANENT_FAILURE_COUNT: i64 = 20;
61const PERMANENT_FAILURE_RATE_PERCENT: i64 = 10;
62/// A reversal is always worth saying; this many at once is an incident.
63const MISREGISTRATION_CRITICAL_COUNT: i64 = 5;
64/// Linking mails one hour may hand over before the volume itself is the problem.
65const LINKING_MAIL_HOURLY_CAP: i64 = 500;
66/// Queued work that makes a day without a single completion mean something.
67const IDLE_QUEUE_DEPTH: i64 = 20;
68/// How long a completion may sit outside the ledger before `materialize` is the suspect rather
69/// than the clock.
70const NEVER_ENTERED_MIN_AGE_SECS: i64 = 6 * 60 * 60;
71/// Bounds the anti-join behind that rule; a bigger backlog reports as this many.
72const NEVER_ENTERED_SAMPLE_LIMIT: i64 = 100;
73const LATENCY_WINDOW_SECS: i64 = 7 * 24 * 60 * 60;
74/// Under this the registry is quick enough that a doubling says nothing.
75const LATENCY_REGRESSION_FLOOR_SECS: i64 = 6 * 60 * 60;
76const LATENCY_REGRESSION_FACTOR: i64 = 2;
77/// One person the registry names differently from the account whose address matched is worth a
78/// look: it is the only signal we get that a university address was reissued.
79const FAST_TRACK_NAME_MISMATCH_COUNT: i64 = 1;
80
81#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy, ToSchema)]
82#[serde(rename_all = "snake_case")]
83pub enum CreditRegistrationAlertId {
84    CredentialsRejected,
85    StudyRegistryUnreachable,
86    SisuUnavailable,
87    StuckRegistrations,
88    LinkingMailSendFailed,
89    LinkingMailRateCapExceeded,
90    PhaseHeartbeatStale,
91    PhaseFailing,
92    PermanentFailuresAccumulating,
93    MisregistrationsDetected,
94    CourseConfigurationBroken,
95    PipelineIdle,
96    CompletionsNeverEntered,
97    ConfirmationLatencyRegressed,
98    FastTrackNameMismatch,
99    PipelinePausedGlobally,
100}
101
102#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, ToSchema)]
103#[serde(rename_all = "snake_case")]
104pub enum CreditRegistrationAlertSeverity {
105    /// Worth knowing, not worth acting on. Never makes the overall status anything but healthy.
106    Info,
107    Warning,
108    Critical,
109}
110
111#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
112pub struct CreditRegistrationAlert {
113    pub id: CreditRegistrationAlertId,
114    pub severity: CreditRegistrationAlertSeverity,
115    /// How many rows, calls or phases the rule found.
116    pub count: i64,
117    /// What `count` is out of, where the rule measured one. Not a threshold: thresholds are the
118    /// same for every evaluation and travel separately.
119    pub total: Option<i64>,
120    /// When it last happened, where the rule has an instant to point at.
121    pub at: Option<DateTime<Utc>>,
122    /// An identifier the operator can act on — a phase name, a ledger state, a mail domain. Never a
123    /// sentence, and never anything the study registry wrote.
124    pub subject: Option<String>,
125}
126
127/// The only thresholds the frontend reads off the health poll: how long a row may sit in each
128/// state before it counts as stuck. The other rule constants stay server-side.
129pub type CreditRegistrationAlertThresholds = StuckThresholds;
130
131#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
132pub struct CreditRegistrationHealth {
133    pub status: HealthStatus,
134    /// Critical first, and a rejected credential first of all: nothing registers until it is fixed.
135    pub alerts: Vec<CreditRegistrationAlert>,
136    pub thresholds: CreditRegistrationAlertThresholds,
137}
138
139/// Alias for [`stuck_thresholds`]: the same values, named for the health-poll wire response.
140pub fn thresholds() -> CreditRegistrationAlertThresholds {
141    stuck_thresholds()
142}
143
144pub fn stuck_thresholds() -> StuckThresholds {
145    STUCK_THRESHOLDS
146}
147
148/// A phase counts as late once more than [`PHASE_HEARTBEAT_INTERVAL_MULTIPLIER`] of its own
149/// interval has passed since its last heartbeat. A paused phase is never late: it is not expected
150/// to be heartbeating at all.
151pub(crate) fn is_heartbeat_late(
152    last_heartbeat_at: Option<DateTime<Utc>>,
153    expected_interval_secs: i32,
154    paused_at: Option<DateTime<Utc>>,
155    now: DateTime<Utc>,
156) -> bool {
157    paused_at.is_none()
158        && last_heartbeat_at.is_some_and(|at| {
159            (now - at).num_seconds()
160                > i64::from(expected_interval_secs) * i64::from(PHASE_HEARTBEAT_INTERVAL_MULTIPLIER)
161        })
162}
163
164/// Runs every rule and ranks what it found.
165///
166/// `stuck` and `depths` are passed in because the caller already reads both aggregates, the two
167/// most expensive reads in the request. `depths` is the live count per state, superseded rows
168/// excluded, as [`credit_registrations::count_by_state`] returns it.
169pub async fn evaluate(
170    conn: &mut PgConnection,
171    stuck: &[StuckRegistrationCount],
172    depths: &[(CreditRegistrationState, i64)],
173) -> ModelResult<CreditRegistrationHealth> {
174    let now = Utc::now();
175    let mut alerts = Vec::new();
176
177    let credentials = suotar_api_calls::count_credential_rejections_since(
178        conn,
179        now - chrono::Duration::seconds(CREDENTIAL_REJECTION_WINDOW_SECS),
180    )
181    .await?;
182    if credentials.count > 0 {
183        alerts.push(CreditRegistrationAlert {
184            id: CreditRegistrationAlertId::CredentialsRejected,
185            severity: CreditRegistrationAlertSeverity::Critical,
186            count: credentials.count,
187            total: None,
188            at: credentials.last_at,
189            subject: None,
190        });
191    }
192
193    let unreachable = suotar_api_calls::count_unreachable_run_since(
194        conn,
195        now - chrono::Duration::seconds(UNREACHABLE_WINDOW_SECS),
196    )
197    .await?;
198    if unreachable.count >= UNREACHABLE_CONSECUTIVE_FAILURES {
199        alerts.push(CreditRegistrationAlert {
200            id: CreditRegistrationAlertId::StudyRegistryUnreachable,
201            severity: CreditRegistrationAlertSeverity::Critical,
202            count: unreachable.count,
203            total: None,
204            at: unreachable.last_at,
205            subject: None,
206        });
207    }
208
209    if let Some(alert) = sisu_outage_alert(conn, now).await? {
210        alerts.push(alert);
211    }
212    if let Some(alert) = stuck_alert(stuck) {
213        alerts.push(alert);
214    }
215    if let Some(alert) = linking_mail_alert(conn, now).await? {
216        alerts.push(alert);
217    }
218    if let Some(alert) = linking_mail_rate_alert(conn, now).await? {
219        alerts.push(alert);
220    }
221    alerts.extend(phase_alerts(conn, now, depths).await?);
222    alerts.extend(terminal_outcome_alerts(conn, now, depths).await?);
223    if let Some(alert) = course_configuration_alert(conn).await? {
224        alerts.push(alert);
225    }
226    if let Some(alert) = never_entered_alert(conn).await? {
227        alerts.push(alert);
228    }
229    if let Some(alert) = latency_regression_alert(conn, now).await? {
230        alerts.push(alert);
231    }
232    if let Some(alert) = fast_track_name_mismatch_alert(conn).await? {
233        alerts.push(alert);
234    }
235
236    alerts.sort_by_key(|alert| {
237        (
238            std::cmp::Reverse(alert.severity),
239            alert.id != CreditRegistrationAlertId::CredentialsRejected,
240        )
241    });
242    let status = match alerts.iter().map(|alert| alert.severity).max() {
243        Some(CreditRegistrationAlertSeverity::Critical) => HealthStatus::Error,
244        Some(CreditRegistrationAlertSeverity::Warning) => HealthStatus::Warning,
245        Some(CreditRegistrationAlertSeverity::Info) | None => HealthStatus::Healthy,
246    };
247
248    Ok(CreditRegistrationHealth {
249        status,
250        alerts,
251        thresholds: thresholds(),
252    })
253}
254
255/// The share of recent items the study registry blamed on Sisu. Our only proxy for Sisu's uptime,
256/// which is why it is a rule of its own rather than part of the request-level one above.
257async fn sisu_outage_alert(
258    conn: &mut PgConnection,
259    now: DateTime<Utc>,
260) -> ModelResult<Option<CreditRegistrationAlert>> {
261    let totals = credit_registration_events::count_item_outcomes_since(
262        conn,
263        now - chrono::Duration::seconds(SISU_OUTAGE_WINDOW_SECS),
264    )
265    .await?;
266    if totals.item_count < SISU_OUTAGE_MIN_ITEMS
267        || totals.sisu_unavailable_count * 100
268            < totals.item_count * SISU_OUTAGE_FAILURE_SHARE_PERCENT
269    {
270        return Ok(None);
271    }
272    Ok(Some(CreditRegistrationAlert {
273        id: CreditRegistrationAlertId::SisuUnavailable,
274        severity: CreditRegistrationAlertSeverity::Critical,
275        count: totals.sisu_unavailable_count,
276        total: Some(totals.item_count),
277        at: totals.last_sisu_unavailable_at,
278        subject: None,
279    }))
280}
281
282/// Rows the pipeline should have moved on by now. Terminal states are outside this by construction
283/// rather than by a filter that could be forgotten.
284fn stuck_alert(stuck: &[StuckRegistrationCount]) -> Option<CreditRegistrationAlert> {
285    let total: i64 = stuck.iter().map(|row| row.count).sum();
286    if total == 0 {
287        return None;
288    }
289    let severe: i64 = stuck.iter().map(|row| row.severely_stuck_count).sum();
290    let worst = stuck.iter().max_by_key(|row| row.count);
291    let severity = if total > STUCK_CRITICAL_COUNT || severe > 0 {
292        CreditRegistrationAlertSeverity::Critical
293    } else {
294        CreditRegistrationAlertSeverity::Warning
295    };
296    Some(CreditRegistrationAlert {
297        id: CreditRegistrationAlertId::StuckRegistrations,
298        severity,
299        count: total,
300        total: None,
301        at: worst.and_then(|row| row.oldest_state_entered_at),
302        subject: worst.map(|row| state_name(row.state)),
303    })
304}
305
306/// Linking mails we could not hand over at all. The recipient domain rides along because an
307/// undeliverable host is the usual cause.
308async fn linking_mail_alert(
309    conn: &mut PgConnection,
310    now: DateTime<Utc>,
311) -> ModelResult<Option<CreditRegistrationAlert>> {
312    let since = now - chrono::Duration::seconds(LINKING_MAIL_WINDOW_SECS);
313    let totals =
314        credit_registration_account_linking_emails::get_send_status_totals_since(conn, since, now)
315            .await?;
316    if totals.send_failed == 0 {
317        return Ok(None);
318    }
319    let top_domain = credit_registration_account_linking_emails::get_send_failure_domains_since(
320        conn, since, now,
321    )
322    .await?
323    .into_iter()
324    .next()
325    .map(|row| row.domain);
326    Ok(Some(CreditRegistrationAlert {
327        id: CreditRegistrationAlertId::LinkingMailSendFailed,
328        severity: CreditRegistrationAlertSeverity::Warning,
329        count: totals.send_failed,
330        total: Some(totals.mails_in_window),
331        at: totals.last_send_failed_at,
332        subject: top_domain,
333    }))
334}
335
336/// The volume guard: how many people we mailed in the last hour against what an hour should hold.
337/// Counts addresses, which is what the per-person caps govern.
338async fn linking_mail_rate_alert(
339    conn: &mut PgConnection,
340    now: DateTime<Utc>,
341) -> ModelResult<Option<CreditRegistrationAlert>> {
342    let sent = credit_registration_account_linking_emails::count_sent_since(
343        conn,
344        now - chrono::Duration::hours(1),
345    )
346    .await?;
347    if sent <= LINKING_MAIL_HOURLY_CAP {
348        return Ok(None);
349    }
350    let severity = if sent > LINKING_MAIL_HOURLY_CAP * 2 {
351        CreditRegistrationAlertSeverity::Critical
352    } else {
353        CreditRegistrationAlertSeverity::Warning
354    };
355    Ok(Some(CreditRegistrationAlert {
356        id: CreditRegistrationAlertId::LinkingMailRateCapExceeded,
357        severity,
358        count: sent,
359        total: Some(LINKING_MAIL_HOURLY_CAP),
360        at: Some(now),
361        subject: None,
362    }))
363}
364
365/// What the phase table says about itself: phases that stopped reporting, and phases that report
366/// but get nowhere.
367///
368/// A phase that has never heartbeated is deliberately outside both: a freshly migrated database has
369/// no heartbeats at all, and that would keep the banner permanently red.
370async fn phase_alerts(
371    conn: &mut PgConnection,
372    now: DateTime<Utc>,
373    depths: &[(CreditRegistrationState, i64)],
374) -> ModelResult<Vec<CreditRegistrationAlert>> {
375    let phases = credit_registration_phase_state::get_all(conn).await?;
376    let mut stale: Vec<&str> = Vec::new();
377    let mut failing: Vec<&str> = Vec::new();
378    let mut paused = 0;
379    let mut last_paused_at = None;
380    for phase in &phases {
381        if let Some(paused_at) = phase.paused_at {
382            paused += 1;
383            last_paused_at = last_paused_at.max(Some(paused_at));
384            continue;
385        }
386        let interval = i64::from(phase.expected_interval_secs);
387        if is_heartbeat_late(
388            phase.last_heartbeat_at,
389            phase.expected_interval_secs,
390            None,
391            now,
392        ) {
393            stale.push(&phase.phase);
394        }
395        let owns_work =
396            crate::domain::credit_registration_phases::CreditRegistrationPhase::from_phase_name(
397                &phase.phase,
398            )
399            .is_some_and(|known| owned_depth(known, depths) > 0);
400        let unproductive = owns_work
401            && phase.last_success_at.is_some_and(|last_success_at| {
402                (now - last_success_at).num_seconds()
403                    > interval * i64::from(PHASE_SUCCESS_INTERVAL_MULTIPLIER)
404            });
405        if phase.consecutive_failures >= PHASE_CONSECUTIVE_FAILURE_LIMIT || unproductive {
406            failing.push(&phase.phase);
407        }
408    }
409
410    let mut alerts = Vec::new();
411    if !stale.is_empty() {
412        alerts.push(CreditRegistrationAlert {
413            id: CreditRegistrationAlertId::PhaseHeartbeatStale,
414            severity: CreditRegistrationAlertSeverity::Critical,
415            count: stale.len() as i64,
416            total: Some(phases.len() as i64),
417            at: None,
418            subject: stale.first().map(|phase| (*phase).to_string()),
419        });
420    }
421    if !failing.is_empty() {
422        alerts.push(CreditRegistrationAlert {
423            id: CreditRegistrationAlertId::PhaseFailing,
424            severity: CreditRegistrationAlertSeverity::Critical,
425            count: failing.len() as i64,
426            total: Some(phases.len() as i64),
427            at: None,
428            subject: failing.first().map(|phase| (*phase).to_string()),
429        });
430    }
431    if paused > 0 && paused == phases.len() {
432        alerts.push(CreditRegistrationAlert {
433            id: CreditRegistrationAlertId::PipelinePausedGlobally,
434            severity: CreditRegistrationAlertSeverity::Info,
435            count: paused as i64,
436            total: Some(phases.len() as i64),
437            at: last_paused_at,
438            subject: None,
439        });
440    }
441    Ok(alerts)
442}
443
444/// The three rules read off the last day's terminal outcomes: failures piling up, reversals, and a
445/// pipeline that finished nothing while holding work.
446async fn terminal_outcome_alerts(
447    conn: &mut PgConnection,
448    now: DateTime<Utc>,
449    depths: &[(CreditRegistrationState, i64)],
450) -> ModelResult<Vec<CreditRegistrationAlert>> {
451    let since = now - chrono::Duration::seconds(TERMINAL_WINDOW_SECS);
452    let totals = credit_registrations::count_terminal_outcomes_since(conn, since).await?;
453    let mut alerts = Vec::new();
454
455    let rate_broken = totals.total_count >= PERMANENT_FAILURE_COUNT
456        && totals.failed_permanent_count * 100
457            > totals.total_count * PERMANENT_FAILURE_RATE_PERCENT;
458    if totals.failed_permanent_count >= PERMANENT_FAILURE_COUNT || rate_broken {
459        alerts.push(CreditRegistrationAlert {
460            id: CreditRegistrationAlertId::PermanentFailuresAccumulating,
461            severity: CreditRegistrationAlertSeverity::Warning,
462            count: totals.failed_permanent_count,
463            total: Some(totals.total_count),
464            at: Some(now),
465            subject: None,
466        });
467    }
468
469    let misregistered = credit_registrations::count_entered_state_since(
470        conn,
471        CreditRegistrationState::Misregistered,
472        since,
473    )
474    .await?;
475    if misregistered > 0 {
476        alerts.push(CreditRegistrationAlert {
477            id: CreditRegistrationAlertId::MisregistrationsDetected,
478            severity: if misregistered >= MISREGISTRATION_CRITICAL_COUNT {
479                CreditRegistrationAlertSeverity::Critical
480            } else {
481                CreditRegistrationAlertSeverity::Warning
482            },
483            count: misregistered,
484            total: None,
485            at: Some(now),
486            subject: None,
487        });
488    }
489
490    let queued = depth_of(depths, CreditRegistrationState::ReadyToSubmit)
491        + depth_of(depths, CreditRegistrationState::AwaitingVerification);
492    if totals.total_count == 0 && queued > IDLE_QUEUE_DEPTH {
493        alerts.push(CreditRegistrationAlert {
494            id: CreditRegistrationAlertId::PipelineIdle,
495            severity: CreditRegistrationAlertSeverity::Warning,
496            count: queued,
497            total: None,
498            at: Some(now),
499            subject: None,
500        });
501    }
502    Ok(alerts)
503}
504
505/// Modules the last configuration check found broken. Never checked is not counted: the Courses tab
506/// renders unknown and broken differently, and so must this.
507async fn course_configuration_alert(
508    conn: &mut PgConnection,
509) -> ModelResult<Option<CreditRegistrationAlert>> {
510    let count =
511        course_module_suotar_configurations::count_modules_failing_config_check(conn).await?;
512    Ok((count > 0).then_some(CreditRegistrationAlert {
513        id: CreditRegistrationAlertId::CourseConfigurationBroken,
514        severity: CreditRegistrationAlertSeverity::Warning,
515        count,
516        total: None,
517        at: None,
518        subject: None,
519    }))
520}
521
522/// Completions old enough that `materialize` has had every chance and still has no ledger row for
523/// them. Sampled rather than counted, so the anti-join stops early on a large backlog.
524async fn never_entered_alert(
525    conn: &mut PgConnection,
526) -> ModelResult<Option<CreditRegistrationAlert>> {
527    let found = get_unmaterialised_eligible_completions(
528        conn,
529        NEVER_ENTERED_MIN_AGE_SECS,
530        NEVER_ENTERED_SAMPLE_LIMIT,
531    )
532    .await?;
533    if found.is_empty() {
534        return Ok(None);
535    }
536    Ok(Some(CreditRegistrationAlert {
537        id: CreditRegistrationAlertId::CompletionsNeverEntered,
538        severity: CreditRegistrationAlertSeverity::Warning,
539        count: found.len() as i64,
540        total: Some(NEVER_ENTERED_SAMPLE_LIMIT),
541        at: found.first().map(|row| row.created_at),
542        subject: None,
543    }))
544}
545
546/// How long the study registry is taking to confirm, this week against last. `count` is this
547/// week's p95 in seconds and `total` last week's, so the banner can name both.
548async fn latency_regression_alert(
549    conn: &mut PgConnection,
550    now: DateTime<Utc>,
551) -> ModelResult<Option<CreditRegistrationAlert>> {
552    let window = chrono::Duration::seconds(LATENCY_WINDOW_SECS);
553    let current =
554        credit_registrations::get_registration_latency_between(conn, now - window, now).await?;
555    let (Some(current_p95), true) = (current.p95_confirmation_secs, current.registered_count > 0)
556    else {
557        return Ok(None);
558    };
559    if current_p95 < LATENCY_REGRESSION_FLOOR_SECS {
560        return Ok(None);
561    }
562    let previous = credit_registrations::get_registration_latency_between(
563        conn,
564        now - window * 2,
565        now - window,
566    )
567    .await?;
568    let Some(previous_p95) = previous
569        .p95_confirmation_secs
570        .filter(|_| previous.registered_count > 0)
571    else {
572        return Ok(None);
573    };
574    if current_p95 <= previous_p95 * LATENCY_REGRESSION_FACTOR {
575        return Ok(None);
576    }
577    Ok(Some(CreditRegistrationAlert {
578        id: CreditRegistrationAlertId::ConfirmationLatencyRegressed,
579        severity: CreditRegistrationAlertSeverity::Info,
580        count: current_p95,
581        total: Some(previous_p95),
582        at: Some(now),
583        subject: None,
584    }))
585}
586
587/// Persons whose university address matched a verified account under a different name. The
588/// observable signature of an address reissued to somebody else, and the only warning we get before
589/// a link is made to the wrong account.
590async fn fast_track_name_mismatch_alert(
591    conn: &mut PgConnection,
592) -> ModelResult<Option<CreditRegistrationAlert>> {
593    let count =
594        course_module_suotar_realisations::sum_last_fast_track_name_mismatches(conn).await?;
595    Ok(
596        (count >= FAST_TRACK_NAME_MISMATCH_COUNT).then_some(CreditRegistrationAlert {
597            id: CreditRegistrationAlertId::FastTrackNameMismatch,
598            severity: CreditRegistrationAlertSeverity::Warning,
599            count,
600            total: None,
601            at: None,
602            subject: None,
603        }),
604    )
605}
606
607fn depth_of(depths: &[(CreditRegistrationState, i64)], state: CreditRegistrationState) -> i64 {
608    depths
609        .iter()
610        .find(|(row_state, _)| *row_state == state)
611        .map_or(0, |(_, count)| *count)
612}
613
614fn owned_depth(
615    phase: crate::domain::credit_registration_phases::CreditRegistrationPhase,
616    depths: &[(CreditRegistrationState, i64)],
617) -> i64 {
618    phase
619        .owned_states()
620        .iter()
621        .map(|state| depth_of(depths, *state))
622        .sum()
623}
624
625/// The state's own wire name, taken from its serialisation so the two cannot drift.
626fn state_name(state: CreditRegistrationState) -> String {
627    serde_json::to_value(state)
628        .ok()
629        .and_then(|value| value.as_str().map(str::to_string))
630        .unwrap_or_default()
631}
632
633#[cfg(test)]
634mod tests {
635    use super::*;
636
637    /// The alert's `subject` has to be the same spelling the ledger and the filters use.
638    #[test]
639    fn a_state_names_itself_the_way_the_wire_does() {
640        assert_eq!(
641            state_name(CreditRegistrationState::AwaitingVerification),
642            "awaiting_verification"
643        );
644        for state in CreditRegistrationState::ALL {
645            assert!(!state_name(state).is_empty());
646        }
647    }
648
649    /// Info exists to be shown without turning the page red.
650    #[test]
651    fn severity_ranks_the_way_the_banner_reads_it() {
652        assert!(
653            CreditRegistrationAlertSeverity::Critical > CreditRegistrationAlertSeverity::Warning
654        );
655        assert!(CreditRegistrationAlertSeverity::Warning > CreditRegistrationAlertSeverity::Info);
656    }
657}
headless_lms_server/domain/credit_registration/health.rs

headless_lms_server/domain/credit_registration/
health.rs
