headless_lms_server/controllers/main_frontend/credit_registration_admin/
errors.rs

1//! The Errors & stuck tab: what is going wrong by error code, and which rows want a human.
2
3use headless_lms_models::credit_registration_events::{self, ErrorCodeWindowCounts};
4use headless_lms_models::credit_registrations::{
5    self, AttentionRegistration, CreditRegistrationErrorCode, CreditRegistrationState,
6};
7use headless_lms_models::library::credit_registration::classification::{
8    Retryability, retryability,
9};
10use headless_lms_models::suotar_api_calls::SuotarEndpoint;
11use utoipa::ToSchema;
12
13use crate::domain::credit_registration::health::{
14    CreditRegistrationAlertThresholds, stuck_thresholds, thresholds,
15};
16use crate::prelude::*;
17
18use super::authorize_credit_registration_admin;
19
20/// The `chatbot_syncer` precedent, and the point past which retrying is not the answer.
21const TOO_MANY_ATTEMPTS: i32 = 5;
22/// Bounds the attention table. A dashboard that renders ten thousand rows helps nobody, and the
23/// per-reason counts beside it say how much is left.
24const ATTENTION_LIMIT: i64 = 500;
25const DEFAULT_ERROR_WINDOW_SECS: i64 = 24 * 60 * 60;
26const MAX_ERROR_WINDOW_SECS: i64 = 90 * 24 * 60 * 60;
27
28/// Why a row is on the attention table. One row can carry several.
29#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy, Hash, ToSchema)]
30#[serde(rename_all = "snake_case")]
31pub enum CreditRegistrationAttentionReason {
32    /// Past its state's threshold with the pipeline still owning it.
33    StuckInState,
34    PermanentError,
35    RetryWindowExpired,
36    Misregistered,
37    TooManyAttempts,
38    /// `submission_uncertain`: never retried automatically, and never in bulk.
39    OutcomeUncertain,
40    /// The pipeline itself asked for a human, e.g. because the completion drifted under the row.
41    FlaggedByPipeline,
42}
43
44#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
45pub struct CreditRegistrationAttentionItem {
46    pub credit_registration_id: Uuid,
47    pub user_id: Uuid,
48    pub first_name: Option<String>,
49    pub last_name: Option<String>,
50    /// In full: this is the list support works from.
51    pub email: Option<String>,
52    pub course_id: Uuid,
53    pub course_name: String,
54    pub course_module_id: Uuid,
55    pub course_module_name: Option<String>,
56    pub state: CreditRegistrationState,
57    pub state_entered_at: DateTime<Utc>,
58    pub error_code: Option<CreditRegistrationErrorCode>,
59    pub attempt_count: i32,
60    pub next_attempt_at: DateTime<Utc>,
61    pub student_number: Option<String>,
62    /// Every detector that picked this row, so the table can group by any of them.
63    pub reasons: Vec<CreditRegistrationAttentionReason>,
64}
65
66#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
67pub struct CreditRegistrationAttentionReasonCount {
68    pub reason: CreditRegistrationAttentionReason,
69    pub count: i64,
70}
71
72#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
73pub struct CreditRegistrationAttentionItems {
74    pub items: Vec<CreditRegistrationAttentionItem>,
75    /// Rows returned, which is the tab badge. Capped at `max_items`; the counts per reason are over
76    /// the same capped set.
77    pub total_count: i64,
78    pub counts_by_reason: Vec<CreditRegistrationAttentionReasonCount>,
79    pub max_items: i64,
80}
81
82/// One error code over the chosen window and the one before it.
83#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
84pub struct CreditRegistrationErrorCodeWindow {
85    pub error_code: CreditRegistrationErrorCode,
86    /// What may be done about the code, which is the difference between a wait and a fix.
87    pub retryability: Retryability,
88    pub current_count: i64,
89    pub previous_count: i64,
90    pub user_count: i64,
91    pub course_count: i64,
92    pub first_seen_at: Option<DateTime<Utc>>,
93    pub last_seen_at: Option<DateTime<Utc>>,
94    pub endpoints: Vec<SuotarEndpoint>,
95}
96
97/// The verdicts an operator needs beside the errors to rule them out. `not_improved` is not a
98/// failure and is never in the error table above.
99#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
100pub struct CreditRegistrationTerminalVerdicts {
101    pub registered_count: i64,
102    pub duplicate_and_not_improved_count: i64,
103    pub failed_permanent_count: i64,
104    pub cancelled_count: i64,
105    /// The denominator of the success rate.
106    pub total_count: i64,
107}
108
109#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
110pub struct CreditRegistrationErrorsByCode {
111    pub window_secs: i64,
112    pub codes: Vec<CreditRegistrationErrorCodeWindow>,
113    pub verdicts: CreditRegistrationTerminalVerdicts,
114}
115
116#[derive(Debug, Deserialize)]
117pub struct ErrorWindowQuery {
118    window_secs: Option<i64>,
119}
120
121/**
122GET `/api/v0/main-frontend/credit-registration-admin/thresholds` - Every number the alert rules and
123the stuck detectors use.
124
125The same values `/overview` embeds in its health block. Separate so a tab explaining "stuck after
1262 hours" can say so without reading the whole overview aggregate.
127*/
128#[instrument(skip(pool))]
129#[utoipa::path(
130    get,
131    path = "/thresholds",
132    operation_id = "getCreditRegistrationThresholds",
133    tag = "credit-registration-admin",
134    responses(
135        (status = 200, description = "The thresholds every rule and detector shares", body = CreditRegistrationAlertThresholds)
136    )
137)]
138pub async fn get_credit_registration_thresholds(
139    user: AuthUser,
140    pool: web::Data<PgPool>,
141) -> ControllerResult<web::Json<CreditRegistrationAlertThresholds>> {
142    let mut conn = pool.acquire().await?;
143    let token = authorize_credit_registration_admin(&mut conn, user.id).await?;
144    token.authorized_ok(web::Json(thresholds()))
145}
146
147/**
148GET `/api/v0/main-frontend/credit-registration-admin/attention` - The rows at least one detector
149wants a human to look at, with the detectors that picked each.
150
151Superseded attempts are outside every detector: acting on a replaced attempt is never right.
152*/
153#[instrument(skip(pool))]
154#[utoipa::path(
155    get,
156    path = "/attention",
157    operation_id = "getCreditRegistrationAttentionItems",
158    tag = "credit-registration-admin",
159    responses(
160        (status = 200, description = "Rows needing a human, and how many for each reason", body = CreditRegistrationAttentionItems)
161    )
162)]
163pub async fn get_credit_registration_attention_items(
164    user: AuthUser,
165    pool: web::Data<PgPool>,
166) -> ControllerResult<web::Json<CreditRegistrationAttentionItems>> {
167    let mut conn = pool.acquire().await?;
168    let token = authorize_credit_registration_admin(&mut conn, user.id).await?;
169
170    let items: Vec<CreditRegistrationAttentionItem> = credit_registrations::get_attention_items(
171        &mut conn,
172        &stuck_thresholds(),
173        TOO_MANY_ATTEMPTS,
174        ATTENTION_LIMIT,
175    )
176    .await?
177    .into_iter()
178    .map(to_attention_item)
179    .collect();
180
181    let counts_by_reason = ALL_ATTENTION_REASONS
182        .into_iter()
183        .map(|reason| CreditRegistrationAttentionReasonCount {
184            reason,
185            count: items
186                .iter()
187                .filter(|item| item.reasons.contains(&reason))
188                .count() as i64,
189        })
190        .filter(|row| row.count > 0)
191        .collect();
192
193    token.authorized_ok(web::Json(CreditRegistrationAttentionItems {
194        total_count: items.len() as i64,
195        items,
196        counts_by_reason,
197        max_items: ATTENTION_LIMIT,
198    }))
199}
200
201/**
202GET `/api/v0/main-frontend/credit-registration-admin/errors/by-code` - Error events per code over a
203window and the window before it, with the terminal verdicts of the same window beside them.
204
205Counts events, not rows: an error that happened really happened, whether or not a later attempt
206succeeded, and hiding it would hide the configuration bug that caused it.
207*/
208#[instrument(skip(pool))]
209#[utoipa::path(
210    get,
211    path = "/errors/by-code",
212    operation_id = "getCreditRegistrationErrorsByCode",
213    tag = "credit-registration-admin",
214    params(("window_secs" = Option<i64>, Query, description = "Window length in seconds; the same length before it is the comparison")),
215    responses(
216        (status = 200, description = "Per-code counts and the window's verdicts", body = CreditRegistrationErrorsByCode)
217    )
218)]
219pub async fn get_credit_registration_errors_by_code(
220    user: AuthUser,
221    pool: web::Data<PgPool>,
222    query: web::Query<ErrorWindowQuery>,
223) -> ControllerResult<web::Json<CreditRegistrationErrorsByCode>> {
224    let mut conn = pool.acquire().await?;
225    let token = authorize_credit_registration_admin(&mut conn, user.id).await?;
226
227    let window_secs = query
228        .window_secs
229        .unwrap_or(DEFAULT_ERROR_WINDOW_SECS)
230        .clamp(60, MAX_ERROR_WINDOW_SECS);
231    let codes =
232        credit_registration_events::get_error_code_counts_for_window(&mut conn, window_secs)
233            .await?
234            .into_iter()
235            .map(to_error_code_window)
236            .collect();
237    let totals = credit_registrations::count_terminal_outcomes_since(
238        &mut conn,
239        Utc::now() - chrono::Duration::seconds(window_secs),
240    )
241    .await?;
242
243    token.authorized_ok(web::Json(CreditRegistrationErrorsByCode {
244        window_secs,
245        codes,
246        verdicts: CreditRegistrationTerminalVerdicts {
247            registered_count: totals.registered_count,
248            duplicate_and_not_improved_count: totals.success_count - totals.registered_count,
249            failed_permanent_count: totals.failed_permanent_count,
250            cancelled_count: totals.cancelled_count,
251            total_count: totals.total_count,
252        },
253    }))
254}
255
256const ALL_ATTENTION_REASONS: [CreditRegistrationAttentionReason; 7] = [
257    CreditRegistrationAttentionReason::StuckInState,
258    CreditRegistrationAttentionReason::PermanentError,
259    CreditRegistrationAttentionReason::RetryWindowExpired,
260    CreditRegistrationAttentionReason::Misregistered,
261    CreditRegistrationAttentionReason::TooManyAttempts,
262    CreditRegistrationAttentionReason::OutcomeUncertain,
263    CreditRegistrationAttentionReason::FlaggedByPipeline,
264];
265
266fn to_attention_item(row: AttentionRegistration) -> CreditRegistrationAttentionItem {
267    let flags = [
268        (
269            row.stuck_in_state,
270            CreditRegistrationAttentionReason::StuckInState,
271        ),
272        (
273            row.permanent_error,
274            CreditRegistrationAttentionReason::PermanentError,
275        ),
276        (
277            row.retry_window_expired,
278            CreditRegistrationAttentionReason::RetryWindowExpired,
279        ),
280        (
281            row.misregistered,
282            CreditRegistrationAttentionReason::Misregistered,
283        ),
284        (
285            row.too_many_attempts,
286            CreditRegistrationAttentionReason::TooManyAttempts,
287        ),
288        (
289            row.outcome_uncertain,
290            CreditRegistrationAttentionReason::OutcomeUncertain,
291        ),
292        (
293            row.flagged_by_pipeline,
294            CreditRegistrationAttentionReason::FlaggedByPipeline,
295        ),
296    ];
297    CreditRegistrationAttentionItem {
298        reasons: flags
299            .into_iter()
300            .filter_map(|(fired, reason)| fired.then_some(reason))
301            .collect(),
302        credit_registration_id: row.id,
303        user_id: row.user_id,
304        first_name: row.first_name,
305        last_name: row.last_name,
306        email: row.email,
307        course_id: row.course_id,
308        course_name: row.course_name,
309        course_module_id: row.course_module_id,
310        course_module_name: row.course_module_name,
311        state: row.state,
312        state_entered_at: row.state_entered_at,
313        error_code: row.error_code,
314        attempt_count: row.attempt_count,
315        next_attempt_at: row.next_attempt_at,
316        student_number: row.student_number,
317    }
318}
319
320fn to_error_code_window(row: ErrorCodeWindowCounts) -> CreditRegistrationErrorCodeWindow {
321    CreditRegistrationErrorCodeWindow {
322        retryability: retryability(row.error_code),
323        error_code: row.error_code,
324        current_count: row.current_count,
325        previous_count: row.previous_count,
326        user_count: row.user_count,
327        course_count: row.course_count,
328        first_seen_at: row.first_seen_at,
329        last_seen_at: row.last_seen_at,
330        endpoints: row.endpoints,
331    }
332}
333
334pub fn _add_routes(cfg: &mut ServiceConfig) {
335    cfg.route(
336        "/thresholds",
337        web::get().to(get_credit_registration_thresholds),
338    )
339    .route(
340        "/attention",
341        web::get().to(get_credit_registration_attention_items),
342    )
343    .route(
344        "/errors/by-code",
345        web::get().to(get_credit_registration_errors_by_code),
346    );
347}
headless_lms_server/controllers/main_frontend/credit_registration_admin/errors.rs

headless_lms_server/controllers/main_frontend/credit_registration_admin/
errors.rs
