headless_lms_server/controllers/main_frontend/credit_registration_admin/
errors.rs

1//! The Errors & stuck tab: what is going wrong by error code, and which rows want a human.
2
3use headless_lms_models::credit_registration_events::{self, ErrorCodeWindowCounts};
4use headless_lms_models::credit_registrations::{
5    self, AttentionReason, AttentionRegistration, AttentionSort, CreditRegistrationErrorCode,
6    CreditRegistrationState, HandActionAvailability, ResubmissionStrictness, StuckThresholds,
7};
8use headless_lms_models::library::credit_registration::classification::{
9    Retryability, retryability,
10};
11use headless_lms_models::suotar_api_calls::SuotarEndpoint;
12use utoipa::ToSchema;
13
14use crate::domain::credit_registration::health::stuck_thresholds;
15use crate::prelude::*;
16use headless_lms_utils::secret_string::expose_option;
17
18use super::{ATTENTION_TOO_MANY_ATTEMPTS, authorize_credit_registration_admin};
19
20/// Rows per page of the attention queue when the caller names no limit.
21const ATTENTION_PAGE_SIZE: u32 = 50;
22const DEFAULT_ERROR_WINDOW_SECS: i64 = 24 * 60 * 60;
23const MAX_ERROR_WINDOW_SECS: i64 = 90 * 24 * 60 * 60;
24
25#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
26pub struct CreditRegistrationAttentionItem {
27    pub credit_registration_id: Uuid,
28    pub user_id: Uuid,
29    pub first_name: Option<String>,
30    pub last_name: Option<String>,
31    /// In full: this is the list support works from.
32    pub email: Option<String>,
33    pub course_id: Uuid,
34    pub course_name: String,
35    pub course_module_id: Uuid,
36    pub course_module_name: Option<String>,
37    pub state: CreditRegistrationState,
38    pub state_entered_at: DateTime<Utc>,
39    pub error_code: Option<CreditRegistrationErrorCode>,
40    pub attempt_count: i32,
41    pub next_attempt_at: DateTime<Utc>,
42    pub student_number: Option<String>,
43    /// Every detector that picked this row, so the table can group by any of them. Empty on a row
44    /// the pipeline flagged that no detector explains.
45    pub reasons: Vec<AttentionReason>,
46    /// The pipeline's cached "a human should look at this". A fact about the row, never a reason:
47    /// it says nothing about why, so it travels beside `reasons` rather than in them.
48    pub needs_admin_attention: bool,
49    /// What the bulk hand transition would allow on this row.
50    pub hand_actions: HandActionAvailability,
51}
52
53#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
54pub struct CreditRegistrationAttentionReasonCount {
55    pub reason: AttentionReason,
56    pub count: i64,
57}
58
59#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
60pub struct CreditRegistrationAttentionItems {
61    /// The requested page of the queue.
62    pub items: Vec<CreditRegistrationAttentionItem>,
63    /// The whole queue, whatever this request filtered to: the canonical "needs a human" count, the
64    /// same number `/overview` reports and the tab badge shows.
65    pub total_count: i64,
66    /// Rows matching this request's narrowing, which is what `total_pages` pages through. Equal to
67    /// `total_count` when neither `reason` nor `without_reason` was given.
68    pub filtered_count: i64,
69    pub total_pages: u32,
70    /// Over the whole queue, not over the page or the filter, so the counts stay usable as facets.
71    pub counts_by_reason: Vec<CreditRegistrationAttentionReasonCount>,
72    /// Queue rows no detector picked, which the pipeline's flag alone put there. No `reason`
73    /// reaches them, so a surface that groups by reason has to offer `without_reason` beside the
74    /// reasons or leave this many rows unreachable.
75    pub flagged_without_reason_count: i64,
76}
77
78/// One error code over the chosen window and the one before it.
79#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
80pub struct CreditRegistrationErrorCodeWindow {
81    pub error_code: CreditRegistrationErrorCode,
82    /// What may be done about the code, which is the difference between a wait and a fix.
83    pub retryability: Retryability,
84    pub current_count: i64,
85    pub previous_count: i64,
86    pub user_count: i64,
87    pub course_count: i64,
88    pub first_seen_at: Option<DateTime<Utc>>,
89    pub last_seen_at: Option<DateTime<Utc>>,
90    pub endpoints: Vec<SuotarEndpoint>,
91}
92
93/// The verdicts an operator needs beside the errors to rule them out. `not_improved` is not a
94/// failure and is never in the error table above.
95#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
96pub struct CreditRegistrationTerminalVerdicts {
97    pub registered_count: i64,
98    pub duplicate_and_not_improved_count: i64,
99    pub failed_permanent_count: i64,
100    pub cancelled_count: i64,
101    /// The denominator of the success rate.
102    pub total_count: i64,
103}
104
105#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
106pub struct CreditRegistrationErrorsByCode {
107    pub window_secs: i64,
108    pub codes: Vec<CreditRegistrationErrorCodeWindow>,
109    pub verdicts: CreditRegistrationTerminalVerdicts,
110}
111
112#[derive(Debug, Deserialize)]
113pub struct ErrorWindowQuery {
114    window_secs: Option<i64>,
115}
116
117/**
118GET `/api/v0/main-frontend/credit-registration-admin/thresholds` - Every number the alert rules and
119the stuck detectors use.
120
121The same values `/overview` embeds in its health block. Separate so a tab explaining "stuck after
1222 hours" can say so without reading the whole overview aggregate.
123*/
124#[instrument(skip(pool))]
125#[utoipa::path(
126    get,
127    path = "/thresholds",
128    operation_id = "getCreditRegistrationThresholds",
129    tag = "credit-registration-admin",
130    responses(
131        (status = 200, description = "The thresholds every rule and detector shares", body = StuckThresholds)
132    )
133)]
134pub async fn get_credit_registration_thresholds(
135    user: AuthUser,
136    pool: web::Data<PgPool>,
137) -> ControllerResult<web::Json<StuckThresholds>> {
138    let mut conn = pool.acquire().await?;
139    let token = authorize_credit_registration_admin(&mut conn, user.id).await?;
140    token.authorized_ok(web::Json(stuck_thresholds()))
141}
142
143#[derive(Debug, Deserialize)]
144pub struct AttentionQuery {
145    page: Option<u32>,
146    limit: Option<u32>,
147    reason: Option<Vec<AttentionReason>>,
148    /// Narrows to the rows `flagged_without_reason_count` counts. Given together with `reason` it
149    /// selects nothing: no row both carries a reason and lacks one.
150    without_reason: Option<bool>,
151    sort: Option<String>,
152}
153
154/**
155GET `/api/v0/main-frontend/credit-registration-admin/attention` - A page of the rows at least one
156detector wants a human to look at, with the detectors that picked each.
157
158Superseded attempts are outside every detector: acting on a replaced attempt is never right.
159`total_count` is the queue's length under the one definition of "needs a human"; `/overview`'s
160`needs_admin_attention_count` is the same number.
161*/
162#[instrument(skip(pool))]
163#[utoipa::path(
164    get,
165    path = "/attention",
166    operation_id = "getCreditRegistrationAttentionItems",
167    tag = "credit-registration-admin",
168    params(
169        ("page" = Option<u32>, Query, description = "Page number, from 1"),
170        ("limit" = Option<u32>, Query, description = "Rows per page"),
171        ("reason" = Option<Vec<AttentionReason>>, Query, description = "Only rows one of these detectors picked; repeat the parameter for several"),
172        ("without_reason" = Option<bool>, Query, description = "Only rows no detector picked, which the pipeline's flag alone put in the queue; selects nothing alongside reason"),
173        ("sort" = Option<String>, Query, description = "time_in_state, next_attempt or course")
174    ),
175    responses(
176        (status = 200, description = "A page of the rows needing a human, and how many for each reason", body = CreditRegistrationAttentionItems)
177    )
178)]
179pub async fn get_credit_registration_attention_items(
180    user: AuthUser,
181    pool: web::Data<PgPool>,
182    query: MultiQuery<AttentionQuery>,
183) -> ControllerResult<web::Json<CreditRegistrationAttentionItems>> {
184    let mut conn = pool.acquire().await?;
185    let token = authorize_credit_registration_admin(&mut conn, user.id).await?;
186
187    let pagination = parse_pagination(query.page, query.limit, ATTENTION_PAGE_SIZE)?;
188    let reasons: &[AttentionReason] = query.reason.as_deref().unwrap_or_default();
189    let only_without_reason = query.without_reason.unwrap_or(false);
190    let sort = match query.sort.as_deref() {
191        Some("next_attempt") => AttentionSort::NextAttempt,
192        Some("course") => AttentionSort::Course,
193        _ => AttentionSort::TimeInState,
194    };
195    let thresholds = stuck_thresholds();
196
197    let rows = credit_registrations::get_attention_items(
198        &mut conn,
199        &thresholds,
200        ATTENTION_TOO_MANY_ATTEMPTS,
201        credit_registrations::AttentionSelection {
202            reasons,
203            only_without_reason,
204            sort,
205            limit: pagination.limit(),
206            offset: pagination.offset(),
207        },
208    )
209    .await?;
210    let filtered_count = rows.first().map_or(0, |row| row.total_count);
211    // An unfiltered page already carries the whole queue's totals; only a narrowed or empty one
212    // needs them asked for separately.
213    let queue = match rows.first() {
214        Some(row) if reasons.is_empty() && !only_without_reason => Some(row.clone()),
215        _ => {
216            credit_registrations::count_needing_attention(
217                &mut conn,
218                &thresholds,
219                ATTENTION_TOO_MANY_ATTEMPTS,
220            )
221            .await?
222        }
223    };
224    let counts_by_reason = queue
225        .as_ref()
226        .map(AttentionRegistration::counts_by_reason)
227        .unwrap_or_default()
228        .into_iter()
229        .filter(|(_, count)| *count > 0)
230        .map(|(reason, count)| CreditRegistrationAttentionReasonCount { reason, count })
231        .collect();
232
233    token.authorized_ok(web::Json(CreditRegistrationAttentionItems {
234        items: rows.into_iter().map(to_attention_item).collect(),
235        total_count: queue.as_ref().map_or(0, |row| row.total_count),
236        filtered_count,
237        total_pages: pagination.total_pages(u32::try_from(filtered_count).unwrap_or(u32::MAX)),
238        counts_by_reason,
239        flagged_without_reason_count: queue
240            .as_ref()
241            .map_or(0, |row| row.flagged_without_reason_count),
242    }))
243}
244
245/**
246GET `/api/v0/main-frontend/credit-registration-admin/errors/by-code` - Error events per code over a
247window and the window before it, with the terminal verdicts of the same window beside them.
248
249Counts events, not rows: an error that happened really happened, whether or not a later attempt
250succeeded, and hiding it would hide the configuration bug that caused it.
251*/
252#[instrument(skip(pool))]
253#[utoipa::path(
254    get,
255    path = "/errors/by-code",
256    operation_id = "getCreditRegistrationErrorsByCode",
257    tag = "credit-registration-admin",
258    params(("window_secs" = Option<i64>, Query, description = "Window length in seconds; the same length before it is the comparison")),
259    responses(
260        (status = 200, description = "Per-code counts and the window's verdicts", body = CreditRegistrationErrorsByCode)
261    )
262)]
263pub async fn get_credit_registration_errors_by_code(
264    user: AuthUser,
265    pool: web::Data<PgPool>,
266    query: web::Query<ErrorWindowQuery>,
267) -> ControllerResult<web::Json<CreditRegistrationErrorsByCode>> {
268    let mut conn = pool.acquire().await?;
269    let token = authorize_credit_registration_admin(&mut conn, user.id).await?;
270
271    let window_secs = query
272        .window_secs
273        .unwrap_or(DEFAULT_ERROR_WINDOW_SECS)
274        .clamp(60, MAX_ERROR_WINDOW_SECS);
275    let codes =
276        credit_registration_events::get_error_code_counts_for_window(&mut conn, window_secs)
277            .await?
278            .into_iter()
279            .map(to_error_code_window)
280            .collect();
281    let totals = credit_registrations::count_terminal_outcomes_since(
282        &mut conn,
283        Utc::now() - chrono::Duration::seconds(window_secs),
284    )
285    .await?;
286
287    token.authorized_ok(web::Json(CreditRegistrationErrorsByCode {
288        window_secs,
289        codes,
290        verdicts: CreditRegistrationTerminalVerdicts {
291            registered_count: totals.registered_count,
292            duplicate_and_not_improved_count: totals.success_count - totals.registered_count,
293            failed_permanent_count: totals.failed_permanent_count,
294            cancelled_count: totals.cancelled_count,
295            total_count: totals.total_count,
296        },
297    }))
298}
299
300fn to_attention_item(row: AttentionRegistration) -> CreditRegistrationAttentionItem {
301    CreditRegistrationAttentionItem {
302        reasons: row.reasons(),
303        hand_actions: row
304            .resubmission_facts()
305            .hand_actions(ResubmissionStrictness::AnyExceptSubmissionUncertain),
306        credit_registration_id: row.id,
307        user_id: row.user_id,
308        first_name: row.first_name,
309        last_name: row.last_name,
310        email: row.email,
311        course_id: row.course_id,
312        course_name: row.course_name,
313        course_module_id: row.course_module_id,
314        course_module_name: row.course_module_name,
315        state: row.state,
316        state_entered_at: row.state_entered_at,
317        error_code: row.error_code,
318        attempt_count: row.attempt_count,
319        next_attempt_at: row.next_attempt_at,
320        student_number: expose_option(&row.student_number).map(str::to_owned),
321        needs_admin_attention: row.needs_admin_attention,
322    }
323}
324
325fn to_error_code_window(row: ErrorCodeWindowCounts) -> CreditRegistrationErrorCodeWindow {
326    CreditRegistrationErrorCodeWindow {
327        retryability: retryability(row.error_code),
328        error_code: row.error_code,
329        current_count: row.current_count,
330        previous_count: row.previous_count,
331        user_count: row.user_count,
332        course_count: row.course_count,
333        first_seen_at: row.first_seen_at,
334        last_seen_at: row.last_seen_at,
335        endpoints: row.endpoints,
336    }
337}
338
339pub fn _add_routes(cfg: &mut ServiceConfig) {
340    cfg.route(
341        "/thresholds",
342        web::get().to(get_credit_registration_thresholds),
343    )
344    .route(
345        "/attention",
346        web::get().to(get_credit_registration_attention_items),
347    )
348    .route(
349        "/errors/by-code",
350        web::get().to(get_credit_registration_errors_by_code),
351    );
352}
headless_lms_server/controllers/main_frontend/credit_registration_admin/errors.rs

headless_lms_server/controllers/main_frontend/credit_registration_admin/
errors.rs
