headless_lms_server/controllers/main_frontend/credit_registration_admin/
reconciliation.rs

1//! The Reconciliation tab: the failures defined by an absence, which no error count can catch.
2
3use headless_lms_models::credit_registration_events;
4use headless_lms_models::credit_registrations::{
5    self, AdminCreditRegistration, AdminCreditRegistrationFilters, AdminCreditRegistrationSort,
6    CreditRegistrationState,
7};
8use headless_lms_models::library::credit_registration::legacy_mirror::{
9    self, LegacyLedgerDivergence,
10};
11use headless_lms_models::library::credit_registration::materialize::{
12    UnmaterialisedCompletion, get_unmaterialised_eligible_completions,
13};
14use utoipa::ToSchema;
15
16use crate::prelude::*;
17use headless_lms_utils::secret_string::expose_option;
18
19use super::authorize_credit_registration_admin;
20
21/// Rows per detector. These are heavy queries and every list here is meant to be worked through,
22/// not scrolled.
23const DETECTOR_LIMIT: i64 = 200;
24/// A completion younger than this is simply waiting for the next `materialize` tick.
25const NEVER_ENTERED_MIN_AGE_SECS: i64 = 60 * 60;
26
27/// A completion that satisfies the materialise predicate and has no ledger row.
28#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
29pub struct NeverEnteredCompletion {
30    pub course_module_completion_id: Uuid,
31    pub user_id: Uuid,
32    pub first_name: Option<String>,
33    pub last_name: Option<String>,
34    pub email: Option<String>,
35    pub course_id: Uuid,
36    pub course_name: String,
37    pub course_module_id: Uuid,
38    pub course_module_name: Option<String>,
39    pub completion_date: DateTime<Utc>,
40    pub created_at: DateTime<Utc>,
41    /// The student has no enrolment on the course, so `materialize` has no course instance to put
42    /// on a ledger row. The one cause running the phase again will not fix.
43    pub missing_enrolment: bool,
44}
45
46/// One ledger row, flattened to what every reconciliation list needs to name a student and link
47/// onwards.
48#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
49pub struct ReconciliationRegistration {
50    pub credit_registration_id: Uuid,
51    pub user_id: Uuid,
52    pub first_name: Option<String>,
53    pub last_name: Option<String>,
54    pub email: Option<String>,
55    pub student_number: Option<String>,
56    pub course_id: Uuid,
57    pub course_name: String,
58    pub course_module_id: Uuid,
59    pub course_module_name: Option<String>,
60    pub uh_course_code: Option<String>,
61    pub state: CreditRegistrationState,
62    pub state_entered_at: DateTime<Utc>,
63    pub submitted_at: Option<DateTime<Utc>>,
64    pub submitted_attainment_id: Option<String>,
65    pub sisu_attainment_id: Option<String>,
66    pub registered_at: Option<DateTime<Utc>>,
67    pub terminal_at: Option<DateTime<Utc>>,
68}
69
70/// A ledger row the legacy study-registry ledger contradicts.
71#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
72pub struct LegacyLedgerDivergenceRow {
73    pub credit_registration_id: Uuid,
74    pub course_module_completion_id: Uuid,
75    pub user_id: Uuid,
76    pub first_name: Option<String>,
77    pub last_name: Option<String>,
78    pub email: Option<String>,
79    pub course_id: Uuid,
80    pub course_name: String,
81    pub course_module_id: Uuid,
82    pub state: CreditRegistrationState,
83    pub state_entered_at: DateTime<Utc>,
84    /// We registered it and the legacy ledger has no row of ours, so the teacher views and the pull
85    /// stream still call the completion unregistered.
86    pub mirror_missing: bool,
87    /// A registrar took the completion through the pull path while our pipeline had not finished
88    /// with it: the shape a double registration would have.
89    pub registered_by_a_registrar: bool,
90}
91
92#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, ToSchema)]
93pub struct CreditRegistrationReconciliation {
94    /// Eligible completions with no ledger row at all.
95    pub never_entered: Vec<NeverEnteredCompletion>,
96    /// `submission_uncertain`: the import may or may not have landed. Verify these, never resubmit.
97    pub outcome_uncertain: Vec<ReconciliationRegistration>,
98    /// Rows whose answers named more than one submitted attainment id, which is what a double
99    /// submission would look like.
100    pub several_submitted_attainments: Vec<ReconciliationRegistration>,
101    /// Attainments the study registry reversed after we had recorded them as registered.
102    pub misregistered: Vec<ReconciliationRegistration>,
103    pub legacy_divergences: Vec<LegacyLedgerDivergenceRow>,
104    /// The detectors' counts, in the same order as the lists, capped at `max_rows_per_detector`.
105    pub never_entered_count: i64,
106    pub outcome_uncertain_count: i64,
107    pub several_submitted_attainments_count: i64,
108    pub misregistered_count: i64,
109    pub legacy_divergence_count: i64,
110    /// The four detector counts, which is the tab badge.
111    pub finding_count: i64,
112    pub max_rows_per_detector: i64,
113}
114
115/**
116GET `/api/v0/main-frontend/credit-registration-admin/reconciliation` - The drift detectors: work the
117ledger should be doing and is not, and outcomes the study registry and the ledger disagree about.
118*/
119#[instrument(skip(pool))]
120#[utoipa::path(
121    get,
122    path = "/reconciliation",
123    operation_id = "getCreditRegistrationReconciliation",
124    tag = "credit-registration-admin",
125    responses(
126        (status = 200, description = "Every detector's findings and counts", body = CreditRegistrationReconciliation)
127    )
128)]
129pub async fn get_credit_registration_reconciliation(
130    user: AuthUser,
131    pool: web::Data<PgPool>,
132) -> ControllerResult<web::Json<CreditRegistrationReconciliation>> {
133    let mut conn = pool.acquire().await?;
134    let token = authorize_credit_registration_admin(&mut conn, user.id).await?;
135
136    let never_entered: Vec<NeverEnteredCompletion> = get_unmaterialised_eligible_completions(
137        &mut conn,
138        NEVER_ENTERED_MIN_AGE_SECS,
139        DETECTOR_LIMIT,
140    )
141    .await?
142    .into_iter()
143    .map(to_never_entered)
144    .collect();
145
146    let (outcome_uncertain, misregistered) = rows_in_states(&mut conn).await?;
147
148    let several_ids = credit_registration_events::get_ids_with_several_submitted_attainments(
149        &mut conn,
150        DETECTOR_LIMIT,
151    )
152    .await?;
153    let several_submitted_attainments =
154        rows_by_ids(&mut conn, &several_ids, DETECTOR_LIMIT).await?;
155
156    let legacy_divergences: Vec<LegacyLedgerDivergenceRow> =
157        legacy_mirror::get_legacy_ledger_divergences(&mut conn, DETECTOR_LIMIT)
158            .await?
159            .into_iter()
160            .map(to_legacy_divergence)
161            .collect();
162
163    let never_entered_count = never_entered.len() as i64;
164    let outcome_uncertain_count = outcome_uncertain.len() as i64;
165    let several_submitted_attainments_count = several_submitted_attainments.len() as i64;
166    let misregistered_count = misregistered.len() as i64;
167    let legacy_divergence_count = legacy_divergences.len() as i64;
168
169    token.authorized_ok(web::Json(CreditRegistrationReconciliation {
170        finding_count: never_entered_count
171            + outcome_uncertain_count
172            + several_submitted_attainments_count
173            + misregistered_count
174            + legacy_divergence_count,
175        never_entered,
176        outcome_uncertain,
177        several_submitted_attainments,
178        misregistered,
179        legacy_divergences,
180        never_entered_count,
181        outcome_uncertain_count,
182        several_submitted_attainments_count,
183        misregistered_count,
184        legacy_divergence_count,
185        max_rows_per_detector: DETECTOR_LIMIT,
186    }))
187}
188
189/// The two detectors that each just read one live state, in one `IN`-list query and one
190/// admin-projection lookup shared across both.
191async fn rows_in_states(
192    conn: &mut PgConnection,
193) -> Result<
194    (
195        Vec<ReconciliationRegistration>,
196        Vec<ReconciliationRegistration>,
197    ),
198    ControllerError,
199> {
200    let states = [
201        CreditRegistrationState::SubmissionUncertain,
202        CreditRegistrationState::Misregistered,
203    ];
204    let ids: Vec<Uuid> = credit_registrations::get_live_by_states(conn, &states, DETECTOR_LIMIT)
205        .await?
206        .into_iter()
207        .map(|row| row.id)
208        .collect();
209    let limit = i64::try_from(ids.len()).unwrap_or(i64::MAX);
210    let rows = rows_by_ids(conn, &ids, limit).await?;
211    // Re-read after the id query, so a row that moved on in between belongs to neither detector.
212    let mut submission_uncertain = Vec::new();
213    let mut misregistered = Vec::new();
214    for row in rows {
215        match row.state {
216            CreditRegistrationState::SubmissionUncertain => submission_uncertain.push(row),
217            CreditRegistrationState::Misregistered => misregistered.push(row),
218            _ => {}
219        }
220    }
221    Ok((submission_uncertain, misregistered))
222}
223
224/// Reads the same admin projection the explorer uses, so a name, a course and a student number are
225/// spelled identically wherever the dashboard shows them.
226async fn rows_by_ids(
227    conn: &mut PgConnection,
228    ids: &[Uuid],
229    limit: i64,
230) -> Result<Vec<ReconciliationRegistration>, ControllerError> {
231    if ids.is_empty() {
232        return Ok(Vec::new());
233    }
234    Ok(credit_registrations::get_admin_facing(
235        conn,
236        &AdminCreditRegistrationFilters {
237            credit_registration_ids: Some(ids),
238            include_superseded: true,
239            ..AdminCreditRegistrationFilters::default()
240        },
241        AdminCreditRegistrationSort::TimeInState,
242        limit,
243        0,
244    )
245    .await?
246    .into_iter()
247    .map(to_reconciliation_row)
248    .collect())
249}
250
251fn to_never_entered(row: UnmaterialisedCompletion) -> NeverEnteredCompletion {
252    NeverEnteredCompletion {
253        course_module_completion_id: row.course_module_completion_id,
254        user_id: row.user_id,
255        first_name: row.first_name,
256        last_name: row.last_name,
257        email: row.email,
258        course_id: row.course_id,
259        course_name: row.course_name,
260        course_module_id: row.course_module_id,
261        course_module_name: row.course_module_name,
262        completion_date: row.completion_date,
263        created_at: row.created_at,
264        missing_enrolment: row.missing_enrolment,
265    }
266}
267
268fn to_reconciliation_row(row: AdminCreditRegistration) -> ReconciliationRegistration {
269    ReconciliationRegistration {
270        credit_registration_id: row.id,
271        user_id: row.user_id,
272        first_name: row.first_name,
273        last_name: row.last_name,
274        email: row.email,
275        student_number: expose_option(&row.student_number).map(str::to_owned),
276        course_id: row.course_id,
277        course_name: row.course_name,
278        course_module_id: row.course_module_id,
279        course_module_name: row.course_module_name,
280        uh_course_code: row.uh_course_code,
281        state: row.state,
282        state_entered_at: row.state_entered_at,
283        submitted_at: row.submitted_at,
284        submitted_attainment_id: row.submitted_attainment_id,
285        sisu_attainment_id: row.sisu_attainment_id,
286        registered_at: row.registered_at,
287        terminal_at: row.terminal_at,
288    }
289}
290
291fn to_legacy_divergence(row: LegacyLedgerDivergence) -> LegacyLedgerDivergenceRow {
292    LegacyLedgerDivergenceRow {
293        credit_registration_id: row.credit_registration_id,
294        course_module_completion_id: row.course_module_completion_id,
295        user_id: row.user_id,
296        first_name: row.first_name,
297        last_name: row.last_name,
298        email: row.email,
299        course_id: row.course_id,
300        course_name: row.course_name,
301        course_module_id: row.course_module_id,
302        state: row.state,
303        state_entered_at: row.state_entered_at,
304        mirror_missing: row.mirror_missing,
305        registered_by_a_registrar: row.registered_by_a_registrar,
306    }
307}
308
309pub fn _add_routes(cfg: &mut ServiceConfig) {
310    cfg.route(
311        "/reconciliation",
312        web::get().to(get_credit_registration_reconciliation),
313    );
314}
headless_lms_server/controllers/main_frontend/credit_registration_admin/reconciliation.rs

headless_lms_server/controllers/main_frontend/credit_registration_admin/
reconciliation.rs
