headless_lms_chatbot/
feedback_categorization.rs

1use crate::{
2    azure_chatbot::azure::protocol::{
3        InputItem, LLMRequestParams, LLMRequestResponseFormatParam, NonThinkingParams, Reasoning,
4        ThinkingParams,
5    },
6    chatbot_error::chatbot_err,
7    llm_utils::{APIInputMessage, MessageContent, model_is_thinking, request_structured_json},
8    prelude::*,
9};
10use headless_lms_base::config::ApplicationConfiguration;
11use headless_lms_base::error::backend_error::BackendError;
12use headless_lms_models::{
13    application_task_default_language_models::TaskLMSpec,
14    chatbot_configurations::ReasoningEffortLevel,
15    chatbot_conversation_message_messages::MessageRole, feedback::NewFeedback,
16    feedback_categories::FeedbackCategory,
17};
18use headless_lms_utils::json_schema_types::{JSONType, JsonItem, Schema, SchemaPropertyType};
19use indexmap::IndexMap;
20
21/// Shape of the structured LLM output response, defined by the JSONSchema in
22/// [response_format]
23#[derive(serde::Deserialize)]
24pub struct FeedbackCategorisationResponse {
25    pub category_name: String,
26}
27
28// Maximum length in chars
29pub const MAX_CATEGORY_NAME_LEN: usize = 255;
30
31/// Names this feature's structured output to Azure. The test-mode mock Azure API picks its canned
32/// answer for this feature by this name.
33pub const RESPONSE_FORMAT_NAME: &str = "FeedbackCategorisationResponse";
34
35/// The structured output format the suggestion LLM is asked to answer in. Must stay in
36/// sync with [FeedbackCategorisationResponse].
37fn response_format() -> LLMRequestResponseFormatParam {
38    LLMRequestResponseFormatParam::JsonSchema {
39        name: "FeedbackCategorisationResponse".to_string(),
40        schema: Schema::strict_object(
41            IndexMap::from([(
42                "category_name".to_string(),
43                SchemaPropertyType::Item(JsonItem {
44                    type_field: JSONType::String,
45                    description: None,
46                }),
47            )]),
48            None,
49        ),
50        strict: true,
51    }
52}
53
54fn format_category_list(categories: &[FeedbackCategory]) -> String {
55    let c = categories
56        .iter()
57        .map(|c| c.name.to_owned())
58        .collect::<Vec<String>>()
59        .join(",");
60    format!("[{c}]")
61}
62
63fn format_feedback(feedback: &NewFeedback) -> String {
64    let start = "\n\nThe feedback to categorize: \n<START>\n".to_string();
65    let end = "\n<END>";
66    let selected_text = feedback.selected_text.as_ref().map_or("".to_string(), |s| {
67        format!(
68            "\n\nAssociated course material text: {}\n",
69            s.replace("<END>", "").replace("<START>", "")
70        )
71    });
72
73    start
74        + &feedback
75            .feedback_given
76            .replace("<END>", "")
77            .replace("<START>", "")
78        + &selected_text
79        + end
80}
81
82/// System prompt instructions for generating suggested next messages
83const SYSTEM_PROMPT: &str = r#"You are an expert text categorisation system. You are given written feedback submitted by a student who is completing a course, and a set of possible categories. Analyse the feedback and label it with one of the categories. If none of the categories fit, create a new category.
84
85Allowed response types:
86- Assign feedback into an existing category
87- Create a new category and assign the feedback to it
88
89When assigning to an existing category, return the category's name exactly as it is. The existing category names are given below. When creating a new category, return the proposed new category name. In both cases, only the category name should be returned: whether it is listed below or not indicates is it an existing category or a new one.
90
91Constraints:
92- analyse the meaning and context of the feedback
93- don't focus on specific details too much
94- understand the intention behind the feedback and what problem it is really aiming to convey
95- compare the feedback to the existing categories: could it fit into one of them?
96- be conservative when adding new cateogries. If the feedback could be assigned to an existing one, do so instead of creating a new category for it.
97
98New category constraints:
99- the name should be as short and concise as possible
100- the name should be only adequately descriptive: based on it, one should be able to understand what kind of feedback the category contains
101- the name must not be overly specific: it should give a reasonable clue about the category's content, but not describe it with detail
102- the name should describe the general type of feedback in that category and not be based on any specific details in the feedback. For example, instead of "Confusion about the topic in chapter 4" try "Material comprehension difficulties"
103- the name should fit in with the existing categories, i.e. follow the same naming logic
104- the name should use normal, correct language
105
106Currently existing categories:
107
108"#;
109
110// todo: truncate long category names
111pub async fn categorize_feedback(
112    app_config: &ApplicationConfiguration,
113    task_lm: &TaskLMSpec,
114    feedback: &NewFeedback,
115    categories: &[FeedbackCategory],
116) -> ChatbotResult<FeedbackCategorisationResponse> {
117    let prompt = SYSTEM_PROMPT.to_string() + &format_category_list(categories);
118    let input = vec![
119        APIInputMessage {
120            message_type: InputItem::Message {
121                role: MessageRole::System,
122                content: MessageContent::Text(prompt),
123            },
124        },
125        APIInputMessage {
126            message_type: InputItem::Message {
127                role: MessageRole::User,
128                content: MessageContent::Text(format_feedback(feedback)),
129            },
130        },
131    ];
132    let (params, max_output_tokens) = if model_is_thinking(task_lm.model_type) {
133        (
134            LLMRequestParams::GPTThinking(ThinkingParams {
135                reasoning: Some(Reasoning {
136                    effort: ReasoningEffortLevel::Low,
137                    summary: None,
138                    context: None,
139                }),
140            }),
141            Some(7000),
142        )
143    } else {
144        (
145            LLMRequestParams::GPTNonThinking(NonThinkingParams {
146                temperature: None,
147                top_p: None,
148                frequency_penalty: None,
149                presence_penalty: None,
150            }),
151            Some(4000),
152        )
153    };
154    let res: FeedbackCategorisationResponse = request_structured_json(
155        input,
156        task_lm.model.to_owned(),
157        params,
158        max_output_tokens,
159        response_format(),
160        app_config,
161        || {
162            chatbot_err!(
163                FailedAzureResponse,
164                "Invalidly structured response from Azure"
165            )
166        },
167    )
168    .await?;
169
170    // truncate if too long
171    if res.category_name.chars().count() > MAX_CATEGORY_NAME_LEN {
172        let category_name = res
173            .category_name
174            .chars()
175            .take(MAX_CATEGORY_NAME_LEN)
176            .collect::<String>();
177        Ok(FeedbackCategorisationResponse { category_name })
178    } else {
179        Ok(res)
180    }
181}
182
183#[cfg(test)]
184mod tests {
185    use super::*;
186
187    /// The test-mode mock Azure API picks this feature's canned answer by the format name, so the
188    /// name is pinned even though the shape it wraps is shared.
189    #[test]
190    fn the_response_format_is_named_after_this_feature() {
191        let serialized =
192            serde_json::to_value(response_format()).expect("The response format serializes");
193        assert_eq!(
194            serialized["name"],
195            serde_json::json!("FeedbackCategorisationResponse")
196        );
197    }
198}
headless_lms_chatbot/feedback_categorization.rs

headless_lms_chatbot/
feedback_categorization.rs
