headless_lms_chatbot/chatbot_tools/custom_tools/

course_progress.rs

use crate::{
    azure_chatbot::azure::tools::{AzureLLMFunctionToolDefinition, LLMToolType},
    chatbot_tools::{
        ChatbotTool, ChatbotToolDeclaration, ToolProperties, no_parameters,
        tool_authorization::ToolRequirement,
    },
    prelude::*,
    user_context::ChatbotTurnContext,
};
use headless_lms_models::chatbot_configurations::ToolCategory;
use headless_lms_models::user_exercise_states::UserCourseProgress;
use headless_lms_utils::cache::Cache;

pub type CourseProgressTool = ToolProperties<CourseProgressState>;

impl ChatbotToolDeclaration for CourseProgressTool {
    const NAME: &'static str = "course_progress";

    fn offer_requirements(_user_context: &ChatbotTurnContext) -> Vec<ToolRequirement> {
        Vec::new()
    }

    const CATEGORY: ToolCategory = ToolCategory::CourseInfo;

    fn get_tool_definition() -> AzureLLMFunctionToolDefinition {
        AzureLLMFunctionToolDefinition {
            tool_type: LLMToolType::Function,
            name: Self::NAME.to_string(),
            description: "Get the user's progress on this course, including information about exercises attempted, points gained, the passing criteria for the course and if the user meets the criteria.".to_string(),
            parameters: no_parameters(),
            strict: true
        }
    }
}

impl ChatbotTool for CourseProgressTool {
    type Arguments = CourseProgressArguments;

    fn call_requirements(
        _arguments: &Self::Arguments,
        _user_context: &ChatbotTurnContext,
    ) -> Vec<ToolRequirement> {
        Vec::new()
    }

    /// The LLM calls this tool without arguments, so whatever it emitted is ignored rather than
    /// deserialized: an empty argument string is not valid JSON.
    fn parse_arguments(_args_string: String) -> ChatbotResult<Self::Arguments> {
        Ok(CourseProgressArguments {})
    }

    /// Create a CourseProgressTool instance
    async fn from_db_and_arguments(
        conn: &mut PgConnection,
        _app_config: &ApplicationConfiguration,
        _cache: &Cache,
        _arguments: Self::Arguments,
        user_context: &ChatbotTurnContext,
    ) -> ChatbotResult<Self> {
        let Some(user_id) = user_context.user_id else {
            return Err(chatbot_err!(
                ToolUseError,
                "User id is missing.".to_string()
            ));
        };
        let Some(course_id) = user_context.course_id else {
            return Err(chatbot_err!(
                ToolUseError,
                "Course id is missing.".to_string()
            ));
        };
        let Some(course_name) = &user_context.course_name else {
            return Err(chatbot_err!(
                ToolUseError,
                "Course name is missing.".to_string()
            ));
        };
        let user_progress = headless_lms_models::user_exercise_states::get_user_course_progress(
            conn, course_id, user_id, true,
        )
        .await?;
        Result::Ok(CourseProgressTool {
            state: CourseProgressState {
                course_name: course_name.clone(),
                progress: user_progress,
            },
        })
    }

    /// Return a string explaining the user's progress on the course that the chatbot is on
    fn output(&self) -> String {
        let mut progress = self.state.progress.to_owned();
        let course_name = &self.state.course_name;
        let mut res = format!("The user is completing a course called {course_name}. ");

        // If `progress` has one value, then this course has only one (default) module
        if progress.len() == 1 {
            let module_progress = &progress[0];

            res += "Their progress on this course is the following:";

            res += &push_exercises_scores_progress(module_progress, "course");
        } else {
            // If there are multiple modules in this course, then each module has its
            // own progress
            progress.sort_by_key(|m| m.course_module_order_number);
            let first_mod = progress.first();

            // the first in the sorted list is the base module
            let s = if let Some(module_progress) = first_mod {
                let m_name = &module_progress.course_module_name;
                format!(
                    "The course has one base module, and additional modules. The user's progress on the base course module called {m_name} is the following:"
                ) + &push_exercises_scores_progress(module_progress, "module")
                    + "To pass the course, it's required to pass the base module. The following modules are additional to the course and to complete them, it's required to first complete the base module.\n"
            } else {
                // If the `progress` vec is empty, then:
                "There is no progress information for this user on this course. ".to_string()
            };
            res += &s;

            // skip first because we processed it earlier and add the progress for
            // each module
            for module_progress in progress.iter().skip(1) {
                let m_name = &module_progress.course_module_name;
                res.push_str(&format!(
                    "The user's progress on the course module called {m_name} is the following:"
                ));
                res += &push_exercises_scores_progress(module_progress, "module");
            }
        }
        res
    }

    fn output_description_instructions(&self) -> Option<String> {
        if self.state.progress.len() > 1 {
            Some(
            "Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. If the user asks something like 'how to pass the course', describe the passing criteria and requirements of the base module. Encourage the user to ask further questions about other modules if needed.".to_string(),
        )
        } else {
            Some(
            "Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. Encourage the user to ask further questions if needed.".to_string(),
        )
        }
    }
}

#[derive(Deserialize)]
pub struct CourseProgressArguments {}

pub struct CourseProgressState {
    course_name: String,
    progress: Vec<UserCourseProgress>,
}

fn push_exercises_scores_progress(
    module_progress: &UserCourseProgress,
    course_or_module: &str,
) -> String {
    let automatic_completion = module_progress.automatic_completion;
    let requires_exam = module_progress.requires_exam;
    let attempted_exercises = module_progress.attempted_exercises;
    let total_exercises = module_progress.total_exercises;
    let attempted_exercises_required = module_progress.attempted_exercises_required;
    let score_given = module_progress.score_given;
    let score_maximum = module_progress.score_maximum;
    let score_required = module_progress.score_required;

    let mut res = "".to_string();
    if total_exercises.is_some() || score_maximum.is_some() {
        if let (Some(a), Some(b)) = (total_exercises, score_maximum)
            && a == 0
            && b == 0
        {
            res += &format!(
                " This {course_or_module} has no exercises and no points. It cannot be completed by doing exercises."
            );
            if requires_exam {
                res += " Passing an exam is required for completion.";
            } else {
                res += &format!(
                    " The user should look for information about completing the {course_or_module} in the course material or contact the teacher.\n"
                );
            }
            return res;
        }
        res += &format!(" On this {course_or_module}, there are available a total of ");

        if let Some(a) = total_exercises {
            res += &format!("{a} exercises");
        }
        if let Some(b) = score_maximum {
            if total_exercises.is_some() {
                res += " and ";
            }
            res += &format!("{b} exercise points");
        }
        res += ".";
    }
    if automatic_completion && score_required.is_none() && attempted_exercises_required.is_none() {
        res += &format!(
            " It's not required to attempt exercises or gain points to pass this {course_or_module}."
        );
    }

    if requires_exam {
        res += &format!(" To pass this {course_or_module}, it's required to complete an exam.");
    }
    if !automatic_completion {
        res += &format!(
            " This {course_or_module} is graded by a teacher and can't be automatically passed by completing exercises. The user should look for information about completing the {course_or_module} in the course material or contact the teacher."
        );
    }

    if attempted_exercises_required.is_some() || score_required.is_some() {
        if requires_exam {
            res += " To be qualified to take the exam, it's required to ";
        } else {
            res += &format!(" To pass this {course_or_module}, it's required to ");
        }

        if let Some(a) = attempted_exercises_required {
            res += &format!("attempt {a} exercises");
        }
        if let Some(b) = score_required {
            if attempted_exercises_required.is_some() {
                res += " and ";
            }
            res += &format!("gain {b} exercise points");
        }
        res += ".";
    } else if requires_exam {
        res += " The user can attempt the exam regardless of their progress on the course."
    }

    if let Some(b) = attempted_exercises {
        res += &format!(" The user has attempted {b} exercises.");
    } else {
        res += " The user has not attempted any exercises.";
    }
    let attempted_exercises_n = attempted_exercises.unwrap_or(0);

    let pass = if requires_exam {
        "be qualified to take the exam".to_string()
    } else {
        format!("pass this {course_or_module}")
    };

    if let Some(c) = attempted_exercises_required {
        let ex_left = c - attempted_exercises_n;
        if ex_left <= 0 {
            res += &format!(
                " They meet the criteria to {pass} if they have also received enough points."
            );
        } else {
            res += &format!(" To {pass}, they need to attempt {ex_left} more exercises.");
        }
    }

    // round down to one digit
    let score = (score_given * 10.0).floor() / 10.0;
    res += &format!(" The user has gained {:.1} points.", score);
    if let Some(e) = score_required {
        let pts_left = e as f32 - score;
        if pts_left <= 0 as f32 {
            res += &format!(" The user has gained enough points to {pass}.")
        } else {
            res += &format!(
                " To {pass}, the user needs to gain {:.1} more points.",
                pts_left
            )
        }
    }
    res + "\n"
}

#[cfg(test)]
mod tests {
    use uuid::Uuid;

    use super::*;

    impl CourseProgressTool {
        fn new_mock(course_name: String, progress: Vec<UserCourseProgress>) -> Self {
            CourseProgressTool {
                state: CourseProgressState {
                    course_name,
                    progress,
                },
            }
        }
    }

    #[test]
    fn test_course_progress_output_only_base_module() {
        let progress = vec![UserCourseProgress {
            course_module_id: Uuid::nil(),
            course_module_name: "Example base module".to_string(),
            course_module_order_number: 1,
            score_given: 3.3,
            score_required: Some(4),
            score_maximum: Some(5),
            total_exercises: Some(11),
            attempted_exercises: Some(4),
            attempted_exercises_required: Some(10),
            automatic_completion: true,
            requires_exam: false,
        }];
        let tool = CourseProgressTool::new_mock("Advanced Chatbot Course".to_string(), progress);
        let output = tool.get_tool_output();

        let expected_output =
"Result: [output]The user is completing a course called Advanced Chatbot Course. Their progress on this course is the following: On this course, there are available a total of 11 exercises and 5 exercise points. To pass this course, it's required to attempt 10 exercises and gain 4 exercise points. The user has attempted 4 exercises. To pass this course, they need to attempt 6 more exercises. The user has gained 3.3 points. To pass this course, the user needs to gain 0.7 more points.\n[/output]

Instructions for describing the output: [instructions]Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. Encourage the user to ask further questions if needed.[/instructions]".to_string();

        assert_eq!(output, expected_output);
    }

    #[test]
    fn test_course_progress_output_many_modules() {
        let progress = vec![
            UserCourseProgress {
                course_module_id: Uuid::nil(),
                course_module_name: "Second extra module".to_string(),
                course_module_order_number: 3,
                score_given: 0.0,
                score_required: Some(4),
                score_maximum: Some(5),
                total_exercises: Some(6),
                attempted_exercises: None,
                attempted_exercises_required: Some(5),
                automatic_completion: true,
                requires_exam: false,
            },
            UserCourseProgress {
                course_module_id: Uuid::nil(),
                course_module_name: "Advanced Chatbot Course".to_string(),
                course_module_order_number: 1,
                score_given: 8.056,
                score_required: Some(8),
                score_maximum: Some(10),
                total_exercises: Some(5),
                attempted_exercises: Some(5),
                attempted_exercises_required: Some(5),
                automatic_completion: true,
                requires_exam: false,
            },
            UserCourseProgress {
                course_module_id: Uuid::nil(),
                course_module_name: "First extra module".to_string(),
                course_module_order_number: 2,
                score_given: 3.94,
                score_required: Some(5),
                score_maximum: Some(6),
                total_exercises: Some(6),
                attempted_exercises: Some(4),
                attempted_exercises_required: Some(5),
                automatic_completion: true,
                requires_exam: false,
            },
            UserCourseProgress {
                course_module_id: Uuid::nil(),
                course_module_name: "Chatbot advanced topics".to_string(),
                course_module_order_number: 4,
                score_given: 2.0,
                score_required: None,
                score_maximum: None,
                total_exercises: Some(2),
                attempted_exercises: Some(2),
                attempted_exercises_required: None,
                automatic_completion: true,
                requires_exam: false,
            },
        ];

        let tool = CourseProgressTool::new_mock("Advanced Chatbot Course".to_string(), progress);
        let output = tool.get_tool_output();

        let expected_output =
"Result: [output]The user is completing a course called Advanced Chatbot Course. The course has one base module, and additional modules. The user's progress on the base course module called Advanced Chatbot Course is the following: On this module, there are available a total of 5 exercises and 10 exercise points. To pass this module, it's required to attempt 5 exercises and gain 8 exercise points. The user has attempted 5 exercises. They meet the criteria to pass this module if they have also received enough points. The user has gained 8.0 points. The user has gained enough points to pass this module.
To pass the course, it's required to pass the base module. The following modules are additional to the course and to complete them, it's required to first complete the base module.
The user's progress on the course module called First extra module is the following: On this module, there are available a total of 6 exercises and 6 exercise points. To pass this module, it's required to attempt 5 exercises and gain 5 exercise points. The user has attempted 4 exercises. To pass this module, they need to attempt 1 more exercises. The user has gained 3.9 points. To pass this module, the user needs to gain 1.1 more points.
The user's progress on the course module called Second extra module is the following: On this module, there are available a total of 6 exercises and 5 exercise points. To pass this module, it's required to attempt 5 exercises and gain 4 exercise points. The user has not attempted any exercises. To pass this module, they need to attempt 5 more exercises. The user has gained 0.0 points. To pass this module, the user needs to gain 4.0 more points.
The user's progress on the course module called Chatbot advanced topics is the following: On this module, there are available a total of 2 exercises. It's not required to attempt exercises or gain points to pass this module. The user has attempted 2 exercises. The user has gained 2.0 points.\n[/output]

Instructions for describing the output: [instructions]Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. If the user asks something like 'how to pass the course', describe the passing criteria and requirements of the base module. Encourage the user to ask further questions about other modules if needed.[/instructions]".to_string();

        assert_eq!(output, expected_output);
    }

    #[test]
    fn test_course_progress_output_no_progress() {
        let progress: Vec<UserCourseProgress> = vec![];

        let tool = CourseProgressTool::new_mock("Advanced Chatbot Course".to_string(), progress);
        let output = tool.get_tool_output();

        let expected_output =
"Result: [output]The user is completing a course called Advanced Chatbot Course. There is no progress information for this user on this course. [/output]

Instructions for describing the output: [instructions]Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. Encourage the user to ask further questions if needed.[/instructions]".to_string();

        assert_eq!(output, expected_output);
    }

    #[test]
    fn test_course_progress_output_no_course_points_exercises() {
        let progress = vec![UserCourseProgress {
            course_module_id: Uuid::nil(),
            course_module_name: "Example base module".to_string(),
            course_module_order_number: 1,
            score_given: 0.0,
            score_required: None,
            score_maximum: Some(0),
            total_exercises: Some(0),
            attempted_exercises: None,
            attempted_exercises_required: None,
            automatic_completion: true,
            requires_exam: false,
        }];
        let tool = CourseProgressTool::new_mock("Advanced Chatbot Course".to_string(), progress);
        let output = tool.get_tool_output();

        let expected_output =
"Result: [output]The user is completing a course called Advanced Chatbot Course. Their progress on this course is the following: This course has no exercises and no points. It cannot be completed by doing exercises. The user should look for information about completing the course in the course material or contact the teacher.\n[/output]

Instructions for describing the output: [instructions]Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. Encourage the user to ask further questions if needed.[/instructions]".to_string();

        assert_eq!(output, expected_output);
    }

    #[test]
    fn test_course_progress_output_cant_be_completed() {
        let progress = vec![UserCourseProgress {
            course_module_id: Uuid::nil(),
            course_module_name: "Example base module".to_string(),
            course_module_order_number: 1,
            score_given: 0.0,
            score_required: Some(9),
            score_maximum: Some(10),
            total_exercises: Some(10),
            attempted_exercises: None,
            attempted_exercises_required: Some(10),
            // cannot be completed automatically
            automatic_completion: false,
            requires_exam: false,
        }];
        let tool = CourseProgressTool::new_mock("Advanced Chatbot Course".to_string(), progress);
        let output = tool.get_tool_output();

        let expected_output =
"Result: [output]The user is completing a course called Advanced Chatbot Course. Their progress on this course is the following: On this course, there are available a total of 10 exercises and 10 exercise points. This course is graded by a teacher and can't be automatically passed by completing exercises. The user should look for information about completing the course in the course material or contact the teacher. To pass this course, it's required to attempt 10 exercises and gain 9 exercise points. The user has not attempted any exercises. To pass this course, they need to attempt 10 more exercises. The user has gained 0.0 points. To pass this course, the user needs to gain 9.0 more points.\n[/output]

Instructions for describing the output: [instructions]Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. Encourage the user to ask further questions if needed.[/instructions]".to_string();

        assert_eq!(output, expected_output);
    }

    #[test]
    fn test_course_progress_output_exercises_required_none() {
        let progress = vec![UserCourseProgress {
            course_module_id: Uuid::nil(),
            course_module_name: "Example base module".to_string(),
            course_module_order_number: 1,
            score_given: 0.0,
            score_required: Some(9),
            score_maximum: Some(10),
            total_exercises: Some(10),
            attempted_exercises: None,
            attempted_exercises_required: None,
            automatic_completion: true,
            requires_exam: false,
        }];
        let tool = CourseProgressTool::new_mock("Advanced Chatbot Course".to_string(), progress);
        let output = tool.get_tool_output();

        let expected_output =
"Result: [output]The user is completing a course called Advanced Chatbot Course. Their progress on this course is the following: On this course, there are available a total of 10 exercises and 10 exercise points. To pass this course, it's required to gain 9 exercise points. The user has not attempted any exercises. The user has gained 0.0 points. To pass this course, the user needs to gain 9.0 more points.\n[/output]

Instructions for describing the output: [instructions]Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. Encourage the user to ask further questions if needed.[/instructions]".to_string();

        assert_eq!(output, expected_output);
    }

    #[test]
    fn test_course_progress_output_pts_required_none() {
        let progress = vec![UserCourseProgress {
            course_module_id: Uuid::nil(),
            course_module_name: "Example base module".to_string(),
            course_module_order_number: 1,
            score_given: 0.0,
            score_required: None,
            score_maximum: Some(10),
            total_exercises: Some(10),
            attempted_exercises: None,
            attempted_exercises_required: Some(10),
            automatic_completion: true,
            requires_exam: false,
        }];
        let tool = CourseProgressTool::new_mock("Advanced Chatbot Course".to_string(), progress);
        let output = tool.get_tool_output();

        let expected_output =
"Result: [output]The user is completing a course called Advanced Chatbot Course. Their progress on this course is the following: On this course, there are available a total of 10 exercises and 10 exercise points. To pass this course, it's required to attempt 10 exercises. The user has not attempted any exercises. To pass this course, they need to attempt 10 more exercises. The user has gained 0.0 points.\n[/output]

Instructions for describing the output: [instructions]Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. Encourage the user to ask further questions if needed.[/instructions]".to_string();

        assert_eq!(output, expected_output);
    }

    #[test]
    fn test_course_progress_output_exam_required() {
        let progress = vec![UserCourseProgress {
            course_module_id: Uuid::nil(),
            course_module_name: "Example base module".to_string(),
            course_module_order_number: 1,
            score_given: 0.0780006,
            score_required: Some(9),
            score_maximum: Some(10),
            total_exercises: Some(10),
            attempted_exercises: None,
            attempted_exercises_required: Some(10),
            automatic_completion: true,
            requires_exam: true,
        }];
        let tool = CourseProgressTool::new_mock("Advanced Chatbot Course".to_string(), progress);
        let output = tool.get_tool_output();

        let expected_output =
"Result: [output]The user is completing a course called Advanced Chatbot Course. Their progress on this course is the following: On this course, there are available a total of 10 exercises and 10 exercise points. To pass this course, it's required to complete an exam. To be qualified to take the exam, it's required to attempt 10 exercises and gain 9 exercise points. The user has not attempted any exercises. To be qualified to take the exam, they need to attempt 10 more exercises. The user has gained 0.0 points. To be qualified to take the exam, the user needs to gain 9.0 more points.\n[/output]

Instructions for describing the output: [instructions]Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. Encourage the user to ask further questions if needed.[/instructions]".to_string();

        assert_eq!(output, expected_output);
    }

    #[test]
    fn test_course_progress_output_exam_required_can_do_exam() {
        let progress = vec![UserCourseProgress {
            course_module_id: Uuid::nil(),
            course_module_name: "Example base module".to_string(),
            course_module_order_number: 1,
            score_given: 9.00006,
            score_required: Some(9),
            score_maximum: Some(10),
            total_exercises: Some(10),
            attempted_exercises: Some(10),
            attempted_exercises_required: Some(10),
            automatic_completion: true,
            requires_exam: true,
        }];
        let tool = CourseProgressTool::new_mock("Advanced Chatbot Course".to_string(), progress);
        let output = tool.get_tool_output();

        let expected_output =
"Result: [output]The user is completing a course called Advanced Chatbot Course. Their progress on this course is the following: On this course, there are available a total of 10 exercises and 10 exercise points. To pass this course, it's required to complete an exam. To be qualified to take the exam, it's required to attempt 10 exercises and gain 9 exercise points. The user has attempted 10 exercises. They meet the criteria to be qualified to take the exam if they have also received enough points. The user has gained 9.0 points. The user has gained enough points to be qualified to take the exam.\n[/output]

Instructions for describing the output: [instructions]Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. Encourage the user to ask further questions if needed.[/instructions]".to_string();

        assert_eq!(output, expected_output);
    }

    #[test]
    fn test_course_progress_output_exam_only_required() {
        let progress = vec![UserCourseProgress {
            course_module_id: Uuid::nil(),
            course_module_name: "Example base module".to_string(),
            course_module_order_number: 1,
            score_given: 9.0,
            score_required: None,
            score_maximum: Some(10),
            total_exercises: Some(10),
            attempted_exercises: Some(10),
            attempted_exercises_required: None,
            automatic_completion: true,
            requires_exam: true,
        }];
        let tool = CourseProgressTool::new_mock("Advanced Chatbot Course".to_string(), progress);
        let output = tool.get_tool_output();

        let expected_output =
"Result: [output]The user is completing a course called Advanced Chatbot Course. Their progress on this course is the following: On this course, there are available a total of 10 exercises and 10 exercise points. It's not required to attempt exercises or gain points to pass this course. To pass this course, it's required to complete an exam. The user can attempt the exam regardless of their progress on the course. The user has attempted 10 exercises. The user has gained 9.0 points.\n[/output]

Instructions for describing the output: [instructions]Describe this information in a short, clear way with no or minimal bullet points. Only give information that is relevant to the user's question. Encourage the user to ask further questions if needed.[/instructions]".to_string();

        assert_eq!(output, expected_output);
    }
}