headless_lms_chatbot/azure_chatbot/azure/
protocol.rs

1//! The Azure Responses API wire types: what a request carries and what a streamed response
2//! deserializes into.
3
4use headless_lms_models::chatbot_configurations::{ReasoningEffortLevel, VerbosityLevel};
5use headless_lms_models::chatbot_conversation_message_messages::MessageRole;
6use headless_lms_utils::json_schema_types::{JSONType, Schema};
7use serde::{Deserialize, Deserializer, Serialize};
8use url::Url;
9
10use super::tools::AzureLLMToolDefinition;
11use crate::llm_utils::{APIInputMessage, MessageContent};
12use crate::prelude::*;
13
14/// Response received from LLM API
15#[derive(Deserialize, Serialize, Debug)]
16pub struct Response {
17    pub id: Option<String>,
18    pub error: Option<ResponseError>,
19    /// Why the model stopped short of a complete response. Present on `response.incomplete`.
20    pub incomplete_details: Option<IncompleteReason>,
21    pub usage: Option<Usage>,
22    pub reasoning: Option<ResponseReasoning>,
23}
24
25/// The reasoning settings a response reports back, as opposed to the ones the request asked for.
26#[derive(Deserialize, Serialize, Debug, Clone, Default)]
27pub struct ResponseReasoning {
28    /// Which turns' reasoning the model drew on. Kept as a string rather than
29    /// [`ReasoningContext`] so that a value this code does not know cannot fail the whole
30    /// response; worth logging because only [`ModelType::GPTHardThinking`](headless_lms_models::chatbot_configurations_models::ModelType::GPTHardThinking) asks for one, and
31    /// everywhere else this is the deployment's own default.
32    pub context: Option<String>,
33}
34
35/// What the request was billed for. Optional throughout: Azure reports the cache fields only on
36/// some deployment types, and PTU-M never reports `cache_write_tokens` at all.
37#[derive(Deserialize, Serialize, Debug, Clone, Default)]
38pub struct Usage {
39    pub input_tokens: Option<i64>,
40    pub output_tokens: Option<i64>,
41    pub total_tokens: Option<i64>,
42    pub input_tokens_details: Option<InputTokensDetails>,
43    pub output_tokens_details: Option<OutputTokensDetails>,
44}
45
46/// How much of the input was served from Azure's prompt cache. The only way to tell whether the
47/// prompt prefix is actually stable, since a perturbed prefix fails silently by costing more.
48#[derive(Deserialize, Serialize, Debug, Clone, Default)]
49pub struct InputTokensDetails {
50    pub cached_tokens: Option<i64>,
51    pub cache_write_tokens: Option<i64>,
52}
53
54/// How much of the output was thinking, which is what moves when the reasoning context widens or
55/// narrows.
56#[derive(Deserialize, Serialize, Debug, Clone, Default)]
57pub struct OutputTokensDetails {
58    pub reasoning_tokens: Option<i64>,
59}
60
61impl Usage {
62    /// Emits the token counts, including how much of the prompt the cache served and, from
63    /// `reasoning`, which turns' reasoning the model drew on.
64    pub fn log(&self, context: &str, reasoning: Option<&ResponseReasoning>) {
65        info!(
66            context,
67            input_tokens = self.input_tokens,
68            output_tokens = self.output_tokens,
69            reasoning_tokens = self
70                .output_tokens_details
71                .as_ref()
72                .and_then(|details| details.reasoning_tokens),
73            cached_tokens = self
74                .input_tokens_details
75                .as_ref()
76                .and_then(|details| details.cached_tokens),
77            cache_write_tokens = self
78                .input_tokens_details
79                .as_ref()
80                .and_then(|details| details.cache_write_tokens),
81            reasoning_context = reasoning.and_then(|reasoning| reasoning.context.as_deref()),
82            "LLM token usage"
83        );
84    }
85}
86
87/// Error object returned by the LLM API on a failed response. Fields are optional so any
88/// error shape deserializes rather than crashing the stream parser.
89#[derive(Deserialize, Serialize, Debug, Clone)]
90pub struct ResponseError {
91    pub code: Option<String>,
92    pub message: Option<String>,
93    #[serde(rename = "type")]
94    pub error_type: Option<String>,
95    pub param: Option<String>,
96}
97
98impl std::fmt::Display for ResponseError {
99    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
100        write!(
101            f,
102            "{}: {} (code: {}, param: {})",
103            self.error_type.as_deref().unwrap_or("Error"),
104            self.message.as_deref().unwrap_or("unknown error"),
105            self.code.as_deref().unwrap_or("none"),
106            self.param.as_deref().unwrap_or("none")
107        )
108    }
109}
110
111/// Why a response is incomplete, e.g. `max_output_tokens` or `content_filter`.
112#[derive(Deserialize, Serialize, Debug)]
113pub struct IncompleteReason {
114    pub reason: String,
115}
116
117/// One line of the Azure event stream. Which fields are set depends on `response_type`: a text
118/// delta, a completed item, a whole response, or an error.
119#[derive(Deserialize, Serialize, Debug)]
120pub struct ResponseOutput {
121    /// The event type of this response
122    // Optional so a streamed `data:` line that omits `type` still deserializes and is ignored,
123    // rather than aborting the whole chat stream.
124    #[serde(rename = "type")]
125    pub response_type: Option<String>, // for examples check out sse::ALL_EXPECTED_EVENTS
126    pub delta: Option<String>,
127    pub item: Option<ReceivedOutputItem>,
128    pub response: Option<Response>,
129    pub error: Option<ResponseError>,
130}
131
132/// An output item as it arrived, which is not necessarily one this code knows.
133///
134/// Azure adds item kinds, and [`OutputItem`] rejects a tag it has no variant for — which would
135/// fail the line it arrived on and end the whole stream. Anything that does not deserialize is
136/// kept as raw JSON and dropped by [`Self::known`] instead.
137#[derive(Deserialize, Serialize, Debug, Clone)]
138#[serde(untagged)]
139pub enum ReceivedOutputItem {
140    Known(OutputItem),
141    Unreadable(serde_json::Value),
142}
143
144impl ReceivedOutputItem {
145    /// The item, or `None` for one this code cannot read, logged as it arrived.
146    pub fn known(self) -> Option<OutputItem> {
147        match self {
148            Self::Known(item) => Some(item),
149            Self::Unreadable(raw) => {
150                warn!("Ignoring an output item from Azure that could not be read: {raw}");
151                None
152            }
153        }
154    }
155}
156
157#[derive(Deserialize, Serialize, Debug, Clone)]
158#[serde(tag = "type")]
159#[serde(rename_all = "snake_case")]
160pub enum OutputItem {
161    Message {
162        response_id: String,
163        role: MessageRole,
164        content: MessageContent,
165    },
166    Reasoning {
167        response_id: String,
168        id: String,
169        summary: Vec<ReasoningOutput>,
170        /// Absent unless the request set `store` to false, which is what makes Azure hand the
171        /// reasoning back instead of keeping it and expecting `id` to resolve against it.
172        #[serde(skip_serializing_if = "Option::is_none")]
173        encrypted_content: Option<String>,
174    },
175    AzureAiSearchCall {
176        response_id: String,
177        call_id: String,
178        /// JSON string
179        arguments: String,
180    },
181    AzureAiSearchCallOutput {
182        response_id: String,
183        call_id: String,
184        /// JSON string
185        output: String,
186    },
187    FunctionCall {
188        response_id: String,
189        call_id: String,
190        #[serde(rename = "name")]
191        tool_name: String,
192        /// JSON string
193        arguments: String,
194    },
195    FunctionCallOutput {
196        response_id: String,
197        call_id: String,
198        output: String,
199    },
200}
201
202#[derive(Deserialize, Serialize, Debug, Clone)]
203#[serde(tag = "type")]
204#[serde(rename_all = "snake_case")]
205pub enum InputItem {
206    Message {
207        role: MessageRole,
208        content: MessageContent,
209    },
210    FunctionCall {
211        call_id: String,
212        #[serde(rename = "name")]
213        tool_name: String,
214        arguments: String,
215    },
216    FunctionCallOutput {
217        call_id: String,
218        output: String,
219    },
220    Reasoning {
221        id: String,
222        summary: Vec<ReasoningOutput>,
223        /// Carries the reasoning itself, so a replayed item survives without Azure holding the
224        /// response `id` refers to. An item sent without it is rejected once `store` is false.
225        #[serde(skip_serializing_if = "Option::is_none")]
226        encrypted_content: Option<String>,
227    },
228}
229
230#[derive(Deserialize, Serialize, Debug, Clone)]
231pub struct AISearchOutput {
232    #[serde(deserialize_with = "urls_that_parse")]
233    pub get_urls: Vec<Url>,
234}
235
236/// The search results whose url this code can actually fetch. One unparseable entry must not
237/// drop every other citation from the same search.
238fn urls_that_parse<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Vec<Url>, D::Error> {
239    let raw = Vec::<String>::deserialize(deserializer)?;
240    Ok(raw
241        .into_iter()
242        .filter_map(|value| match Url::parse(&value) {
243            Ok(url) => Some(url),
244            Err(error) => {
245                warn!("Ignoring a cited document url the search returned: {value} ({error})");
246                None
247            }
248        })
249        .collect())
250}
251
252#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
253#[serde(rename_all = "snake_case")]
254pub enum LLMToolChoice {
255    Auto,
256    None,
257}
258
259#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
260#[serde(deny_unknown_fields)]
261pub struct ThinkingParams {
262    #[serde(skip_serializing_if = "Option::is_none")]
263    pub reasoning: Option<Reasoning>,
264}
265
266#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
267pub struct RequestTextOptions {
268    #[serde(skip_serializing_if = "Option::is_none")]
269    pub verbosity: Option<VerbosityLevel>,
270    #[serde(skip_serializing_if = "Option::is_none")]
271    pub format: Option<LLMRequestResponseFormatParam>,
272}
273#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
274pub struct Reasoning {
275    pub effort: ReasoningEffortLevel,
276    pub summary: Option<SummaryType>,
277    /// Which turns' reasoning the model may draw on. Leave unset for anything that is not certainly
278    /// GPT-5.6: an older reasoning deployment rejects the parameter rather than ignoring it.
279    #[serde(skip_serializing_if = "Option::is_none")]
280    pub context: Option<ReasoningContext>,
281}
282
283/// How far back the model reuses its own reasoning.
284#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
285#[serde(rename_all = "snake_case")]
286pub enum ReasoningContext {
287    /// Only this turn's reasoning, including the reasoning between the calls of one tool loop.
288    /// Reasoning replayed from completed earlier turns still travels in the request, but is not
289    /// rendered into the next sample.
290    CurrentTurn,
291    /// Also the reasoning items replayed from earlier turns, not only this turn's. GPT-5.6's own
292    /// default when the request leaves the parameter unset.
293    AllTurns,
294}
295
296/// Untagged would serialize these unit variants as `null`, which asks Azure for no summary at all.
297#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
298#[serde(rename_all = "snake_case")]
299pub enum SummaryType {
300    Concise,
301    Detailed,
302    Auto,
303}
304
305#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
306pub struct ReasoningOutput {
307    #[serde(rename = "type")]
308    pub output_type: String, //summary_text
309    pub text: String,
310}
311
312#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
313#[serde(deny_unknown_fields)]
314pub struct NonThinkingParams {
315    #[serde(skip_serializing_if = "Option::is_none")]
316    pub temperature: Option<f32>,
317    #[serde(skip_serializing_if = "Option::is_none")]
318    pub top_p: Option<f32>,
319    #[serde(skip_serializing_if = "Option::is_none")]
320    pub frequency_penalty: Option<f32>,
321    #[serde(skip_serializing_if = "Option::is_none")]
322    pub presence_penalty: Option<f32>,
323}
324
325/// Mistral's request parameters are not decided yet; this field exists only so
326/// [`LLMRequestParams::Mistral`] has a shape of its own to serialize and deserialize as, distinct
327/// from the other two variants.
328#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
329#[serde(deny_unknown_fields)]
330pub struct MistralParams {
331    pub placeholder: bool,
332}
333
334/// Each variant's own struct rejects a field belonging to another, which is what tells them apart
335/// on the way back in: untagged tries them in order and they otherwise all match any object.
336#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
337#[serde(untagged)]
338pub enum LLMRequestParams {
339    GPTThinking(ThinkingParams),
340    GPTNonThinking(NonThinkingParams),
341    Mistral(MistralParams),
342}
343
344#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
345pub struct LLMRequestResponseFormatParam {
346    #[serde(rename = "type")]
347    pub format_type: JSONType, //should be JsonSchema
348    pub name: String,
349    pub schema: Schema,
350    pub strict: bool, // should be true
351}
352
353#[derive(Serialize, Deserialize, Debug, Clone)]
354pub struct LLMRequest {
355    pub input: Vec<APIInputMessage>,
356    pub model: String,
357    #[serde(skip_serializing_if = "Vec::is_empty", default)]
358    pub tools: Vec<AzureLLMToolDefinition>,
359    #[serde(skip_serializing_if = "Option::is_none")]
360    pub tool_choice: Option<LLMToolChoice>,
361    #[serde(skip_serializing_if = "Option::is_none")]
362    pub parallel_tool_calls: Option<bool>,
363    #[serde(skip_serializing_if = "Option::is_none")]
364    pub max_output_tokens: Option<i32>,
365    #[serde(skip_serializing_if = "Option::is_none")]
366    pub text: Option<RequestTextOptions>,
367    /// Routes requests sharing a prompt prefix to the same cache entry. `None` for a model that has
368    /// no Azure prompt cache.
369    #[serde(skip_serializing_if = "Option::is_none")]
370    pub prompt_cache_key: Option<String>,
371    #[serde(flatten)]
372    pub params: LLMRequestParams,
373}
374
375/// Builds the error the stream parsers raise when Azure reports a failure on the line they are
376/// reading, attaching `err` as the error's Azure source.
377pub(super) fn azure_stream_error(response_id: Option<&str>, err: ResponseError) -> ChatbotError {
378    let mut error = chatbot_err!(
379        UpstreamReportedError,
380        format!(
381            "Error received from Azure API. Response id: {}",
382            response_id.unwrap_or("not received")
383        )
384    );
385    error.add_azure_source(err);
386    error
387}
388
389/// The error Azure reported on this line, if any: the one nested under `response` when a response
390/// object came with it, otherwise the top-level one. `fallback_response_id` is used only for the
391/// latter, since the former carries its own response id.
392pub(crate) fn reported_azure_error(
393    output: &ResponseOutput,
394    fallback_response_id: Option<&str>,
395) -> Option<ChatbotError> {
396    if let Some(response) = &output.response
397        && let Some(err) = &response.error
398    {
399        return Some(azure_stream_error(response.id.as_deref(), err.clone()));
400    }
401    output
402        .error
403        .as_ref()
404        .map(|err| azure_stream_error(fallback_response_id, err.clone()))
405}
406
407/// Why the response this line carries is incomplete, if it says so. A response that stopped short
408/// still streams as if it had finished, so this is the only thing that tells the two apart.
409pub(crate) fn reported_incomplete_reason(output: &ResponseOutput) -> Option<&str> {
410    output
411        .response
412        .as_ref()?
413        .incomplete_details
414        .as_ref()
415        .map(|details| details.reason.as_str())
416}
417
418/// Logs the usage a response line carries, if any, under `context`.
419pub(crate) fn log_response_usage(response_output: &ResponseOutput, context: &str) {
420    if let Some(response) = response_output.response.as_ref()
421        && let Some(usage) = response.usage.as_ref()
422    {
423        usage.log(context, response.reasoning.as_ref());
424    }
425}
426
427/// Logs `output`'s usage under `context`, then errors if it reports a failure — the two checks
428/// both stream parsers make on every line, before either looks at its own content.
429pub(crate) fn check_response_output(
430    output: &ResponseOutput,
431    response_id: Option<&str>,
432    context: &str,
433) -> ChatbotResult<()> {
434    log_response_usage(output, context);
435    match reported_azure_error(output, response_id) {
436        Some(error) => Err(error),
437        None => Ok(()),
438    }
439}
440
441/// Errors once a response has finished streaming if it stopped short of a complete one, whether
442/// `output` names the reason or the caller only knows a `response.incomplete` event fired with none.
443pub(crate) fn check_response_complete(
444    output: &ResponseOutput,
445    incomplete_without_reason: bool,
446) -> ChatbotResult<()> {
447    let reason = reported_incomplete_reason(output)
448        .or_else(|| incomplete_without_reason.then_some("not reported"));
449    match reason {
450        Some(reason) => Err(chatbot_err!(
451            ResponseIncomplete,
452            format!("The LLM response is incomplete. Reason: {reason}")
453        )),
454        None => Ok(()),
455    }
456}
457
458#[cfg(test)]
459mod tests {
460    use super::*;
461
462    /// A `response.failed` line carries `error` as an object, not a string. Deserializing it
463    /// must succeed so the error can be surfaced instead of crashing the stream parser.
464    #[test]
465    fn response_failed_with_error_object_deserializes() {
466        let line = r#"{"type":"response.failed","response":{"id":"resp_abc","status":"failed","error":{"code":"tool_user_error","message":"Could not complete vectorization action."}},"sequence_number":8}"#;
467
468        let parsed: ResponseOutput = serde_json::from_str(line).unwrap();
469        let error = parsed
470            .response
471            .expect("response object")
472            .error
473            .expect("error object");
474
475        assert_eq!(error.code.as_deref(), Some("tool_user_error"));
476        assert!(error.message.unwrap().contains("vectorization"));
477    }
478
479    /// Azure returns azure_ai_search call/output items with `arguments`/`output` as strings.
480    #[test]
481    fn azure_ai_search_output_items_deserialize() {
482        let call = r#"{"type":"azure_ai_search_call","id":"fc_1","response_id":"resp_abc","call_id":"call_1","arguments":"{\"query\":\"trademarks\"}","status":"completed"}"#;
483        match serde_json::from_str::<OutputItem>(call).unwrap() {
484            OutputItem::AzureAiSearchCall { arguments, .. } => {
485                assert!(arguments.contains("trademarks"))
486            }
487            other => panic!("expected AzureAiSearchCall, got {other:?}"),
488        }
489
490        let output = r#"{"type":"azure_ai_search_call_output","id":"fco_1","response_id":"resp_abc","call_id":"call_1","output":"remote tool call failed","status":"in_progress"}"#;
491        match serde_json::from_str::<OutputItem>(output).unwrap() {
492            OutputItem::AzureAiSearchCallOutput { output, .. } => {
493                assert_eq!(output, "remote tool call failed")
494            }
495            other => panic!("expected AzureAiSearchCallOutput, got {other:?}"),
496        }
497    }
498
499    /// An item kind Azure adds and this code has no variant for must not fail the whole line, or
500    /// it would kill the stream over an item nothing forced it to react to.
501    #[test]
502    fn an_unknown_item_type_deserializes_as_unreadable_instead_of_failing() {
503        let line = r#"{"type":"response.output_item.done","item":{"type":"web_search_call","id":"ws_1","status":"completed"}}"#;
504
505        let parsed: ResponseOutput =
506            serde_json::from_str(line).expect("the line deserializes despite the unknown item");
507        assert!(matches!(
508            parsed.item,
509            Some(ReceivedOutputItem::Unreadable(_))
510        ));
511        assert!(parsed.item.and_then(ReceivedOutputItem::known).is_none());
512    }
513}
headless_lms_chatbot/azure_chatbot/azure/protocol.rs

headless_lms_chatbot/azure_chatbot/azure/
protocol.rs
