1use secrecy::ExposeSecret;
2
3use crate::prelude::*;
4use headless_lms_base::config::ApplicationConfiguration;
5use headless_lms_utils::http::REQWEST_CLIENT;
6
7const API_VERSION: &str = "2024-07-01";
8
9#[derive(Serialize, Deserialize)]
10#[serde(rename_all = "camelCase")]
11pub struct NewIndex {
12 name: String,
13 fields: Vec<Field>,
14 scoring_profiles: Vec<ScoringProfile>,
15 default_scoring_profile: Option<String>,
16 suggesters: Vec<Suggester>,
17 analyzers: Vec<Analyzer>,
18 tokenizers: Vec<serde_json::Value>,
19 token_filters: Vec<serde_json::Value>,
20 char_filters: Vec<serde_json::Value>,
21 cors_options: CorsOptions,
22 encryption_key: Option<EncryptionKey>,
23 similarity: Similarity,
24 semantic: Semantic,
25 vector_search: VectorSearch,
26}
27
28#[derive(Serialize, Deserialize)]
29#[serde(rename_all = "camelCase")]
30pub struct Analyzer {
31 name: String,
32 #[serde(rename = "@odata.type")]
33 odata_type: String,
34 char_filters: Vec<String>,
35 tokenizer: String,
36}
37
38#[derive(Serialize, Deserialize)]
39#[serde(rename_all = "camelCase")]
40pub struct CorsOptions {
41 allowed_origins: Vec<String>,
42 max_age_in_seconds: i64,
43}
44
45#[derive(Serialize, Deserialize)]
46#[serde(rename_all = "camelCase")]
47pub struct EncryptionKey {
48 key_vault_key_name: String,
49 key_vault_key_version: String,
50 key_vault_uri: String,
51 access_credentials: AccessCredentials,
52}
53
54#[derive(Serialize, Deserialize)]
55#[serde(rename_all = "camelCase")]
56pub struct AccessCredentials {
57 application_id: String,
58 application_secret: String,
59}
60
61#[derive(Serialize, Deserialize)]
62#[serde(rename_all = "camelCase")]
63pub struct Field {
64 name: String,
65 #[serde(rename = "type")]
66 field_type: String,
67 key: Option<bool>,
68 searchable: Option<bool>,
69 filterable: Option<bool>,
70 sortable: Option<bool>,
71 facetable: Option<bool>,
72 retrievable: Option<bool>,
73 index_analyzer: Option<String>,
74 search_analyzer: Option<String>,
75 analyzer: Option<String>,
76 synonym_maps: Option<Vec<String>>,
77 dimensions: Option<i64>,
78 vector_search_profile: Option<String>,
79 stored: Option<bool>,
80 vector_encoding: Option<serde_json::Value>,
81}
82
83#[derive(Serialize, Deserialize)]
84pub struct ScoringProfile {
85 name: String,
86 text: Text,
87 functions: Vec<Function>,
88}
89
90#[derive(Serialize, Deserialize)]
91#[serde(rename_all = "camelCase")]
92pub struct Function {
93 #[serde(rename = "type")]
94 function_type: String,
95 boost: i64,
96 field_name: String,
97 interpolation: String,
98 distance: Distance,
99}
100
101#[derive(Serialize, Deserialize)]
102#[serde(rename_all = "camelCase")]
103pub struct Distance {
104 reference_point_parameter: String,
105 boosting_distance: i64,
106}
107
108#[derive(Serialize, Deserialize)]
109pub struct Text {
110 weights: Weights,
111}
112
113#[derive(Serialize, Deserialize)]
114#[serde(rename_all = "camelCase")]
115pub struct Weights {
116 hotel_name: i64,
117}
118
119#[derive(Serialize, Deserialize)]
120#[serde(rename_all = "camelCase")]
121pub struct Semantic {
122 default_configuration: String,
123 configurations: Vec<SemanticConfiguration>,
124}
125
126#[derive(Serialize, Deserialize)]
127#[serde(rename_all = "camelCase")]
128pub struct SemanticConfiguration {
129 name: String,
130 prioritized_fields: SemanticConfigurationPrioritizedFields,
131}
132
133#[derive(Serialize, Deserialize)]
134#[serde(rename_all = "camelCase")]
135pub struct SemanticConfigurationPrioritizedFields {
136 title_field: FieldDescriptor,
137 prioritized_content_fields: Vec<FieldDescriptor>,
138 prioritized_keywords_fields: Vec<FieldDescriptor>,
139}
140
141#[derive(Serialize, Deserialize)]
142#[serde(rename_all = "camelCase")]
143pub struct FieldDescriptor {
144 field_name: String,
145}
146
147#[derive(Serialize, Deserialize)]
148pub struct Similarity {
149 #[serde(rename = "@odata.type")]
150 odata_type: String,
151 b: Option<f64>,
152 k1: Option<f64>,
153}
154
155#[derive(Serialize, Deserialize)]
156#[serde(rename_all = "camelCase")]
157pub struct Suggester {
158 name: String,
159 search_mode: String,
160 source_fields: Vec<String>,
161}
162
163#[derive(Serialize, Deserialize)]
164pub struct VectorSearch {
165 profiles: Vec<Profile>,
166 algorithms: Vec<Algorithm>,
167 compressions: Vec<Compression>,
168 vectorizers: Vec<Vectorizer>,
169}
170
171#[derive(Serialize, Deserialize)]
172#[serde(rename_all = "camelCase")]
173pub struct Vectorizer {
174 name: String,
175 kind: String,
176 #[serde(rename = "azureOpenAIParameters")]
177 azure_open_ai_parameters: AzureOpenAiParameters,
178 custom_web_api_parameters: Option<serde_json::Value>,
179}
180
181#[derive(Serialize, Deserialize)]
182#[serde(rename_all = "camelCase")]
183pub struct AzureOpenAiParameters {
184 resource_uri: String,
185 deployment_id: String,
186 api_key: String,
187 model_name: String,
188 auth_identity: Option<serde_json::Value>,
189}
190
191#[derive(Serialize, Deserialize)]
192#[serde(rename_all = "camelCase")]
193pub struct Algorithm {
194 name: String,
195 kind: String,
196 hnsw_parameters: Option<HnswParameters>,
197 exhaustive_knn_parameters: Option<ExhaustiveKnnParameters>,
198}
199
200#[derive(Serialize, Deserialize)]
201pub struct ExhaustiveKnnParameters {
202 metric: String,
203}
204
205#[derive(Serialize, Deserialize)]
206#[serde(rename_all = "camelCase")]
207pub struct HnswParameters {
208 m: i64,
209 metric: String,
210 ef_construction: i64,
211 ef_search: i64,
212}
213
214#[derive(Serialize, Deserialize)]
215#[serde(rename_all = "camelCase")]
216pub struct Compression {
217 name: String,
218 kind: String,
219 scalar_quantization_parameters: Option<ScalarQuantizationParameters>,
220 rerank_with_original_vectors: bool,
221 default_oversampling: i64,
222}
223
224#[derive(Serialize, Deserialize)]
225#[serde(rename_all = "camelCase")]
226pub struct ScalarQuantizationParameters {
227 quantized_data_type: String,
228}
229
230#[derive(Serialize, Deserialize)]
231pub struct Profile {
232 name: String,
233 algorithm: String,
234 compression: Option<String>,
235 vectorizer: Option<String>,
236}
237
238pub async fn does_search_index_exist(
239 index_name: &str,
240 app_config: &ApplicationConfiguration,
241) -> ChatbotResult<bool> {
242 let azure_config = app_config.azure_configuration.as_ref().ok_or_else(|| {
243 chatbot_err!(
244 AzureRequestBuildError,
245 "Azure configuration is missing from the application configuration"
246 )
247 })?;
248
249 let search_config = azure_config.search_config.as_ref().ok_or_else(|| {
250 chatbot_err!(
251 AzureRequestBuildError,
252 "Azure search configuration is missing from the Azure configuration"
253 )
254 })?;
255
256 let mut url = search_config.search_endpoint.clone();
257 url.set_path(&format!("indexes('{}')", index_name));
258 url.set_query(Some(&format!("api-version={}", API_VERSION)));
259
260 let response = REQWEST_CLIENT
261 .get(url)
262 .header("Content-Type", "application/json")
263 .header("api-key", search_config.search_api_key.expose_secret())
264 .send()
265 .await?;
266
267 if response.status().is_success() {
268 Ok(true)
269 } else if response.status() == 404 {
270 Ok(false)
271 } else {
272 let status = response.status();
273 let error_text = response.text().await?;
274 Err(chatbot_err!(
275 FailedAzureResponse,
276 format!(
277 "Error checking if index exists. Status: {}. Error: {}",
278 status, error_text
279 )
280 ))
281 }
282}
283
284pub async fn create_search_index(
285 index_name: String,
286 app_config: &ApplicationConfiguration,
287) -> ChatbotResult<()> {
288 let azure_config = app_config.azure_configuration.as_ref().ok_or_else(|| {
289 chatbot_err!(
290 AzureRequestBuildError,
291 "Azure configuration is missing from the application configuration"
292 )
293 })?;
294
295 let search_config = azure_config.search_config.as_ref().ok_or_else(|| {
296 chatbot_err!(
297 AzureRequestBuildError,
298 "Azure search configuration is missing from the Azure configuration"
299 )
300 })?;
301
302 let fields = vec![
303 Field {
304 name: "chunk_id".to_string(),
305 field_type: "Edm.String".to_string(),
306 key: Some(true),
307 searchable: Some(true),
308 filterable: Some(true),
309 retrievable: Some(true),
310 stored: Some(true),
311 sortable: Some(false),
312 facetable: Some(false),
313 analyzer: Some("keyword".to_string()),
314 index_analyzer: None,
315 search_analyzer: None,
316 synonym_maps: Some(vec![]),
317 dimensions: None,
318 vector_search_profile: None,
319 vector_encoding: None,
320 },
321 Field {
322 name: "language".to_string(),
323 field_type: "Edm.String".to_string(),
324 key: Some(false),
325 searchable: Some(true),
326 filterable: Some(true),
327 retrievable: Some(true),
328 stored: Some(true),
329 sortable: Some(false),
330 facetable: Some(false),
331 analyzer: None,
332 index_analyzer: None,
333 search_analyzer: None,
334 synonym_maps: Some(vec![]),
335 dimensions: None,
336 vector_search_profile: None,
337 vector_encoding: None,
338 },
339 Field {
340 name: "parent_id".to_string(),
341 field_type: "Edm.String".to_string(),
342 key: Some(false),
343 searchable: Some(true),
344 filterable: Some(true),
345 retrievable: Some(true),
346 stored: Some(true),
347 sortable: Some(true),
348 facetable: Some(true),
349 analyzer: None,
350 index_analyzer: None,
351 search_analyzer: None,
352 synonym_maps: Some(vec![]),
353 dimensions: None,
354 vector_search_profile: None,
355 vector_encoding: None,
356 },
357 Field {
358 name: "chunk".to_string(),
359 field_type: "Edm.String".to_string(),
360 key: Some(false),
361 searchable: Some(true),
362 filterable: Some(false),
363 retrievable: Some(true),
364 stored: Some(true),
365 sortable: Some(false),
366 facetable: Some(false),
367 analyzer: Some("keyword".to_string()),
368 index_analyzer: None,
369 search_analyzer: None,
370 synonym_maps: Some(vec![]),
371 dimensions: None,
372 vector_search_profile: None,
373 vector_encoding: None,
374 },
375 Field {
376 name: "title".to_string(),
377 field_type: "Edm.String".to_string(),
378 key: Some(false),
379 searchable: Some(true),
380 filterable: Some(true),
381 retrievable: Some(true),
382 stored: Some(true),
383 sortable: Some(false),
384 facetable: Some(false),
385 analyzer: None,
386 index_analyzer: None,
387 search_analyzer: None,
388 synonym_maps: Some(vec![]),
389 dimensions: None,
390 vector_search_profile: None,
391 vector_encoding: None,
392 },
393 Field {
394 name: "url".to_string(),
395 field_type: "Edm.String".to_string(),
396 key: Some(false),
397 searchable: Some(false),
398 filterable: Some(true),
399 retrievable: Some(true),
400 stored: Some(true),
401 sortable: Some(false),
402 facetable: Some(false),
403 analyzer: None,
404 index_analyzer: None,
405 search_analyzer: None,
406 synonym_maps: Some(vec![]),
407 dimensions: None,
408 vector_search_profile: None,
409 vector_encoding: None,
410 },
411 Field {
412 name: "course_id".to_string(),
413 field_type: "Edm.String".to_string(),
414 key: Some(false),
415 searchable: Some(false),
416 filterable: Some(true),
417 retrievable: Some(true),
418 stored: Some(true),
419 sortable: Some(false),
420 facetable: Some(false),
421 analyzer: None,
422 index_analyzer: None,
423 search_analyzer: None,
424 synonym_maps: Some(vec![]),
425 dimensions: None,
426 vector_search_profile: None,
427 vector_encoding: None,
428 },
429 Field {
430 name: "text_vector".to_string(),
431 field_type: "Collection(Edm.Single)".to_string(),
432 key: Some(false),
433 searchable: Some(true),
434 filterable: Some(false),
435 retrievable: Some(true),
436 stored: Some(true),
437 sortable: Some(false),
438 facetable: Some(false),
439 analyzer: None,
440 index_analyzer: None,
441 search_analyzer: None,
442 synonym_maps: Some(vec![]),
443 dimensions: Some(1536),
444 vector_search_profile: Some(format!("{}-azureOpenAi-text-profile", index_name)),
445 vector_encoding: None,
446 },
447 Field {
448 name: "filepath".to_string(),
449 field_type: "Edm.String".to_string(),
450 key: Some(false),
451 searchable: Some(true),
452 filterable: Some(true),
453 retrievable: Some(true),
454 stored: Some(true),
455 sortable: Some(false),
456 facetable: Some(false),
457 analyzer: None,
458 index_analyzer: None,
459 search_analyzer: None,
460 synonym_maps: Some(vec![]),
461 dimensions: None,
462 vector_search_profile: None,
463 vector_encoding: None,
464 },
465 Field {
466 name: "chunk_context".to_string(),
467 field_type: "Edm.String".to_string(),
468 key: Some(false),
469 searchable: Some(true),
470 filterable: Some(false),
471 retrievable: Some(true),
472 stored: Some(true),
473 sortable: Some(false),
474 facetable: Some(false),
475 analyzer: None,
476 index_analyzer: None,
477 search_analyzer: None,
478 synonym_maps: Some(vec![]),
479 dimensions: None,
480 vector_search_profile: None,
481 vector_encoding: None,
482 },
483 ];
484
485 let index = NewIndex {
486 name: index_name.clone(),
487 fields,
488 scoring_profiles: vec![],
489 default_scoring_profile: None,
490 suggesters: vec![],
491 analyzers: vec![],
492 tokenizers: vec![],
493 token_filters: vec![],
494 char_filters: vec![],
495 cors_options: CorsOptions {
496 allowed_origins: vec!["*".to_string()],
497 max_age_in_seconds: 300,
498 },
499 encryption_key: None,
500 similarity: Similarity {
501 odata_type: "#Microsoft.Azure.Search.BM25Similarity".to_string(),
502 b: None,
503 k1: None,
504 },
505 semantic: Semantic {
506 default_configuration: format!("{}-semantic-configuration", index_name),
507 configurations: vec![SemanticConfiguration {
508 name: format!("{}-semantic-configuration", index_name),
509 prioritized_fields: SemanticConfigurationPrioritizedFields {
510 title_field: FieldDescriptor {
511 field_name: "title".to_string(),
512 },
513 prioritized_content_fields: vec![FieldDescriptor {
514 field_name: "chunk".to_string(),
515 }],
516 prioritized_keywords_fields: vec![],
517 },
518 }],
519 },
520 vector_search: VectorSearch {
521 profiles: vec![Profile {
522 name: format!("{}-azureOpenAi-text-profile", index_name),
523 algorithm: format!("{}-algorithm", index_name),
524 vectorizer: Some(format!("{}-azureOpenAi-text-vectorizer", index_name)),
525 compression: None,
526 }],
527 algorithms: vec![Algorithm {
528 name: format!("{}-algorithm", index_name),
529 kind: "hnsw".to_string(),
530 hnsw_parameters: Some(HnswParameters {
531 m: 4,
532 metric: "cosine".to_string(),
533 ef_construction: 400,
534 ef_search: 500,
535 }),
536 exhaustive_knn_parameters: None,
537 }],
538 vectorizers: vec![Vectorizer {
539 name: format!("{}-azureOpenAi-text-vectorizer", index_name),
540 kind: "azureOpenAI".to_string(),
541 azure_open_ai_parameters: AzureOpenAiParameters {
542 resource_uri: search_config.vectorizer_resource_uri.clone(),
543 deployment_id: search_config.vectorizer_deployment_id.clone(),
544 api_key: search_config.vectorizer_api_key.expose_secret().to_string(),
545 model_name: search_config.vectorizer_model_name.clone(),
546 auth_identity: None,
547 },
548 custom_web_api_parameters: None,
549 }],
550 compressions: vec![],
551 },
552 };
553
554 let index_json = serde_json::to_string(&index)?;
555
556 let mut url = search_config.search_endpoint.clone();
557 url.set_path("/indexes");
558 url.set_query(Some(&format!("api-version={}", API_VERSION)));
559
560 let response = REQWEST_CLIENT
561 .post(url)
562 .header("Content-Type", "application/json")
563 .header("api-key", search_config.search_api_key.expose_secret())
564 .body(index_json)
565 .send()
566 .await?;
567
568 if response.status().is_success() {
570 println!("Index created successfully: {}", index_name);
571 Ok(())
572 } else {
573 let status = response.status();
574 let error_text = response.text().await?;
575 Err(chatbot_err!(
576 FailedAzureResponse,
577 format!(
578 "Failed to create index. Status: {}. Error: {}",
579 status, error_text
580 )
581 ))
582 }
583}
584
585#[derive(Serialize, Deserialize)]
586pub struct IndexAction<T> {
587 #[serde(rename = "@search.action")]
588 pub search_action: String,
589 pub document: T,
590}
591
592#[derive(Serialize, Deserialize)]
593pub struct IndexBatch<T> {
594 pub value: Vec<IndexAction<T>>,
595}
596
597pub async fn add_documents_to_index<T>(
598 index_name: &str,
599 documents: Vec<T>,
600 app_config: &ApplicationConfiguration,
601) -> ChatbotResult<()>
602where
603 T: Serialize,
604{
605 let azure_config = app_config.azure_configuration.as_ref().ok_or_else(|| {
606 chatbot_err!(
607 AzureRequestBuildError,
608 "Azure configuration is missing from the application configuration"
609 )
610 })?;
611
612 let search_config = azure_config.search_config.as_ref().ok_or_else(|| {
613 chatbot_err!(
614 AzureRequestBuildError,
615 "Azure search configuration is missing from the Azure configuration"
616 )
617 })?;
618
619 let mut url = search_config.search_endpoint.clone();
620 url.set_path(&format!("indexes('{}')/docs/index", index_name));
621 url.set_query(Some(&format!("api-version={}", API_VERSION)));
622
623 let index_actions: ChatbotResult<Vec<IndexAction<String>>> = documents
624 .into_iter()
625 .map(|doc| {
626 serde_json::to_string(&doc)
627 .map(|document| IndexAction {
628 search_action: "upload".to_string(),
629 document,
630 })
631 .map_err(|e| chatbot_err!(SerdeJson, "Failed to serialize document", e))
632 })
633 .collect();
634 let index_actions = index_actions?;
635
636 let batch = IndexBatch {
637 value: index_actions,
638 };
639
640 let batch_json = serde_json::to_string(&batch)?;
641
642 let response = REQWEST_CLIENT
643 .post(url)
644 .header("Content-Type", "application/json")
645 .header("api-key", search_config.search_api_key.expose_secret())
646 .body(batch_json)
647 .send()
648 .await?;
649
650 if response.status().is_success() {
651 println!("Documents added successfully to index: {}", index_name);
652 Ok(())
653 } else {
654 let status = response.status();
655 let error_text = response.text().await?;
656 Err(chatbot_err!(
657 FailedAzureResponse,
658 format!(
659 "Failed to add documents to index. Status: {}. Error: {}",
660 status, error_text
661 )
662 ))
663 }
664}