headless_lms_mcp/engine/
find_replace.rs

1//! `find_replace`: literal or regex search, optionally replace, over the raw stored strings of
2//! text-bearing block attributes. Matching sees markup: `<em>for</em> loops` doesn't match "for loops".
3
4use std::collections::BTreeMap;
5
6use regex::Regex;
7use serde::Serialize;
8use serde_json::Value;
9
10use super::blocks::{self, TEXT_ATTRIBUTES};
11use super::error::{EditError, EditErrorCode};
12use super::{BlockMatch, Document, DocumentRef};
13use crate::text::{plain_text, preview, whole_word_matches};
14
15/// What a `find_replace` call searches for, where, and what it puts in place of each match.
16#[derive(Debug, Clone, Default)]
17pub struct FindReplaceRequest {
18    pub find: String,
19    /// None searches only.
20    pub replace: Option<String>,
21    pub is_regex: bool,
22    /// Attribute names, or `/regex/` patterns on names; None means [`TEXT_ATTRIBUTES`].
23    pub attributes: Option<Vec<String>>,
24    /// When set, a different number of matches fails the call.
25    pub expected_count: Option<usize>,
26}
27
28/// An attribute whose text matched, with how many times.
29#[derive(Debug, Clone, PartialEq, Serialize)]
30pub struct TextMatch {
31    #[serde(flatten)]
32    pub block: BlockMatch,
33    pub count: usize,
34}
35
36/// A word that starts with a literal `find` but goes on, and how often it occurs.
37#[derive(Debug, Clone, PartialEq, Serialize)]
38pub struct InflectedVariant {
39    pub word: String,
40    pub count: usize,
41}
42
43/// What a search found and, for a replace, the pages it changed.
44#[derive(Debug, Clone, PartialEq)]
45pub struct FindReplaceResult {
46    pub matches: Vec<TextMatch>,
47    /// Documents with replacements applied; empty for a search.
48    pub changed_pages: Vec<Document>,
49    /// Words that start with a literal `find` but continue (e.g. "kurssin" for "kurssi"); not changed.
50    pub inflected_variants: Vec<InflectedVariant>,
51}
52
53impl FindReplaceResult {
54    /// The number of matches over all pages.
55    pub fn total(&self) -> usize {
56        self.matches.iter().map(|m| m.count).sum()
57    }
58}
59
60enum Matcher {
61    Literal(String),
62    Pattern(Regex),
63}
64
65impl Matcher {
66    /// Byte ranges of matches. Literal matches don't start or end inside a word.
67    fn ranges(&self, text: &str) -> Vec<(usize, usize)> {
68        match self {
69            Matcher::Pattern(re) => re
70                .find_iter(text)
71                .map(|m| (m.start(), m.end()))
72                .filter(|(s, e)| e > s)
73                .collect(),
74            Matcher::Literal(needle) => whole_word_matches(text, needle).collect(),
75        }
76    }
77
78    /// Replaces the matches [`Self::ranges`] counts, and no others: a pattern's empty matches
79    /// are left alone.
80    fn replace(&self, text: &str, replacement: &str) -> String {
81        match self {
82            Matcher::Pattern(re) => {
83                let mut out = String::with_capacity(text.len());
84                let mut last = 0;
85                for captures in re.captures_iter(text) {
86                    let Some(found) = captures.get(0).filter(|m| !m.is_empty()) else {
87                        continue;
88                    };
89                    out.push_str(&text[last..found.start()]);
90                    captures.expand(replacement, &mut out);
91                    last = found.end();
92                }
93                out.push_str(&text[last..]);
94                out
95            }
96            Matcher::Literal(_) => {
97                let mut out = String::with_capacity(text.len());
98                let mut last = 0;
99                for (start, end) in self.ranges(text) {
100                    out.push_str(&text[last..start]);
101                    out.push_str(replacement);
102                    last = end;
103                }
104                out.push_str(&text[last..]);
105                out
106            }
107        }
108    }
109}
110
111enum AttributeFilter {
112    Name(String),
113    Pattern(Regex),
114}
115
116fn attribute_filters(request: &FindReplaceRequest) -> Result<Vec<AttributeFilter>, EditError> {
117    let names: Vec<String> = match &request.attributes {
118        Some(names) if !names.is_empty() => names.clone(),
119        _ => TEXT_ATTRIBUTES.iter().map(|s| s.to_string()).collect(),
120    };
121    names
122        .into_iter()
123        .map(
124            |name| match name.strip_prefix('/').and_then(|n| n.strip_suffix('/')) {
125                Some(pattern) => Regex::new(pattern)
126                    .map(AttributeFilter::Pattern)
127                    .map_err(|e| {
128                        EditError::new(
129                            EditErrorCode::InvalidRegex,
130                            format!("attribute pattern {name} isn't a valid regex: {e}"),
131                        )
132                    }),
133                None => Ok(AttributeFilter::Name(name)),
134            },
135        )
136        .collect()
137}
138
139fn attribute_selected(filters: &[AttributeFilter], name: &str) -> bool {
140    filters.iter().any(|f| match f {
141        AttributeFilter::Name(n) => n == name,
142        AttributeFilter::Pattern(re) => re.is_match(name),
143    })
144}
145
146/// Strings of one attribute value that count as text: the value itself, or nested `content`
147/// strings (table cells).
148fn for_each_text<'v>(value: &'v Value, is_top: bool, visit: &mut dyn FnMut(&'v str)) {
149    match value {
150        Value::String(s) if is_top => visit(s),
151        Value::Array(items) => items.iter().for_each(|v| for_each_text(v, false, visit)),
152        Value::Object(map) => {
153            for (key, v) in map {
154                match v {
155                    Value::String(s) if key == "content" => visit(s),
156                    Value::Array(_) | Value::Object(_) => for_each_text(v, false, visit),
157                    _ => {}
158                }
159            }
160        }
161        _ => {}
162    }
163}
164
165/// [`for_each_text`], for changing the strings in place.
166fn for_each_text_mut(value: &mut Value, is_top: bool, visit: &mut dyn FnMut(&mut String)) {
167    match value {
168        Value::String(s) if is_top => visit(s),
169        Value::Array(items) => items
170            .iter_mut()
171            .for_each(|v| for_each_text_mut(v, false, visit)),
172        Value::Object(map) => {
173            for (key, v) in map.iter_mut() {
174                match v {
175                    Value::String(s) if key == "content" => visit(s),
176                    Value::Array(_) | Value::Object(_) => for_each_text_mut(v, false, visit),
177                    _ => {}
178                }
179            }
180        }
181        _ => {}
182    }
183}
184
185/// Searches `pages` and, with `request.replace`, replaces every match. Fails without changing
186/// anything when `request.expected_count` differs from the matches found.
187pub fn find_replace(
188    pages: &[DocumentRef],
189    request: &FindReplaceRequest,
190) -> Result<FindReplaceResult, EditError> {
191    if request.find.is_empty() {
192        return Err(EditError::new(
193            EditErrorCode::InvalidValue,
194            "find can't be empty",
195        ));
196    }
197    let matcher = if request.is_regex {
198        Matcher::Pattern(Regex::new(&request.find).map_err(|e| {
199            EditError::new(
200                EditErrorCode::InvalidRegex,
201                format!("find isn't a valid regex: {e}"),
202            )
203            .with_fix("Rust regex syntax; flags go inline, e.g. (?i) for case-insensitive")
204        })?)
205    } else {
206        Matcher::Literal(request.find.clone())
207    };
208    let filters = attribute_filters(request)?;
209    let variant_pattern = (!request.is_regex
210        && request
211            .find
212            .chars()
213            .last()
214            .is_some_and(char::is_alphanumeric))
215    .then(|| Regex::new(&format!(r"(?i)\b{}\w+", regex::escape(&request.find))).ok())
216    .flatten();
217
218    let mut matches = Vec::new();
219    let mut changed_pages = Vec::new();
220    let mut variants: BTreeMap<String, usize> = BTreeMap::new();
221    for page in pages {
222        let page_first_match = matches.len();
223        blocks::walk_outside_exercises(page.blocks, &mut |block, _| {
224            for (name, value) in &block.attributes {
225                if !attribute_selected(&filters, name) {
226                    continue;
227                }
228                let mut count = 0;
229                let mut first_preview = None;
230                for_each_text(value, true, &mut |text| {
231                    if let Some(pattern) = &variant_pattern {
232                        for word in pattern.find_iter(&plain_text(text)) {
233                            *variants.entry(word.as_str().to_string()).or_default() += 1;
234                        }
235                    }
236                    let ranges = matcher.ranges(text);
237                    if let Some(first) = ranges.first() {
238                        count += ranges.len();
239                        first_preview.get_or_insert_with(|| context_preview(text, *first));
240                    }
241                });
242                if count > 0 {
243                    matches.push(TextMatch {
244                        block: page.key.block_match(
245                            block,
246                            format!("/{}/attributes/{name}", block.client_id),
247                            first_preview.unwrap_or_default(),
248                        ),
249                        count,
250                    });
251                }
252            }
253        });
254        if let Some(replacement) = &request.replace
255            && matches.len() > page_first_match
256        {
257            let mut doc = page.to_document();
258            let mut is_changed = false;
259            blocks::walk_mut_outside_exercises(&mut doc.blocks, &mut |block| {
260                for (name, value) in block.attributes.iter_mut() {
261                    if !attribute_selected(&filters, name) {
262                        continue;
263                    }
264                    for_each_text_mut(value, true, &mut |text| {
265                        let replaced = matcher.replace(text, replacement);
266                        if replaced != *text {
267                            *text = replaced;
268                            is_changed = true;
269                        }
270                    });
271                }
272            });
273            if is_changed {
274                changed_pages.push(doc);
275            }
276        }
277    }
278    let find_lower = request.find.to_lowercase();
279    let inflected_variants = variants
280        .into_iter()
281        .filter(|(word, _)| word.to_lowercase() != find_lower)
282        .map(|(word, count)| InflectedVariant { word, count })
283        .collect();
284    let result = FindReplaceResult {
285        matches,
286        changed_pages,
287        inflected_variants,
288    };
289    if let Some(expected) = request.expected_count
290        && expected != result.total()
291    {
292        return Err(EditError::expected_count_mismatch(
293            expected,
294            result.total(),
295            result
296                .matches
297                .iter()
298                .map(|m| m.block.location(Some(m.count)))
299                .collect(),
300        ));
301    }
302    Ok(result)
303}
304
305fn context_preview(text: &str, (start, end): (usize, usize)) -> String {
306    let before: String = text[..start]
307        .chars()
308        .rev()
309        .take(30)
310        .collect::<Vec<_>>()
311        .into_iter()
312        .rev()
313        .collect();
314    let after: String = text[end..].chars().take(30).collect();
315    preview(&format!("{before}{}{after}", &text[start..end]), 80)
316}
317
318#[cfg(test)]
319mod tests {
320    use pretty_assertions::assert_eq;
321    use serde_json::json;
322
323    use super::*;
324    use crate::engine::{PageDocument, PageType};
325    use crate::test_fixtures::{block, document_refs, exercise, heading, id, page, para};
326
327    fn pages() -> Vec<PageDocument> {
328        vec![
329            page(
330                "pg1",
331                PageType::CoursePage,
332                vec![
333                    heading("h1", "Python 3.11 kurssi"),
334                    para("p1", "Tällä kurssilla opit. Kurssin aikana kurssi etenee."),
335                    block(
336                        "img",
337                        "core/image",
338                        json!({"alt": "kurssi logo", "url": "https://example.com/kurssi.png"}),
339                        vec![],
340                    ),
341                    block(
342                        "t1",
343                        "core/table",
344                        json!({"body": [{"cells": [{"content": "kurssi", "tag": "td"}]}]}),
345                        vec![],
346                    ),
347                    exercise("ex1", "e1"),
348                ],
349            ),
350            page(
351                "pg2",
352                PageType::CoursePage,
353                vec![para("p2", "Use <em>for</em> loops in Python 3.12.")],
354            ),
355        ]
356    }
357
358    fn request(find: &str) -> FindReplaceRequest {
359        FindReplaceRequest {
360            find: find.to_string(),
361            ..Default::default()
362        }
363    }
364
365    #[test]
366    fn literal_search_counts_whole_words_and_reports_variants() {
367        let result = find_replace(&document_refs(&pages()), &request("kurssi")).unwrap();
368        assert_eq!(result.total(), 4);
369        let paths: Vec<_> = result
370            .matches
371            .iter()
372            .map(|m| m.block.path.rsplit('/').next().unwrap().to_string())
373            .collect();
374        assert_eq!(paths, vec!["content", "content", "alt", "body"]);
375        let words: Vec<_> = result
376            .inflected_variants
377            .iter()
378            .map(|v| v.word.as_str())
379            .collect();
380        assert!(words.contains(&"kurssilla"));
381        assert!(words.contains(&"Kurssin"));
382        assert!(result.changed_pages.is_empty());
383    }
384
385    #[test]
386    fn literal_replace_changes_only_matches() {
387        let mut req = request("kurssi");
388        req.replace = Some("course".into());
389        req.expected_count = Some(4);
390        let result = find_replace(&document_refs(&pages()), &req).unwrap();
391        let page = &result.changed_pages[0];
392        assert_eq!(
393            page.blocks[1].attributes["content"],
394            json!("Tällä kurssilla opit. Kurssin aikana course etenee.")
395        );
396        assert_eq!(
397            page.blocks[2].attributes["url"],
398            json!("https://example.com/kurssi.png")
399        );
400    }
401
402    #[test]
403    fn regex_replace_with_groups() {
404        let req = FindReplaceRequest {
405            find: r"Python ?3\.(\d+)".into(),
406            replace: Some("Python 3.13 (was 3.$1)".into()),
407            is_regex: true,
408            ..Default::default()
409        };
410        let result = find_replace(&document_refs(&pages()), &req).unwrap();
411        assert_eq!(result.total(), 2);
412        assert_eq!(
413            result.changed_pages[1].blocks[0].attributes["content"],
414            json!("Use <em>for</em> loops in Python 3.13 (was 3.12).")
415        );
416        assert!(result.inflected_variants.is_empty());
417    }
418
419    #[test]
420    fn a_pattern_replaces_only_the_matches_it_counts() {
421        let doc = vec![page("pg1", PageType::CoursePage, vec![para("p1", "baaa")])];
422        let req = FindReplaceRequest {
423            find: "a*".into(),
424            replace: Some("X".into()),
425            is_regex: true,
426            expected_count: Some(1),
427            ..Default::default()
428        };
429        let result = find_replace(&document_refs(&doc), &req).unwrap();
430        assert_eq!(
431            result.changed_pages[0].blocks[0].attributes["content"],
432            json!("bX")
433        );
434    }
435
436    #[test]
437    fn matching_sees_markup() {
438        assert_eq!(
439            find_replace(&document_refs(&pages()), &request("for loops"))
440                .unwrap()
441                .total(),
442            0
443        );
444        assert_eq!(
445            find_replace(&document_refs(&pages()), &request("<em>for</em> loops"))
446                .unwrap()
447                .total(),
448            1
449        );
450    }
451
452    #[test]
453    fn attribute_selection_by_name_and_regex() {
454        let mut req = request("example.com");
455        req.attributes = Some(vec!["/^ur.$/".into()]);
456        let result = find_replace(&document_refs(&pages()), &req).unwrap();
457        assert_eq!(result.total(), 1);
458        assert_eq!(
459            result.matches[0].block.path,
460            format!("/{}/attributes/url", id("img"))
461        );
462        req.attributes = Some(vec!["href".into()]);
463        assert_eq!(
464            find_replace(&document_refs(&pages()), &req)
465                .unwrap()
466                .total(),
467            0
468        );
469    }
470
471    #[test]
472    fn expected_count_mismatch_fails_with_matches() {
473        let mut req = request("kurssi");
474        req.replace = Some("x".into());
475        req.expected_count = Some(2);
476        let error = find_replace(&document_refs(&pages()), &req).unwrap_err();
477        assert_eq!(error.code, EditErrorCode::ExpectedCountMismatch);
478        assert_eq!(
479            error.details.unwrap()["matches"].as_array().unwrap().len(),
480            4
481        );
482    }
483
484    #[test]
485    fn invalid_regex_is_rejected() {
486        let req = FindReplaceRequest {
487            find: "(".into(),
488            is_regex: true,
489            ..Default::default()
490        };
491        assert_eq!(
492            find_replace(&document_refs(&pages()), &req)
493                .unwrap_err()
494                .code,
495            EditErrorCode::InvalidRegex
496        );
497    }
498}
headless_lms_mcp/engine/find_replace.rs

headless_lms_mcp/engine/
find_replace.rs
