headless_lms_mcp/
text.rs

1//! Text helpers shared by the engines: markup stripping, previews, whole-word matching, tag
2//! balance and serialized sizes.
3
4use std::borrow::Cow;
5use std::sync::LazyLock;
6
7use headless_lms_utils::strings::{HTML_TAG_REGEX, decode_html_entities, strip_html_tags};
8use regex::Regex;
9use serde::Serialize;
10
11/// Whitespace that isn't a single space; collapsing only these leaves most text borrowed.
12static WHITESPACE_TO_COLLAPSE: LazyLock<Regex> = LazyLock::new(|| {
13    Regex::new(r"\s{2,}|[^\S ]").expect("a constant pattern the unit tests compile")
14});
15
16/// The visible text of a rich-text string: tags removed, common entities decoded, whitespace
17/// collapsed to single spaces.
18pub fn plain_text(html: &str) -> Cow<'_, str> {
19    if !html.contains(['<', '&']) {
20        return collapse_whitespace(html.trim());
21    }
22    let stripped = strip_html_tags(html);
23    let decoded = decode_html_entities(&stripped);
24    Cow::Owned(collapse_whitespace(decoded.trim()).into_owned())
25}
26
27fn collapse_whitespace(text: &str) -> Cow<'_, str> {
28    WHITESPACE_TO_COLLAPSE.replace_all(text, " ")
29}
30
31/// The size of `value` as JSON, in bytes, without building the JSON.
32pub fn serialized_len<T: Serialize + ?Sized>(value: &T) -> usize {
33    struct ByteCount(usize);
34    impl std::io::Write for ByteCount {
35        fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
36            self.0 += bytes.len();
37            Ok(bytes.len())
38        }
39
40        fn flush(&mut self) -> std::io::Result<()> {
41            Ok(())
42        }
43    }
44    let mut count = ByteCount(0);
45    serde_json::to_writer(&mut count, value).map_or(0, |()| count.0)
46}
47
48/// Cuts `text` to at most `max_chars` characters at a word boundary, adding an ellipsis when cut.
49pub fn preview(text: &str, max_chars: usize) -> String {
50    if text.chars().count() <= max_chars {
51        return text.to_string();
52    }
53    let cut: String = text.chars().take(max_chars).collect();
54    let at_word = match cut.rfind(' ') {
55        Some(space) if cut[..space].chars().count() > max_chars * 2 / 3 => &cut[..space],
56        _ => cut.as_str(),
57    };
58    format!("{}…", at_word.trim_end())
59}
60
61/// Letters, digits and `_`, non-ASCII ones included. main-frontend's glossary highlighting
62/// (textParsing.tsx) uses the same set; change both together.
63fn is_word_char(c: char) -> bool {
64    c.is_alphanumeric() || c == '_'
65}
66
67/// Byte ranges of the occurrences of `needle` in `text` that don't start or end inside a word. An
68/// end of `needle` that isn't a word character (as in "C++") needs no word boundary.
69pub fn whole_word_matches<'a>(
70    text: &'a str,
71    needle: &'a str,
72) -> impl Iterator<Item = (usize, usize)> + 'a {
73    let starts_word = needle.chars().next().is_some_and(is_word_char);
74    let ends_word = needle.chars().next_back().is_some_and(is_word_char);
75    text.match_indices(needle)
76        .map(|(start, found)| (start, start + found.len()))
77        .filter(move |&(start, end)| {
78            (!starts_word || !text[..start].chars().next_back().is_some_and(is_word_char))
79                && (!ends_word || !text[end..].chars().next().is_some_and(is_word_char))
80        })
81}
82
83/// Whether byte offset `at` of `html` falls inside a tag (`<a href="…">`).
84pub fn is_inside_tag(html: &str, at: usize) -> bool {
85    let before = &html[..at];
86    match (before.rfind('<'), before.rfind('>')) {
87        (Some(open), Some(close)) => open > close,
88        (Some(_), None) => true,
89        _ => false,
90    }
91}
92
93/// Whether byte offset `at` falls after the `&` of a character reference such as `&amp;` and at or
94/// before its `;`.
95pub fn is_inside_character_reference(html: &str, at: usize) -> bool {
96    let Some(ampersand) = html[..at].rfind('&') else {
97        return false;
98    };
99    let reference = &html[ampersand + 1..];
100    reference.find(';').is_some_and(|end| {
101        ampersand + 1 + end >= at
102            && !reference[..end].is_empty()
103            && reference[..end]
104                .chars()
105                .all(|c| c.is_ascii_alphanumeric() || c == '#')
106    })
107}
108
109/// Inline elements still open at byte offset `at`, outermost first.
110pub fn open_tags_at(html: &str, at: usize) -> Vec<String> {
111    let mut open: Vec<String> = Vec::new();
112    for tag in HTML_TAG_REGEX.find_iter(&html[..at]) {
113        let inner = tag.as_str().trim_start_matches('<').trim_end_matches('>');
114        if inner.ends_with('/') {
115            continue;
116        }
117        let name: String = inner
118            .trim_start_matches('/')
119            .chars()
120            .take_while(|c| c.is_ascii_alphanumeric())
121            .collect::<String>()
122            .to_lowercase();
123        if name.is_empty() || name == "br" || name == "img" {
124            continue;
125        }
126        if inner.starts_with('/') {
127            if let Some(pos) = open.iter().rposition(|t| *t == name) {
128                open.truncate(pos);
129            }
130        } else {
131            open.push(name);
132        }
133    }
134    open
135}
136
137#[cfg(test)]
138mod tests {
139    use super::*;
140
141    #[test]
142    fn plain_text_strips_markup_and_entities() {
143        assert_eq!(
144            plain_text("Use <code>for</code> &amp;  <em>iterate</em>."),
145            "Use for & iterate."
146        );
147        assert_eq!(
148            plain_text(" One\ttwo\n\nthree  four "),
149            "One two three four"
150        );
151        assert!(matches!(plain_text("One two"), Cow::Borrowed(_)));
152    }
153
154    #[test]
155    fn preview_cuts_at_word_boundary() {
156        let text = "A function that calls itself is called recursive, and recursion is everywhere";
157        let cut = preview(text, 50);
158        assert!(cut.ends_with('…'));
159        assert!(cut.chars().count() <= 51);
160        assert!(!cut.contains("recursion is"));
161    }
162
163    #[test]
164    fn preview_counts_characters_in_finnish_text() {
165        let text = "Äänestäjät äänestävät pääministeriehdokkaasta";
166        assert_eq!(preview(text, 20), "Äänestäjät äänestävä…");
167        assert_eq!(preview(text, 30), "Äänestäjät äänestävät…");
168    }
169
170    #[test]
171    fn tag_balance() {
172        let html = "One <em>two three</em> four <a href=\"x\">five</a>";
173        assert_eq!(
174            open_tags_at(html, html.find("three").unwrap_or(0)),
175            vec!["em"]
176        );
177        assert!(open_tags_at(html, html.find("four").unwrap_or(0)).is_empty());
178        assert!(is_inside_tag(html, html.find("href").unwrap_or(0)));
179        assert!(!is_inside_tag(html, html.find("five").unwrap_or(0)));
180    }
181}
headless_lms_mcp/text.rs

headless_lms_mcp/
text.rs
