tmc_langs_framework/
meta_syntax.rs

1//! Contains utilities for parsing annotated exercise source files, separating lines into
2//! strings, stubs and solutions so that they can be more easily filtered later.
3
4use once_cell::sync::Lazy;
5use regex::{Captures, Regex};
6
7// rules for finding comments in various languages
8static META_SYNTAXES_C: Lazy<[MetaSyntax; 2]> = Lazy::new(|| {
9    [
10        MetaSyntax::new("//", None),
11        MetaSyntax::new(r"/\*", Some(r"\*/")),
12    ]
13});
14static META_SYNTAXES_HTML: Lazy<[MetaSyntax; 1]> =
15    Lazy::new(|| [MetaSyntax::new("<!--", Some("-->"))]);
16static META_SYNTAXES_PY: Lazy<[MetaSyntax; 1]> = Lazy::new(|| [MetaSyntax::new("#", None)]);
17
18/// Used to classify lines of code based on the annotations in the file.
19#[derive(Debug, PartialEq, Eq)]
20pub enum MetaString {
21    String(String),
22    Stub(String),
23    Solution(String),
24    Hidden(String),
25    SolutionFileMarker,
26    HiddenFileMarker,
27}
28
29/// Contains the needed regexes for a given comment syntax.
30#[derive(Debug)]
31struct MetaSyntax {
32    solution_file: Regex,
33    solution_begin: Regex,
34    solution_end: Regex,
35    stub_begin: Regex,
36    stub_end: Regex,
37    hidden_file: Regex,
38    hidden_begin: Regex,
39    hidden_end: Regex,
40}
41
42#[allow(clippy::unwrap_used)]
43impl MetaSyntax {
44    fn new(comment_start: &'static str, comment_end: Option<&'static str>) -> Self {
45        // comment patterns
46        let comment_start_pattern = format!(r"^(\s*){comment_start}\s*");
47        let comment_end_pattern = match comment_end {
48            Some(s) => format!(r"(.*){s}\s*"),
49            None => "(.*)".to_string(),
50        };
51
52        // annotation patterns
53        let solution_file = Regex::new(&format!(
54            r"{comment_start_pattern}SOLUTION\s+FILE{comment_end_pattern}"
55        ))
56        .unwrap();
57        let solution_begin = Regex::new(&format!(
58            r"{comment_start_pattern}BEGIN\s+SOLUTION{comment_end_pattern}"
59        ))
60        .unwrap();
61        let solution_end = Regex::new(&format!(
62            r"{comment_start_pattern}END\s+SOLUTION{comment_end_pattern}"
63        ))
64        .unwrap();
65        let stub_begin = Regex::new(&format!(r"{comment_start_pattern}STUB:[\s&&[^\n]]*")).unwrap();
66        let stub_end = Regex::new(&comment_end_pattern).unwrap();
67        let hidden_file = Regex::new(&format!(
68            r"{comment_start_pattern}HIDDEN\s+FILE{comment_end_pattern}"
69        ))
70        .unwrap();
71        let hidden_begin = Regex::new(&format!(
72            r"{comment_start_pattern}BEGIN\s+HIDDEN{comment_end_pattern}"
73        ))
74        .unwrap();
75        let hidden_end = Regex::new(&format!(
76            r"{comment_start_pattern}END\s+HIDDEN{comment_end_pattern}"
77        ))
78        .unwrap();
79
80        Self {
81            solution_file,
82            solution_begin,
83            solution_end,
84            stub_begin,
85            stub_end,
86            hidden_file,
87            hidden_begin,
88            hidden_end,
89        }
90    }
91}
92
93/// Parses a given text file into an iterator of `MetaString`s.
94#[derive(Debug)]
95pub struct MetaSyntaxParser<I> {
96    meta_syntaxes: &'static [MetaSyntax],
97    line_iterator: I,
98    // contains the syntax that started the current stub block
99    // used to make sure only the appropriate terminator ends the block
100    in_stub: Option<&'static MetaSyntax>,
101    in_solution: bool,
102    in_hidden: bool,
103}
104
105impl<E, I: Iterator<Item = Result<String, E>>> MetaSyntaxParser<I> {
106    pub fn new(line_iterator: I, target_extension: &str) -> Self {
107        // assigns each supported file extension with the proper comment syntax
108        // todo: stop checking extension twice here and in submission_processing
109        // NOTE: if you change these extensions make sure to change them in submission_processing.rs as well
110        let meta_syntaxes: &[MetaSyntax] = match target_extension {
111            "java" | "c" | "cpp" | "h" | "hpp" | "js" | "css" | "rs" | "qml" | "cs" => {
112                &*META_SYNTAXES_C
113            }
114            "xml" | "http" | "html" | "qrc" => &*META_SYNTAXES_HTML,
115            "properties" | "py" | "R" | "pro" | "ipynb" => &*META_SYNTAXES_PY,
116            _ => &[],
117        };
118
119        Self {
120            meta_syntaxes,
121            line_iterator,
122            in_stub: None,
123            in_solution: false,
124            in_hidden: false,
125        }
126    }
127}
128
129// iterates through the lines in the underlying file, parsing them to MetaStrings
130impl<E, I: Iterator<Item = Result<String, E>>> Iterator for MetaSyntaxParser<I> {
131    type Item = Result<MetaString, E>;
132
133    fn next(&mut self) -> Option<Self::Item> {
134        match self.line_iterator.next() {
135            Some(Ok(mut s)) => {
136                // check line with each meta syntax
137                for meta_syntax in self.meta_syntaxes {
138                    // check for stub
139                    if self.in_stub.is_none() && meta_syntax.stub_begin.is_match(&s) {
140                        log::trace!("stub start: '{s}'");
141                        // remove stub start
142                        s = meta_syntax
143                            .stub_begin
144                            .replace(&s, |caps: &Captures| caps[1].to_string())
145                            .to_string();
146
147                        if s.trim().is_empty() && meta_syntax.stub_end.is_match(&s) {
148                            // empty oneliner stubs are replaced by a newline
149                            return Some(Ok(MetaString::Stub("\n".to_string())));
150                        }
151
152                        // save the syntax that started the current stub
153                        self.in_stub = Some(meta_syntax);
154
155                        if s.trim().is_empty() {
156                            // only metadata, skip
157                            return self.next();
158                        }
159                    }
160                    // if the line matches stub_end and the saved syntax matches
161                    // the start of the current meta syntax, return stub contents if any
162                    if meta_syntax.stub_end.is_match(&s)
163                        && self.in_stub.map(|r| r.stub_begin.as_str())
164                            == Some(meta_syntax.stub_begin.as_str())
165                    {
166                        log::trace!("stub end: '{s}'");
167                        self.in_stub = None;
168                        // remove stub end
169                        s = meta_syntax
170                            .stub_end
171                            .replace(&s, |caps: &Captures| caps[1].to_string())
172                            .to_string();
173                        if s.trim().is_empty() {
174                            // only metadata, skip
175                            return self.next();
176                        }
177                        // return the stub contents
178                        return Some(Ok(MetaString::Stub(s)));
179                    }
180
181                    // check for solution, skip solution begin/end markers
182                    if meta_syntax.solution_file.is_match(&s) {
183                        log::trace!("solution file marker");
184                        return Some(Ok(MetaString::SolutionFileMarker));
185                    } else if meta_syntax.solution_begin.is_match(&s) {
186                        self.in_solution = true;
187                        return self.next();
188                    } else if meta_syntax.solution_end.is_match(&s) && self.in_solution {
189                        self.in_solution = false;
190                        return self.next();
191                    } else if meta_syntax.hidden_file.is_match(&s) {
192                        log::trace!("hidden file marker");
193                        return Some(Ok(MetaString::HiddenFileMarker));
194                    } else if meta_syntax.hidden_begin.is_match(&s) {
195                        self.in_hidden = true;
196                        return self.next();
197                    } else if meta_syntax.hidden_end.is_match(&s) {
198                        self.in_hidden = false;
199                        return self.next();
200                    }
201                }
202                // after processing the line with each meta syntax,
203                // parse the current line accordingly
204                if self.in_solution {
205                    log::trace!("solution: '{s}'");
206                    Some(Ok(MetaString::Solution(s)))
207                } else if self.in_stub.is_some() {
208                    log::trace!("stub: '{s}'");
209                    Some(Ok(MetaString::Stub(s)))
210                } else if self.in_hidden {
211                    log::trace!("hidden: '{s}'");
212                    Some(Ok(MetaString::Hidden(s)))
213                } else {
214                    log::trace!("string: '{s}'");
215                    Some(Ok(MetaString::String(s)))
216                }
217            }
218            Some(Err(e)) => Some(Err(e)),
219            None => None,
220        }
221    }
222}
223
224#[cfg(test)]
225#[allow(clippy::unwrap_used)]
226mod test {
227    use super::*;
228    use std::convert::Infallible;
229
230    fn init() {
231        use log::*;
232        use simple_logger::*;
233        let _ = SimpleLogger::new().with_level(LevelFilter::Debug).init();
234    }
235
236    impl MetaString {
237        fn str(s: &str) -> Self {
238            Self::String(s.to_string())
239        }
240
241        fn solution(s: &str) -> Self {
242            Self::Solution(s.to_string())
243        }
244
245        fn stub(s: &str) -> Self {
246            Self::Stub(s.to_string())
247        }
248    }
249
250    #[test]
251    fn parse_simple() {
252        init();
253
254        const JAVA_FILE: &str = r#"
255public class JavaTestCase {
256    // BEGIN SOLUTION
257    public int foo() {
258        return 3;
259    }
260    // END SOLUTION
261}
262"#;
263        let expected: Vec<MetaString> = vec![
264            MetaString::str("\n"),
265            MetaString::str("public class JavaTestCase {\n"),
266            MetaString::solution("    public int foo() {\n"),
267            MetaString::solution("        return 3;\n"),
268            MetaString::solution("    }\n"),
269            MetaString::str("}\n"),
270        ];
271
272        let filter = MetaSyntaxParser::new(
273            JAVA_FILE
274                .lines()
275                .map(|s| Ok::<_, Infallible>(format!("{s}\n"))),
276            "java",
277        );
278        let actual = filter.map(|l| l.unwrap()).collect::<Vec<MetaString>>();
279        assert_eq!(expected, actual);
280    }
281
282    #[test]
283    fn parse_solution() {
284        init();
285
286        const JAVA_FILE_SOLUTION: &str = r#"
287/*    SOLUTION  FILE    */
288public class JavaTestCase {
289    public int foo() {
290        return 3;
291    }
292}
293"#;
294        let expected: Vec<MetaString> = vec![
295            MetaString::str("\n"),
296            MetaString::SolutionFileMarker,
297            MetaString::str("public class JavaTestCase {\n"),
298            MetaString::str("    public int foo() {\n"),
299            MetaString::str("        return 3;\n"),
300            MetaString::str("    }\n"),
301            MetaString::str("}\n"),
302        ];
303
304        let filter = MetaSyntaxParser::new(
305            JAVA_FILE_SOLUTION
306                .lines()
307                .map(|s| Ok::<_, Infallible>(format!("{s}\n"))),
308            "java",
309        );
310        let actual = filter.map(|l| l.unwrap()).collect::<Vec<MetaString>>();
311        assert_eq!(expected, actual);
312    }
313
314    #[test]
315    fn parse_stubs() {
316        init();
317
318        const JAVA_FILE_STUB: &str = r#"
319public class JavaTestCase {
320    public int foo() {
321        return 3;
322        // STUB: return 0;
323        /* STUB:
324        stubs
325        stubs
326        */
327    }
328}
329"#;
330
331        let expected: Vec<MetaString> = vec![
332            MetaString::str("\n"),
333            MetaString::str("public class JavaTestCase {\n"),
334            MetaString::str("    public int foo() {\n"),
335            MetaString::str("        return 3;\n"),
336            MetaString::stub("        return 0;\n"),
337            MetaString::stub("        stubs\n"),
338            MetaString::stub("        stubs\n"),
339            MetaString::str("    }\n"),
340            MetaString::str("}\n"),
341        ];
342
343        let filter = MetaSyntaxParser::new(
344            JAVA_FILE_STUB
345                .lines()
346                .map(|s| Ok::<_, Infallible>(format!("{s}\n"))),
347            "java",
348        );
349        let actual = filter.map(|l| l.unwrap()).collect::<Vec<MetaString>>();
350        assert_eq!(expected, actual);
351    }
352
353    #[test]
354    fn stube() {
355        init();
356
357        const PYTHON_FILE_STUB: &str = r#"
358# BEGIN SOLUTION
359print("a")
360# END SOLUTION
361# KOMMENTTI
362#STUB:class Kauppalista:
363    #STUB:def __init__(self):
364        #STUB:self.tuotteet = []
365    #STUB:
366        #STUB:def tuotteita(self):
367            #STUB:return len(self.tuotteet)
368    #STUB:
369        #STUB:def lisaa(self, tuote: str, maara: int):
370            #STUB:self.tuotteet.append((tuote, maara))
371    #STUB:
372        #STUB:def tuote(self, n: int):
373            #STUB:return self.tuotteet[n - 1][0]
374    #STUB:
375        #STUB:def maara(self, n:int):
376            #STUB:return self.uotteet[n - 1][1]
377"#;
378
379        let expected: Vec<MetaString> = vec![
380            MetaString::str("\n"),
381            MetaString::solution("print(\"a\")\n"),
382            MetaString::str("# KOMMENTTI\n"),
383            MetaString::stub("class Kauppalista:\n"),
384            MetaString::stub("    def __init__(self):\n"),
385            MetaString::stub("        self.tuotteet = []\n"),
386            MetaString::stub("\n"),
387            MetaString::stub("        def tuotteita(self):\n"),
388            MetaString::stub("            return len(self.tuotteet)\n"),
389            MetaString::stub("\n"),
390            MetaString::stub("        def lisaa(self, tuote: str, maara: int):\n"),
391            MetaString::stub("            self.tuotteet.append((tuote, maara))\n"),
392            MetaString::stub("\n"),
393            MetaString::stub("        def tuote(self, n: int):\n"),
394            MetaString::stub("            return self.tuotteet[n - 1][0]\n"),
395            MetaString::stub("\n"),
396            MetaString::stub("        def maara(self, n:int):\n"),
397            MetaString::stub("            return self.uotteet[n - 1][1]\n"),
398        ];
399
400        let filter = MetaSyntaxParser::new(
401            PYTHON_FILE_STUB
402                .lines()
403                .map(|s| Ok::<_, Infallible>(format!("{s}\n"))),
404            "py",
405        );
406        let actual = filter.map(|l| l.unwrap()).collect::<Vec<MetaString>>();
407        assert_eq!(expected, actual);
408    }
409}
tmc_langs_framework/meta_syntax.rs

tmc_langs_framework/
meta_syntax.rs
