rust_i18n_support/
lib.rs

1use normpath::PathExt;
2use std::fs::File;
3use std::io::prelude::*;
4use std::{collections::BTreeMap, path::Path};
5
6mod atomic_str;
7mod backend;
8mod config;
9mod cow_str;
10mod minify_key;
11pub use atomic_str::AtomicStr;
12pub use backend::{Backend, BackendExt, SimpleBackend};
13pub use config::I18nConfig;
14pub use cow_str::CowStr;
15pub use minify_key::{
16    minify_key, MinifyKey, DEFAULT_MINIFY_KEY, DEFAULT_MINIFY_KEY_LEN, DEFAULT_MINIFY_KEY_PREFIX,
17    DEFAULT_MINIFY_KEY_THRESH,
18};
19
20type Locale = String;
21type Value = serde_json::Value;
22type Translations = BTreeMap<Locale, Value>;
23
24pub fn is_debug() -> bool {
25    std::env::var("RUST_I18N_DEBUG").unwrap_or_else(|_| "0".to_string()) == "1"
26}
27
28/// Merge JSON Values, merge b into a
29fn merge_value(a: &mut Value, b: &Value) {
30    match (a, b) {
31        (Value::Object(a), Value::Object(b)) => {
32            for (k, v) in b {
33                merge_value(a.entry(k.clone()).or_insert(Value::Null), v);
34            }
35        }
36        (a, b) => {
37            *a = b.clone();
38        }
39    }
40}
41
42// Load locales into flatten key, value HashMap
43pub fn load_locales<F: Fn(&str) -> bool>(
44    locales_path: &str,
45    ignore_if: F,
46) -> BTreeMap<String, BTreeMap<String, String>> {
47    let mut result: BTreeMap<String, BTreeMap<String, String>> = BTreeMap::new();
48    let mut translations = BTreeMap::new();
49    let locales_path = match Path::new(locales_path).normalize() {
50        Ok(p) => p,
51        Err(e) => {
52            if is_debug() {
53                println!("cargo:i18n-error={}", e);
54            }
55            return result;
56        }
57    };
58    let locales_path = match locales_path.as_path().to_str() {
59        Some(p) => p,
60        None => {
61            if is_debug() {
62                println!("cargo:i18n-error=could not convert path");
63            }
64            return result;
65        }
66    };
67
68    let path_pattern = format!("{locales_path}/**/*.{{yml,yaml,json,toml}}");
69
70    if is_debug() {
71        println!("cargo:i18n-locale={}", &path_pattern);
72    }
73
74    // check dir exists
75    if !Path::new(locales_path).exists() {
76        if is_debug() {
77            println!("cargo:i18n-error=path not exists: {}", locales_path);
78        }
79        return result;
80    }
81
82    for entry in globwalk::glob(&path_pattern).expect("Failed to read glob pattern") {
83        let entry = entry.unwrap().into_path();
84        if is_debug() {
85            println!("cargo:i18n-load={}", &entry.display());
86        }
87
88        if ignore_if(&entry.display().to_string()) {
89            continue;
90        }
91
92        let locale = entry
93            .file_stem()
94            .and_then(|s| s.to_str())
95            .and_then(|s| s.split('.').last())
96            .unwrap();
97
98        let ext = entry.extension().and_then(|s| s.to_str()).unwrap();
99
100        let file = File::open(&entry).expect("Failed to open file");
101        let mut reader = std::io::BufReader::new(file);
102        let mut content = String::new();
103
104        reader
105            .read_to_string(&mut content)
106            .expect("Read file failed.");
107
108        let trs = parse_file(&content, ext, locale)
109            .unwrap_or_else(|_| panic!("Parse file `{}` failed", entry.display()));
110
111        trs.into_iter().for_each(|(k, new_value)| {
112            translations
113                .entry(k)
114                .and_modify(|old_value| merge_value(old_value, &new_value))
115                .or_insert(new_value);
116        });
117    }
118
119    translations.iter().for_each(|(locale, trs)| {
120        result.insert(locale.to_string(), flatten_keys("", trs));
121    });
122
123    result
124}
125
126// Parse Translations from file to support multiple formats
127fn parse_file(content: &str, ext: &str, locale: &str) -> Result<Translations, String> {
128    let result = match ext {
129        "yml" | "yaml" => serde_yaml::from_str::<serde_json::Value>(content)
130            .map_err(|err| format!("Invalid YAML format, {}", err)),
131        "json" => serde_json::from_str::<serde_json::Value>(content)
132            .map_err(|err| format!("Invalid JSON format, {}", err)),
133        "toml" => toml::from_str::<serde_json::Value>(content)
134            .map_err(|err| format!("Invalid TOML format, {}", err)),
135        _ => Err("Invalid file extension".into()),
136    };
137
138    match result {
139        Ok(v) => match get_version(&v) {
140            2 => {
141                if let Some(trs) = parse_file_v2("", &v) {
142                    return Ok(trs);
143                }
144
145                Err("Invalid locale file format, please check the version field".into())
146            }
147            _ => Ok(parse_file_v1(locale, &v)),
148        },
149        Err(e) => Err(e),
150    }
151}
152
153/// Locale file format v1
154///
155/// For example:
156/// ```yml
157/// welcome: Welcome
158/// foo: Foo bar
159/// ```
160fn parse_file_v1(locale: &str, data: &serde_json::Value) -> Translations {
161    Translations::from([(locale.to_string(), data.clone())])
162}
163
164/// Locale file format v2
165/// Iter all nested keys, if the value is not a object (Map<locale, string>), then convert into multiple locale translations
166///
167/// If the final value is Map<locale, string>, then convert them and insert into trs
168///
169/// For example (only support 1 level):
170///
171/// ```yml
172/// _version: 2
173/// welcome.first:
174///   en: Welcome
175///   zh-CN: 欢迎
176/// welcome1:
177///   en: Welcome 1
178///   zh-CN: 欢迎 1
179/// ```
180///
181/// into
182///
183/// ```yml
184/// en.welcome.first: Welcome
185/// zh-CN.welcome.first: 欢迎
186/// en.welcome1: Welcome 1
187/// zh-CN.welcome1: 欢迎 1
188/// ```
189fn parse_file_v2(key_prefix: &str, data: &serde_json::Value) -> Option<Translations> {
190    let mut trs = Translations::new();
191
192    if let serde_json::Value::Object(messages) = data {
193        for (key, value) in messages {
194            if let serde_json::Value::Object(sub_messages) = value {
195                // If all values are string, then convert them into multiple locale translations
196                for (locale, text) in sub_messages {
197                    // Ignore if the locale is not a locale
198                    // e.g:
199                    //  en: Welcome
200                    //  zh-CN: 欢迎
201                    if text.is_string() {
202                        let key = format_keys(&[key_prefix, key]);
203                        let sub_trs = BTreeMap::from([(key, text.clone())]);
204                        let sub_value = serde_json::to_value(&sub_trs).unwrap();
205
206                        trs.entry(locale.clone())
207                            .and_modify(|old_value| merge_value(old_value, &sub_value))
208                            .or_insert(sub_value);
209                        continue;
210                    }
211
212                    if text.is_object() {
213                        // Parse the nested keys
214                        // If the value is object (Map<locale, string>), iter them and convert them and insert into trs
215                        let key = format_keys(&[key_prefix, key]);
216                        if let Some(sub_trs) = parse_file_v2(&key, value) {
217                            // Merge the sub_trs into trs
218                            for (locale, sub_value) in sub_trs {
219                                trs.entry(locale)
220                                    .and_modify(|old_value| merge_value(old_value, &sub_value))
221                                    .or_insert(sub_value);
222                            }
223                        }
224                    }
225                }
226            }
227        }
228    }
229
230    if !trs.is_empty() {
231        return Some(trs);
232    }
233
234    None
235}
236
237/// Get `_version` from JSON root
238/// If `_version` is not found, then return 1 as default.
239fn get_version(data: &serde_json::Value) -> usize {
240    if let Some(version) = data.get("_version") {
241        return version.as_u64().unwrap_or(1) as usize;
242    }
243
244    1
245}
246
247/// Join the keys with dot, if any key is empty, omit it.
248fn format_keys(keys: &[&str]) -> String {
249    keys.iter()
250        .filter(|k| !k.is_empty())
251        .map(|k| k.to_string())
252        .collect::<Vec<String>>()
253        .join(".")
254}
255
256fn flatten_keys(prefix: &str, trs: &Value) -> BTreeMap<String, String> {
257    let mut v = BTreeMap::<String, String>::new();
258    let prefix = prefix.to_string();
259
260    match &trs {
261        serde_json::Value::String(s) => {
262            v.insert(prefix, s.to_string());
263        }
264        serde_json::Value::Object(o) => {
265            for (k, vv) in o {
266                let key = if prefix.is_empty() {
267                    k.clone()
268                } else {
269                    format!("{}.{}", prefix, k)
270                };
271                v.extend(flatten_keys(key.as_str(), vv));
272            }
273        }
274        serde_json::Value::Null => {
275            v.insert(prefix, "".into());
276        }
277        serde_json::Value::Bool(s) => {
278            v.insert(prefix, format!("{}", s));
279        }
280        serde_json::Value::Number(s) => {
281            v.insert(prefix, format!("{}", s));
282        }
283        serde_json::Value::Array(_) => {
284            v.insert(prefix, "".into());
285        }
286    }
287
288    v
289}
290
291#[cfg(test)]
292mod tests {
293    use super::{merge_value, parse_file};
294
295    #[test]
296    fn test_merge_value() {
297        let a = serde_json::from_str::<serde_json::Value>(
298            r#"{"foo": "Foo", "dar": { "a": "1", "b": "2" }}"#,
299        )
300        .unwrap();
301        let b = serde_json::from_str::<serde_json::Value>(
302            r#"{"foo": "Foo1", "bar": "Bar", "dar": { "b": "21" }}"#,
303        )
304        .unwrap();
305
306        let mut c = a;
307        merge_value(&mut c, &b);
308
309        assert_eq!(c["foo"], "Foo1");
310        assert_eq!(c["bar"], "Bar");
311        assert_eq!(c["dar"]["a"], "1");
312        assert_eq!(c["dar"]["b"], "21");
313    }
314
315    #[test]
316    fn test_parse_file_in_yaml() {
317        let content = "foo: Foo\nbar: Bar";
318        let mut trs = parse_file(content, "yml", "en").expect("Should ok");
319        assert_eq!(trs["en"]["foo"], "Foo");
320        assert_eq!(trs["en"]["bar"], "Bar");
321
322        trs = parse_file(content, "yaml", "en").expect("Should ok");
323        assert_eq!(trs["en"]["foo"], "Foo");
324
325        trs = parse_file(content, "yml", "zh-CN").expect("Should ok");
326        assert_eq!(trs["zh-CN"]["foo"], "Foo");
327
328        parse_file(content, "foo", "en").expect_err("Should error");
329    }
330
331    #[test]
332    fn test_parse_file_in_json() {
333        let content = r#"
334        {
335            "foo": "Foo",
336            "bar": "Bar"
337        }
338        "#;
339        let trs = parse_file(content, "json", "en").expect("Should ok");
340        assert_eq!(trs["en"]["foo"], "Foo");
341        assert_eq!(trs["en"]["bar"], "Bar");
342    }
343
344    #[test]
345    fn test_parse_file_in_toml() {
346        let content = r#"
347        foo = "Foo"
348        bar = "Bar"
349        "#;
350        let trs = parse_file(content, "toml", "en").expect("Should ok");
351        assert_eq!(trs["en"]["foo"], "Foo");
352        assert_eq!(trs["en"]["bar"], "Bar");
353    }
354
355    #[test]
356    fn test_get_version() {
357        let json = serde_yaml::from_str::<serde_json::Value>("_version: 2").unwrap();
358        assert_eq!(super::get_version(&json), 2);
359
360        let json = serde_yaml::from_str::<serde_json::Value>("_version: 1").unwrap();
361        assert_eq!(super::get_version(&json), 1);
362
363        // Default fallback to 1
364        let json = serde_yaml::from_str::<serde_json::Value>("foo: Foo").unwrap();
365        assert_eq!(super::get_version(&json), 1);
366    }
367
368    #[test]
369    fn test_parse_file_in_json_with_nested_locale_texts() {
370        let content = r#"{
371            "_version": 2,
372            "welcome": {
373                "en": "Welcome",
374                "zh-CN": "欢迎",
375                "zh-HK": "歡迎"
376            }
377        }"#;
378
379        let trs = parse_file(content, "json", "filename").expect("Should ok");
380        assert_eq!(trs["en"]["welcome"], "Welcome");
381        assert_eq!(trs["zh-CN"]["welcome"], "欢迎");
382        assert_eq!(trs["zh-HK"]["welcome"], "歡迎");
383    }
384
385    #[test]
386    fn test_parse_file_in_yaml_with_nested_locale_texts() {
387        let content = r#"
388        _version: 2
389        welcome:
390            en: Welcome
391            zh-CN: 欢迎
392            jp: ようこそ
393        welcome.sub:
394            en: Welcome 1
395            zh-CN: 欢迎 1
396            jp: ようこそ 1
397        "#;
398
399        let trs = parse_file(content, "yml", "filename").expect("Should ok");
400        assert_eq!(trs["en"]["welcome"], "Welcome");
401        assert_eq!(trs["zh-CN"]["welcome"], "欢迎");
402        assert_eq!(trs["jp"]["welcome"], "ようこそ");
403        assert_eq!(trs["en"]["welcome.sub"], "Welcome 1");
404        assert_eq!(trs["zh-CN"]["welcome.sub"], "欢迎 1");
405        assert_eq!(trs["jp"]["welcome.sub"], "ようこそ 1");
406    }
407}
rust_i18n_support/lib.rs

rust_i18n_support/
lib.rs
