shuyi_assets/lang/
tag.rs

1//! A language's tag ([`LangCode`]): BCP 47 (RFC 5646) in the subset a game
2//! names its languages with, `pt-BR`, `zh-Hans`, `es-419`, written one way
3//! only, and the reasons a text isn't one ([`TagError`]).
4use crate::prelude::*;
5
6use core::borrow::Borrow;
7use core::fmt;
8
9/// Longest [`LangCode`], in characters.
10const MAX_CODE_CHARS: usize = 35;
11
12/// The tag of the source language when a game declares none: English.
13pub(super) const DEFAULT_SOURCE: &str = "en";
14
15/// What starts a tag's private-use part.
16const PRIVATE_USE: &str = "x";
17
18/// A private-use part's start inside a tag.
19const PRIVATE_USE_PART: &str = "-x-";
20
21/// A language's tag, the name of its folder among the languages: BCP 47
22/// (RFC 5646) in this subset, its subtags joined by `-`:
23///
24/// - the language: 2 to 8 letters (`en`, `ja`);
25/// - a script, if any: 4 letters (`zh-Hans`);
26/// - a region, if any: 2 letters or 3 digits (`pt-BR`, `es-419`);
27/// - variants, each once: 5 to 8 letters or digits, or a digit and 3
28///   letters or digits (`de-CH-1996`);
29/// - a private-use part, if any: `x` and one or more subtags of 1 to 8
30///   letters or digits (`en-x-pseudo`).
31///
32/// At most 35 characters in all. A tag has one spelling: the language,
33/// variants and private use in lower case, the script in title case and the
34/// region in upper case. A tag compares, orders and hashes as its text, so
35/// a map keyed by tags is looked up by `&str` too.
36#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
37pub struct LangCode(String);
38
39/// Why a text isn't a language tag ([`LangCode::parse`]).
40#[derive(Debug, Clone, PartialEq, Eq)]
41#[non_exhaustive]
42pub enum TagError {
43    /// There is no text.
44    Empty,
45    /// The text is longer than 35 characters.
46    TooLong,
47    /// The text holds a character a tag can't: anything but letters, digits
48    /// and `-`.
49    Character(char),
50    /// Two `-` side by side, or one at an end.
51    EmptySubtag,
52    /// A POSIX locale with no language: `C` or `POSIX`.
53    NoLanguage,
54    /// The first subtag isn't a language of 2 to 8 letters.
55    Language(String),
56    /// A grandfathered tag (`i-klingon`), which the subset leaves out.
57    Grandfathered,
58    /// 3 letters right after the language (`zh-yue`): an extended language
59    /// subtag, which the subset leaves out.
60    Extlang(String),
61    /// An extension (`-u-ca-gregory`, `-t-`), which the subset leaves out.
62    Extension(char),
63    /// A variant written twice.
64    RepeatedVariant(String),
65    /// `x` with nothing after it, or a private-use subtag longer than 8.
66    PrivateUse,
67    /// A subtag where it can't be: a script after the region, say.
68    Misplaced(String),
69}
70
71impl fmt::Display for TagError {
72    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
73        match self {
74            Self::Empty => f.write_str("a tag can't be empty"),
75            Self::TooLong => write!(f, "a tag is at most {MAX_CODE_CHARS} characters"),
76            Self::Character(c) => write!(f, "{c:?} can't be in a tag: only letters, digits and -"),
77            Self::EmptySubtag => f.write_str("a tag has no empty part: no -- and no - at an end"),
78            Self::NoLanguage => f.write_str("C and POSIX name no language"),
79            Self::Language(subtag) => {
80                write!(
81                    f,
82                    "a tag starts with its language, 2 to 8 letters, not {subtag:?}"
83                )
84            }
85            Self::Grandfathered => f.write_str("an old tag starting with i- isn't taken"),
86            Self::Extlang(subtag) => write!(
87                f,
88                "{subtag:?} after the language is an extended language, which isn't taken; \
89                 write the language alone"
90            ),
91            Self::Extension(singleton) => write!(
92                f,
93                "extensions (-{singleton}-) aren't taken; only a private-use part (-x-)"
94            ),
95            Self::RepeatedVariant(subtag) => write!(f, "the variant {subtag:?} is written twice"),
96            Self::PrivateUse => f.write_str(
97                "a private-use part is -x- then one or more parts of 1 to 8 letters or digits",
98            ),
99            Self::Misplaced(subtag) => write!(
100                f,
101                "{subtag:?} can't be there: a tag is its language, then a script, a region, \
102                 variants and a private-use part, each if it has one"
103            ),
104        }
105    }
106}
107
108/// Whether `s` is `len` ASCII letters long, any case.
109fn letters(s: &str, len: core::ops::RangeInclusive<usize>) -> bool {
110    len.contains(&s.len()) && s.bytes().all(|b| b.is_ascii_alphabetic())
111}
112
113/// Whether `s` is a script subtag: 4 letters.
114fn is_script(s: &str) -> bool {
115    letters(s, 4..=4)
116}
117
118/// Whether `s` is a region subtag: 2 letters or 3 digits.
119fn is_region(s: &str) -> bool {
120    letters(s, 2..=2) || (s.len() == 3 && s.bytes().all(|b| b.is_ascii_digit()))
121}
122
123/// Whether `s` is a variant subtag: 5 to 8 letters or digits, or a digit
124/// and 3 letters or digits.
125fn is_variant(s: &str) -> bool {
126    let alnum = s.bytes().all(|b| b.is_ascii_alphanumeric());
127    let digit_first = s.len() == 4 && s.as_bytes()[0].is_ascii_digit();
128    alnum && ((5..=8).contains(&s.len()) || digit_first)
129}
130
131/// `tag` without its last subtag, a private-use part all at once; `None`
132/// for a bare language. `tag` is canonical, so the result is too.
133pub(super) fn parent_of(tag: &str) -> Option<&str> {
134    match tag.find(PRIVATE_USE_PART) {
135        Some(at) => Some(&tag[..at]),
136        None => tag.rsplit_once('-').map(|(parent, _)| parent),
137    }
138}
139
140/// `tag`, then each of its parents, down to its language. A parent is
141/// shorter than its tag, so there are never more than `tag` has
142/// characters: the walk stops there even if a parent weren't.
143pub(super) fn ancestry(tag: &str) -> impl Iterator<Item = &str> {
144    core::iter::successors(Some(tag), |tag| parent_of(tag)).take(tag.len())
145}
146
147/// What a subtag can be next, in a tag's order.
148#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
149enum Slot {
150    Script,
151    Region,
152    Variant,
153}
154
155impl LangCode {
156    /// `text` as a tag, if it is one written as a tag is written (`pt-BR`,
157    /// not `pt_br`); `None` otherwise. For text from outside, use
158    /// [`parse`](Self::parse).
159    pub fn new(text: &str) -> Option<Self> {
160        Self::parse(text).ok().filter(|code| code.as_str() == text)
161    }
162
163    /// The tag `text` names, written as a tag is written: any case, `_` for
164    /// `-`, and a POSIX locale's `.charset` and `@modifier` left out
165    /// (`pt_BR.UTF-8` is `pt-BR`). For the system's languages and the
166    /// command line. Fails with what is wrong.
167    pub fn parse(text: &str) -> Result<Self, TagError> {
168        let text = text.split(['.', '@']).next().unwrap_or_default();
169        if text.eq_ignore_ascii_case("C") || text.eq_ignore_ascii_case("POSIX") {
170            return Err(TagError::NoLanguage);
171        }
172        if text.is_empty() {
173            return Err(TagError::Empty);
174        }
175        if let Some(c) = text
176            .chars()
177            .find(|&c| !c.is_ascii_alphanumeric() && c != '-' && c != '_')
178        {
179            return Err(TagError::Character(c));
180        }
181        if text.len() > MAX_CODE_CHARS {
182            return Err(TagError::TooLong);
183        }
184        let mut subtags = text.split(['-', '_']);
185        if subtags.clone().any(str::is_empty) {
186            return Err(TagError::EmptySubtag);
187        }
188        let language = subtags.next().unwrap_or_default().to_ascii_lowercase();
189        if language == "i" {
190            return Err(TagError::Grandfathered);
191        }
192        if !letters(&language, 2..=8) {
193            return Err(TagError::Language(language));
194        }
195        let mut tag = language;
196        let mut next = Slot::Script;
197        let mut variants: Vec<String> = Vec::new();
198        let mut first = true;
199        while let Some(subtag) = subtags.next() {
200            let lower = subtag.to_ascii_lowercase();
201            tag.push('-');
202            if first && letters(subtag, 3..=3) {
203                return Err(TagError::Extlang(lower));
204            }
205            first = false;
206            if lower == PRIVATE_USE {
207                tag.push_str(PRIVATE_USE);
208                return Self::private_use(tag, subtags);
209            }
210            if lower.len() == 1 {
211                return Err(TagError::Extension(char::from(lower.as_bytes()[0])));
212            }
213            if next <= Slot::Script && is_script(subtag) {
214                tag.push_str(&subtag[..1].to_ascii_uppercase());
215                tag.push_str(&lower[1..]);
216                next = Slot::Region;
217            } else if next <= Slot::Region && is_region(subtag) {
218                tag.push_str(&subtag.to_ascii_uppercase());
219                next = Slot::Variant;
220            } else if is_variant(subtag) {
221                if variants.contains(&lower) {
222                    return Err(TagError::RepeatedVariant(lower));
223                }
224                tag.push_str(&lower);
225                variants.push(lower);
226                next = Slot::Variant;
227            } else {
228                return Err(TagError::Misplaced(subtag.to_owned()));
229            }
230        }
231        Ok(Self(tag))
232    }
233
234    /// `tag` (ending in `x`) with the private-use subtags `rest` after it.
235    fn private_use<'a>(
236        mut tag: String,
237        rest: impl Iterator<Item = &'a str>,
238    ) -> Result<Self, TagError> {
239        let mut any = false;
240        for subtag in rest {
241            if subtag.len() > 8 {
242                return Err(TagError::PrivateUse);
243            }
244            tag.push('-');
245            tag.push_str(&subtag.to_ascii_lowercase());
246            any = true;
247        }
248        if any {
249            Ok(Self(tag))
250        } else {
251            Err(TagError::PrivateUse)
252        }
253    }
254
255    /// English, the source language when a game declares none.
256    #[deprecated(note = "a game's source language is `Lang::source_lang()`")]
257    pub fn english() -> Self {
258        Self(DEFAULT_SOURCE.to_owned())
259    }
260
261    /// The tag as text.
262    pub fn as_str(&self) -> &str {
263        &self.0
264    }
265
266    /// The language subtag: `pt` for `pt-BR`.
267    pub fn language(&self) -> &str {
268        self.0.split('-').next().unwrap_or_default()
269    }
270
271    /// The subtags after the language, up to a private-use part.
272    fn subtags(&self) -> impl Iterator<Item = &str> {
273        self.0.split('-').skip(1).take_while(|s| *s != PRIVATE_USE)
274    }
275
276    /// The script subtag, if the tag has one: `Hans` for `zh-Hans-CN`.
277    pub fn script(&self) -> Option<&str> {
278        self.subtags().next().filter(|s| is_script(s))
279    }
280
281    /// The region subtag, if the tag has one: `BR` for `pt-BR`.
282    pub fn region(&self) -> Option<&str> {
283        self.subtags().take(2).find(|s| is_region(s))
284    }
285
286    /// The tag without its last subtag (`pt` for `pt-BR`), a private-use
287    /// part all at once (`en` for `en-x-pseudo`). `None` for a bare
288    /// language.
289    pub fn parent(&self) -> Option<LangCode> {
290        parent_of(&self.0).map(|parent| Self(parent.to_owned()))
291    }
292}
293
294impl Default for LangCode {
295    /// `en`, English: the source language when a game declares none.
296    fn default() -> Self {
297        Self(DEFAULT_SOURCE.to_owned())
298    }
299}
300
301impl fmt::Display for LangCode {
302    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
303        f.write_str(&self.0)
304    }
305}
306
307impl Borrow<str> for LangCode {
308    fn borrow(&self) -> &str {
309        &self.0
310    }
311}
312
313#[cfg(test)]
314mod tests {
315    use super::*;
316
317    fn tag(text: &str) -> LangCode {
318        LangCode::new(text).unwrap_or_else(|| panic!("{text}"))
319    }
320
321    #[test]
322    fn canonical_tags_are_taken_as_they_are() {
323        for good in [
324            "en",
325            "ja",
326            "dbg",
327            "test",
328            "abcdefgh",
329            "pt-BR",
330            "es-419",
331            "zh-Hans",
332            "zh-Hant-TW",
333            "sr-Latn-RS",
334            "de-CH-1996",
335            "en-x-pseudo",
336            "sl-rozaj-biske",
337            "en-US-x-a-12345678",
338        ] {
339            assert_eq!(tag(good).as_str(), good);
340            assert_eq!(tag(good).to_string(), good);
341        }
342    }
343
344    #[test]
345    fn other_spellings_and_other_tags_are_not() {
346        for bad in [
347            "",
348            "e",
349            "EN",
350            "e1",
351            "pt-br",
352            "PT-BR",
353            "pt_BR",
354            "zh-hans",
355            "zh-HANS",
356            "en-us",
357            "pt-BR.UTF-8",
358            "abcdefghi",
359            "日本",
360            "en-u-ca-gregory",
361            "en-t-ja",
362            "i-klingon",
363            "en--US",
364            "-en",
365            "en-",
366            "zh-yue",
367            "x-pseudo",
368            "en-x",
369            "en-US-Latn",
370            "en-BR-US",
371            "de-1996-1996",
372            "en-x-abcdefghi",
373            "en US",
374        ] {
375            assert_eq!(LangCode::new(bad), None, "{bad}");
376        }
377        let long = "en-x-aaaaaaaa-bbbbbbbb-cccccccc-ddd";
378        assert_eq!(long.len(), MAX_CODE_CHARS);
379        assert!(LangCode::new(long).is_some());
380        assert_eq!(LangCode::parse(&format!("{long}d")), Err(TagError::TooLong));
381    }
382
383    #[test]
384    fn text_from_outside_is_parsed_to_its_one_spelling() {
385        for (text, canonical) in [
386            ("pt_br.UTF-8", "pt-BR"),
387            ("ZH-hans-cn", "zh-Hans-CN"),
388            ("en_US@euro", "en-US"),
389            ("ja_JP.eucJP@x", "ja-JP"),
390            ("DE-ch-1996", "de-CH-1996"),
391            ("EN-X-Pseudo", "en-x-pseudo"),
392            ("es_419", "es-419"),
393        ] {
394            assert_eq!(LangCode::parse(text).unwrap().as_str(), canonical, "{text}");
395        }
396    }
397
398    #[test]
399    fn each_refusal_says_what_is_wrong() {
400        let refused = |text: &str| LangCode::parse(text).unwrap_err();
401        assert_eq!(refused("C"), TagError::NoLanguage);
402        assert_eq!(refused("C.UTF-8"), TagError::NoLanguage);
403        assert_eq!(refused("POSIX"), TagError::NoLanguage);
404        assert_eq!(refused(""), TagError::Empty);
405        assert_eq!(refused(".UTF-8"), TagError::Empty);
406        assert_eq!(refused("en US"), TagError::Character(' '));
407        assert_eq!(refused("en--US"), TagError::EmptySubtag);
408        assert_eq!(refused("x1"), TagError::Language("x1".into()));
409        assert_eq!(refused("x-pseudo"), TagError::Language("x".into()));
410        assert_eq!(refused("i-klingon"), TagError::Grandfathered);
411        assert_eq!(refused("zh-YUE"), TagError::Extlang("yue".into()));
412        assert_eq!(refused("en-u-ca-gregory"), TagError::Extension('u'));
413        assert_eq!(
414            refused("de-1996-1996"),
415            TagError::RepeatedVariant("1996".into())
416        );
417        assert_eq!(refused("en-x"), TagError::PrivateUse);
418        assert_eq!(refused("en-x-abcdefghi"), TagError::PrivateUse);
419        assert_eq!(refused("en-US-Latn"), TagError::Misplaced("Latn".into()));
420        let words: Vec<String> = [
421            TagError::Empty,
422            TagError::TooLong,
423            TagError::Character(' '),
424            TagError::EmptySubtag,
425            TagError::NoLanguage,
426            TagError::Language("x1".into()),
427            TagError::Grandfathered,
428            TagError::Extlang("yue".into()),
429            TagError::Extension('u'),
430            TagError::RepeatedVariant("1996".into()),
431            TagError::PrivateUse,
432            TagError::Misplaced("Latn".into()),
433        ]
434        .iter()
435        .map(ToString::to_string)
436        .collect();
437        assert_eq!(
438            words,
439            [
440                "a tag can't be empty",
441                "a tag is at most 35 characters",
442                "' ' can't be in a tag: only letters, digits and -",
443                "a tag has no empty part: no -- and no - at an end",
444                "C and POSIX name no language",
445                "a tag starts with its language, 2 to 8 letters, not \"x1\"",
446                "an old tag starting with i- isn't taken",
447                "\"yue\" after the language is an extended language, which isn't taken; write \
448                 the language alone",
449                "extensions (-u-) aren't taken; only a private-use part (-x-)",
450                "the variant \"1996\" is written twice",
451                "a private-use part is -x- then one or more parts of 1 to 8 letters or digits",
452                "\"Latn\" can't be there: a tag is its language, then a script, a region, \
453                 variants and a private-use part, each if it has one",
454            ]
455        );
456    }
457
458    #[test]
459    fn a_tag_names_its_parts() {
460        let parts = |text: &str| {
461            let t = tag(text);
462            (
463                t.language().to_owned(),
464                t.script().map(str::to_owned),
465                t.region().map(str::to_owned),
466            )
467        };
468        let own = |s: &str| Some(s.to_owned());
469        assert_eq!(parts("en"), ("en".into(), None, None));
470        assert_eq!(parts("pt-BR"), ("pt".into(), None, own("BR")));
471        assert_eq!(parts("es-419"), ("es".into(), None, own("419")));
472        assert_eq!(parts("zh-Hans"), ("zh".into(), own("Hans"), None));
473        assert_eq!(parts("zh-Hant-TW"), ("zh".into(), own("Hant"), own("TW")));
474        assert_eq!(parts("de-CH-1996"), ("de".into(), None, own("CH")));
475        assert_eq!(parts("de-1996"), ("de".into(), None, None));
476        assert_eq!(parts("sl-rozaj"), ("sl".into(), None, None));
477        assert_eq!(parts("en-x-ab"), ("en".into(), None, None));
478        assert_eq!(parts("en-x-abcd"), ("en".into(), None, None));
479    }
480
481    #[test]
482    fn a_parent_drops_the_last_subtag_and_private_use_at_once() {
483        let parent = |text: &str| tag(text).parent().map(|p| p.as_str().to_owned());
484        let own = |s: &str| Some(s.to_owned());
485        assert_eq!(parent("en"), None);
486        assert_eq!(parent("pt-BR"), own("pt"));
487        assert_eq!(parent("zh-Hans"), own("zh"));
488        assert_eq!(parent("zh-Hant-TW"), own("zh-Hant"));
489        assert_eq!(parent("de-CH-1996"), own("de-CH"));
490        assert_eq!(parent("sl-rozaj-biske"), own("sl-rozaj"));
491        assert_eq!(parent("en-x-pseudo"), own("en"));
492        assert_eq!(parent("en-US-x-a-b"), own("en-US"));
493        let all: Vec<&str> = ancestry("zh-Hant-TW-x-a").collect();
494        assert_eq!(all, ["zh-Hant-TW-x-a", "zh-Hant-TW", "zh-Hant", "zh"]);
495    }
496
497    #[test]
498    fn the_default_is_english() {
499        assert_eq!(LangCode::default().as_str(), "en");
500        let en = LangCode::default();
501        let found: &str = en.borrow();
502        assert_eq!(found, "en");
503    }
504
505    #[test]
506    #[expect(deprecated)]
507    fn english_is_the_default_source() {
508        assert_eq!(LangCode::english(), LangCode::default());
509    }
510
511    /// A tag built from the subset's parts, as subtags.
512    fn subtags() -> impl proptest::strategy::Strategy<Value = Vec<String>> {
513        use proptest::prelude::*;
514        (
515            "[a-z]{2,8}",
516            proptest::option::of("[A-Z][a-z]{3}"),
517            proptest::option::of("[A-Z]{2}|[0-9]{3}"),
518            proptest::collection::btree_set("[0-9][a-z0-9]{3}|[a-z0-9]{5,8}", 0..3),
519            proptest::option::of(proptest::collection::vec("[a-z0-9]{1,8}", 1..3)),
520        )
521            .prop_map(|(language, script, region, variants, private)| {
522                let mut subtags = vec![language];
523                subtags.extend(script);
524                subtags.extend(region);
525                subtags.extend(variants);
526                if let Some(private) = private {
527                    subtags.push(PRIVATE_USE.to_owned());
528                    subtags.extend(private);
529                }
530                subtags
531            })
532    }
533
534    proptest::proptest! {
535        /// Any tag of the subset is taken as it is, read back from upper
536        /// case and from `_`, and its parents end within its subtags.
537        #[test]
538        fn a_tag_reads_back_and_its_parents_end(subtags in subtags()) {
539            let text = subtags.join("-");
540            proptest::prop_assume!(text.len() <= MAX_CODE_CHARS);
541            let t = LangCode::new(&text);
542            proptest::prop_assert_eq!(t.as_ref().map(LangCode::as_str), Some(text.as_str()));
543            let upper = LangCode::parse(&text.to_ascii_uppercase()).unwrap();
544            proptest::prop_assert_eq!(upper.as_str(), text.as_str());
545            let posix = LangCode::parse(&subtags.join("_")).unwrap();
546            proptest::prop_assert_eq!(posix.as_str(), text.as_str());
547            let steps = core::iter::successors(t, LangCode::parent).take(subtags.len() + 1).count();
548            proptest::prop_assert!(steps <= subtags.len(), "{} steps for {}", steps, text);
549        }
550    }
551}
shuyi_assets/lang/tag.rs

shuyi_assets/lang/
tag.rs
