SickGear/lib/langcodes/__init__.py

"""
langcodes knows what languages are. It knows the standardized codes that
refer to them, such as `en` for English, `es` for Spanish and `hi` for Hindi.
Often, it knows what these languages are called *in* a language, and that
language doesn't have to be English.

See README.md for the main documentation, or read it on GitHub at
https://github.com/LuminosoInsight/langcodes/ . For more specific documentation
on the functions in langcodes, scroll down and read the docstrings.

Some of these functions, particularly those that work with the names of
languages, require the `language_data` module to be installed.
"""
from operator import itemgetter
from typing import Any, List, Tuple, Dict, Sequence, Iterable, Optional, Mapping, Union
import warnings
import sys

from langcodes.tag_parser import LanguageTagError, parse_tag, normalize_characters
from langcodes.language_distance import tuple_distance_cached
from langcodes.data_dicts import (
    ALL_SCRIPTS,
    DEFAULT_SCRIPTS,
    LANGUAGE_REPLACEMENTS,
    LANGUAGE_ALPHA3,
    LANGUAGE_ALPHA3_BIBLIOGRAPHIC,
    TERRITORY_REPLACEMENTS,
    NORMALIZED_MACROLANGUAGES,
    LIKELY_SUBTAGS,
    VALIDITY,
)

# When we're getting natural language information *about* languages, it's in
# English if you don't specify the language.
DEFAULT_LANGUAGE = 'en'


LANGUAGE_NAME_IMPORT_MESSAGE = """
Looking up language names now requires the `language_data` package.

Install it with:
    pip install language_data
Or as an optional feature of langcodes:
    pip install langcodes[data]
"""


class Language:
    """
    The Language class defines the results of parsing a language tag.
    Language objects have the following attributes, any of which may be
    unspecified (in which case their value is None):

    - *language*: the code for the language itself.
    - *script*: the 4-letter code for the writing system being used.
    - *territory*: the 2-letter or 3-digit code for the country or similar territory
      of the world whose usage of the language appears in this text.
    - *extlangs*: a list of more specific language codes that follow the language
      code. (This is allowed by the language code syntax, but deprecated.)
    - *variants*: codes for specific variations of language usage that aren't
      covered by the *script* or *territory* codes.
    - *extensions*: information that's attached to the language code for use in
      some specific system, such as Unicode collation orders.
    - *private*: a code starting with `x-` that has no defined meaning.

    The `Language.get` method converts a string to a Language instance.
    It's also available at the top level of this module as the `get` function.
    """

    ATTRIBUTES = [
        'language',
        'extlangs',
        'script',
        'territory',
        'variants',
        'extensions',
        'private',
    ]

    # When looking up "likely subtags" data, we try looking up the data for
    # increasingly less specific versions of the language code.
    BROADER_KEYSETS = [
        {'language', 'script', 'territory'},
        {'language', 'territory'},
        {'language', 'script'},
        {'language'},
        {'script'},
        {},
    ]

    MATCHABLE_KEYSETS = [
        {'language', 'script', 'territory'},
        {'language', 'script'},
        {'language'},
    ]

    # Values cached at the class level
    _INSTANCES: Dict[tuple, 'Language'] = {}
    _PARSE_CACHE: Dict[Tuple[str, bool], 'Language'] = {}

    def __init__(
        self,
        language: Optional[str] = None,
        extlangs: Optional[Sequence[str]] = None,
        script: Optional[str] = None,
        territory: Optional[str] = None,
        variants: Optional[Sequence[str]] = None,
        extensions: Optional[Sequence[str]] = None,
        private: Optional[str] = None,
    ):
        """
        The constructor for Language objects.

        It's inefficient to call this directly, because it can't return
        an existing instance. Instead, call Language.make(), which
        has the same signature.
        """
        self.language = language
        self.extlangs = extlangs
        self.script = script
        self.territory = territory
        self.variants = variants
        self.extensions = extensions
        self.private = private

        # Cached values
        self._simplified: 'Language' = None
        self._searchable: 'Language' = None
        self._broader: List[str] = None
        self._assumed: 'Language' = None
        self._filled: 'Language' = None
        self._macrolanguage: Optional['Language'] = None
        self._str_tag: str = None
        self._dict: dict = None
        self._disp_separator: str = None
        self._disp_pattern: str = None

        # Make sure the str_tag value is cached
        self.to_tag()

    @classmethod
    def make(
        cls,
        language: Optional[str] = None,
        extlangs: Optional[Sequence[str]] = None,
        script: Optional[str] = None,
        territory: Optional[str] = None,
        variants: Optional[Sequence[str]] = None,
        extensions: Optional[Sequence[str]] = None,
        private: Optional[str] = None,
    ) -> 'Language':
        """
        Create a Language object by giving any subset of its attributes.

        If this value has been created before, return the existing value.
        """
        values = (
            language,
            tuple(extlangs or ()),
            script,
            territory,
            tuple(variants or ()),
            tuple(extensions or ()),
            private,
        )
        if values in cls._INSTANCES:
            return cls._INSTANCES[values]

        instance = cls(
            language=language,
            extlangs=extlangs,
            script=script,
            territory=territory,
            variants=variants,
            extensions=extensions,
            private=private,
        )
        cls._INSTANCES[values] = instance
        return instance

    @staticmethod
    def get(tag: Union[str, 'Language'], normalize=True) -> 'Language':
        """
        Create a Language object from a language tag string.

        If normalize=True, non-standard or overlong tags will be replaced as
        they're interpreted. This is recommended.

        Here are several examples of language codes, which are also test cases.
        Most language codes are straightforward, but these examples will get
        pretty obscure toward the end.

        >>> Language.get('en-US')
        Language.make(language='en', territory='US')

        >>> Language.get('zh-Hant')
        Language.make(language='zh', script='Hant')

        >>> Language.get('und')
        Language.make()

        This function is idempotent, in case you already have a Language object:

        >>> Language.get(Language.get('en-us'))
        Language.make(language='en', territory='US')

        The non-code 'root' is sometimes used to represent the lack of any
        language information, similar to 'und'.

        >>> Language.get('root')
        Language.make()

        By default, getting a Language object will automatically convert
        deprecated tags:

        >>> Language.get('iw')
        Language.make(language='he')

        >>> Language.get('in')
        Language.make(language='id')

        One type of deprecated tag that should be replaced is for sign
        languages, which used to all be coded as regional variants of a
        fictitious global sign language called 'sgn'. Of course, there is no
        global sign language, so sign languages now have their own language
        codes.

        >>> Language.get('sgn-US')
        Language.make(language='ase')

        >>> Language.get('sgn-US', normalize=False)
        Language.make(language='sgn', territory='US')

        'en-gb-oed' is a tag that's grandfathered into the standard because it
        has been used to mean "spell-check this with Oxford English Dictionary
        spelling", but that tag has the wrong shape. We interpret this as the
        new standardized tag 'en-gb-oxendict', unless asked not to normalize.

        >>> Language.get('en-gb-oed')
        Language.make(language='en', territory='GB', variants=['oxendict'])

        >>> Language.get('en-gb-oed', normalize=False)
        Language.make(language='en-gb-oed')

        'zh-min-nan' is another oddly-formed tag, used to represent the
        Southern Min language, which includes Taiwanese as a regional form. It
        now has its own language code.

        >>> Language.get('zh-min-nan')
        Language.make(language='nan')

        The vague tag 'zh-min' is now also interpreted as 'nan', with a private
        extension indicating that it had a different form:

        >>> Language.get('zh-min')
        Language.make(language='nan', private='x-zh-min')

        Occasionally Wiktionary will use 'extlang' tags in strange ways, such
        as using the tag 'und-ibe' for some unspecified Iberian language.

        >>> Language.get('und-ibe')
        Language.make(extlangs=['ibe'])

        Here's an example of replacing multiple deprecated tags.

        The language tag 'sh' (Serbo-Croatian) ended up being politically
        problematic, and different standards took different steps to address
        this. The IANA made it into a macrolanguage that contains 'sr', 'hr',
        and 'bs'. Unicode further decided that it's a legacy tag that should
        be interpreted as 'sr-Latn', which the language matching rules say
        is mutually intelligible with all those languages.

        We complicate the example by adding on the territory tag 'QU', an old
        provisional tag for the European Union, which is now standardized as
        'EU'.

        >>> Language.get('sh-QU')
        Language.make(language='sr', script='Latn', territory='EU')
        """
        if isinstance(tag, Language):
            if not normalize:
                # shortcut: we have the tag already
                return tag

            # We might need to normalize this tag. Convert it back into a
            # string tag, to cover all the edge cases of normalization in a
            # way that we've already solved.
            tag = tag.to_tag()

        if (tag, normalize) in Language._PARSE_CACHE:
            return Language._PARSE_CACHE[tag, normalize]

        data: Dict[str, Any] = {}

        # If the complete tag appears as something to normalize, do the
        # normalization right away. Smash case and convert underscores to
        # hyphens when checking, because the case normalization that comes from
        # parse_tag() hasn't been applied yet.

        tag_lower = normalize_characters(tag)
        if normalize and tag_lower in LANGUAGE_REPLACEMENTS:
            tag = LANGUAGE_REPLACEMENTS[tag_lower]

        components = parse_tag(tag)

        for typ, value in components:
            if typ == 'extlang' and normalize and 'language' in data:
                # smash extlangs when possible
                minitag = f"{data['language']}-{value}"
                norm = LANGUAGE_REPLACEMENTS.get(normalize_characters(minitag))
                if norm is not None:
                    data.update(Language.get(norm, normalize).to_dict())
                else:
                    data.setdefault('extlangs', []).append(value)
            elif typ in {'extlang', 'variant', 'extension'}:
                data.setdefault(typ + 's', []).append(value)
            elif typ == 'language':
                if value == 'und':
                    pass
                elif normalize:
                    replacement = LANGUAGE_REPLACEMENTS.get(value.lower())
                    if replacement is not None:
                        # parse the replacement if necessary -- this helps with
                        # Serbian and Moldovan
                        data.update(Language.get(replacement, normalize).to_dict())
                    else:
                        data['language'] = value
                else:
                    data['language'] = value
            elif typ == 'territory':
                if normalize:
                    data['territory'] = TERRITORY_REPLACEMENTS.get(value.lower(), value)
                else:
                    data['territory'] = value
            elif typ == 'grandfathered':
                # If we got here, we got a grandfathered tag but we were asked
                # not to normalize it, or the CLDR data doesn't know how to
                # normalize it. The best we can do is set the entire tag as the
                # language.
                data['language'] = value
            else:
                data[typ] = value

        result = Language.make(**data)
        Language._PARSE_CACHE[tag, normalize] = result
        return result

    def to_tag(self) -> str:
        """
        Convert a Language back to a standard language tag, as a string.
        This is also the str() representation of a Language object.

        >>> Language.make(language='en', territory='GB').to_tag()
        'en-GB'

        >>> Language.make(language='yue', script='Hant', territory='HK').to_tag()
        'yue-Hant-HK'

        >>> Language.make(script='Arab').to_tag()
        'und-Arab'

        >>> str(Language.make(territory='IN'))
        'und-IN'
        """
        if self._str_tag is not None:
            return self._str_tag
        subtags = ['und']
        if self.language:
            subtags[0] = self.language
        if self.extlangs:
            for extlang in sorted(self.extlangs):
                subtags.append(extlang)
        if self.script:
            subtags.append(self.script)
        if self.territory:
            subtags.append(self.territory)
        if self.variants:
            for variant in sorted(self.variants):
                subtags.append(variant)
        if self.extensions:
            for ext in self.extensions:
                subtags.append(ext)
        if self.private:
            subtags.append(self.private)
        self._str_tag = '-'.join(subtags)
        return self._str_tag

    def simplify_script(self) -> 'Language':
        """
        Remove the script from some parsed language data, if the script is
        redundant with the language.

        >>> Language.make(language='en', script='Latn').simplify_script()
        Language.make(language='en')

        >>> Language.make(language='yi', script='Latn').simplify_script()
        Language.make(language='yi', script='Latn')

        >>> Language.make(language='yi', script='Hebr').simplify_script()
        Language.make(language='yi')
        """
        if self._simplified is not None:
            return self._simplified

        if self.language and self.script:
            if DEFAULT_SCRIPTS.get(self.language) == self.script:
                result = self.update_dict({'script': None})
                self._simplified = result
                return self._simplified

        self._simplified = self
        return self._simplified

    def assume_script(self) -> 'Language':
        """
        Fill in the script if it's missing, and if it can be assumed from the
        language subtag. This is the opposite of `simplify_script`.

        >>> Language.make(language='en').assume_script()
        Language.make(language='en', script='Latn')

        >>> Language.make(language='yi').assume_script()
        Language.make(language='yi', script='Hebr')

        >>> Language.make(language='yi', script='Latn').assume_script()
        Language.make(language='yi', script='Latn')

        This fills in nothing when the script cannot be assumed -- such as when
        the language has multiple scripts, or it has no standard orthography:

        >>> Language.make(language='sr').assume_script()
        Language.make(language='sr')

        >>> Language.make(language='eee').assume_script()
        Language.make(language='eee')

        It also dosn't fill anything in when the language is unspecified.

        >>> Language.make(territory='US').assume_script()
        Language.make(territory='US')
        """
        if self._assumed is not None:
            return self._assumed
        if self.language and not self.script:
            try:
                self._assumed = self.update_dict(
                    {'script': DEFAULT_SCRIPTS[self.language]}
                )
            except KeyError:
                self._assumed = self
        else:
            self._assumed = self
        return self._assumed

    def prefer_macrolanguage(self) -> 'Language':
        """
        BCP 47 doesn't specify what to do with macrolanguages and the languages
        they contain. The Unicode CLDR, on the other hand, says that when a
        macrolanguage has a dominant standardized language, the macrolanguage
        code should be used for that language. For example, Mandarin Chinese
        is 'zh', not 'cmn', according to Unicode, and Malay is 'ms', not 'zsm'.

        This isn't a rule you'd want to follow in all cases -- for example, you may
        want to be able to specifically say that 'ms' (the Malay macrolanguage)
        contains both 'zsm' (Standard Malay) and 'id' (Indonesian). But applying
        this rule helps when interoperating with the Unicode CLDR.

        So, applying `prefer_macrolanguage` to a Language object will
        return a new object, replacing the language with the macrolanguage if
        it is the dominant language within that macrolanguage. It will leave
        non-dominant languages that have macrolanguages alone.

        >>> Language.get('arb').prefer_macrolanguage()
        Language.make(language='ar')

        >>> Language.get('cmn-Hant').prefer_macrolanguage()
        Language.make(language='zh', script='Hant')

        >>> Language.get('yue-Hant').prefer_macrolanguage()
        Language.make(language='yue', script='Hant')
        """
        if self._macrolanguage is not None:
            return self._macrolanguage
        language = self.language or 'und'
        if language in NORMALIZED_MACROLANGUAGES:
            self._macrolanguage = self.update_dict(
                {'language': NORMALIZED_MACROLANGUAGES[language]}
            )
        else:
            self._macrolanguage = self
        return self._macrolanguage

    def to_alpha3(self, variant: str = 'T') -> str:
        """
        Get the three-letter language code for this language, even if it's
        canonically written with a two-letter code.

        These codes are the 'alpha3' codes defined by ISO 639-2.

        When this function returns, it always returns a 3-letter string. If
        there is no known alpha3 code for the language, it raises a LookupError.

        In cases where the distinction matters, we default to the 'terminology'
        code. You can pass `variant='B'` to get the 'bibliographic' code instead.
        For example, the terminology code for German is 'deu', while the
        bibliographic code is 'ger'.

        (The confusion between these two sets of codes is a good reason to avoid
        using alpha3 codes. Every language that has two different alpha3 codes
        also has an alpha2 code that's preferred, such as 'de' for German.)

        >>> Language.get('fr').to_alpha3()
        'fra'
        >>> Language.get('fr-CA').to_alpha3()
        'fra'
        >>> Language.get('fr').to_alpha3(variant='B')
        'fre'
        >>> Language.get('de').to_alpha3(variant='T')
        'deu'
        >>> Language.get('ja').to_alpha3()
        'jpn'
        >>> Language.get('un').to_alpha3()
        Traceback (most recent call last):
            ...
        LookupError: 'un' is not a known language code, and has no alpha3 code.


        All valid two-letter language codes have corresponding alpha3 codes,
        even the un-normalized ones. If they were assigned an alpha3 code by ISO
        before they were assigned a normalized code by CLDR, these codes may be
        different:

        >>> Language.get('tl', normalize=False).to_alpha3()
        'tgl'
        >>> Language.get('tl').to_alpha3()
        'fil'
        >>> Language.get('sh', normalize=False).to_alpha3()
        'hbs'


        Three-letter codes are preserved, even if they're unknown:

        >>> Language.get('qqq').to_alpha3()
        'qqq'
        >>> Language.get('und').to_alpha3()
        'und'
        """
        variant = variant.upper()
        if variant not in 'BT':
            raise ValueError("Variant must be 'B' or 'T'")

        language = self.language
        if language is None:
            return 'und'
        elif len(language) == 3:
            return language
        else:
            if variant == 'B' and language in LANGUAGE_ALPHA3_BIBLIOGRAPHIC:
                return LANGUAGE_ALPHA3_BIBLIOGRAPHIC[language]
            elif language in LANGUAGE_ALPHA3:
                return LANGUAGE_ALPHA3[language]
            else:
                raise LookupError(
                    f"{language!r} is not a known language code, "
                    "and has no alpha3 code."
                )

    def broader_tags(self) -> List[str]:
        """
        Iterate through increasingly general tags for this language.

        This isn't actually that useful for matching two arbitrary language tags
        against each other, but it is useful for matching them against a known
        standardized form, such as in the CLDR data.

        The list of broader versions to try appears in UTR 35, section 4.3,
        "Likely Subtags".

        >>> Language.get('nn-Latn-NO-x-thingy').broader_tags()
        ['nn-Latn-NO-x-thingy', 'nn-Latn-NO', 'nn-NO', 'nn-Latn', 'nn', 'und-Latn', 'und']

        >>> Language.get('arb-Arab').broader_tags()
        ['arb-Arab', 'ar-Arab', 'arb', 'ar', 'und-Arab', 'und']
        """
        if self._broader is not None:
            return self._broader
        self._broader = [self.to_tag()]
        seen = set([self.to_tag()])
        for keyset in self.BROADER_KEYSETS:
            for start_language in (self, self.prefer_macrolanguage()):
                filtered = start_language._filter_attributes(keyset)
                tag = filtered.to_tag()
                if tag not in seen:
                    self._broader.append(tag)
                    seen.add(tag)
        return self._broader

    def broaden(self) -> 'List[Language]':
        """
        Like `broader_tags`, but returrns Language objects instead of strings.
        """
        return [Language.get(tag) for tag in self.broader_tags()]

    def maximize(self) -> 'Language':
        """
        The Unicode CLDR contains a "likelySubtags" data file, which can guess
        reasonable values for fields that are missing from a language tag.

        This is particularly useful for comparing, for example, "zh-Hant" and
        "zh-TW", two common language tags that say approximately the same thing
        via rather different information. (Using traditional Han characters is
        not the same as being in Taiwan, but each implies that the other is
        likely.)

        These implications are provided in the CLDR supplemental data, and are
        based on the likelihood of people using the language to transmit text
        on the Internet. (This is why the overall default is English, not
        Chinese.)

        It's important to recognize that these tags amplify majorities, and
        that not all language support fits into a "likely" language tag.

        >>> str(Language.get('zh-Hant').maximize())
        'zh-Hant-TW'
        >>> str(Language.get('zh-TW').maximize())
        'zh-Hant-TW'
        >>> str(Language.get('ja').maximize())
        'ja-Jpan-JP'
        >>> str(Language.get('pt').maximize())
        'pt-Latn-BR'
        >>> str(Language.get('und-Arab').maximize())
        'ar-Arab-EG'
        >>> str(Language.get('und-CH').maximize())
        'de-Latn-CH'

        As many standards are, this is US-centric:

        >>> str(Language.make().maximize())
        'en-Latn-US'

        "Extlangs" have no likely-subtags information, so they will give
        maximized results that make no sense:

        >>> str(Language.get('und-ibe').maximize())
        'en-ibe-Latn-US'
        """
        if self._filled is not None:
            return self._filled

        for tag in self.broader_tags():
            if tag in LIKELY_SUBTAGS:
                result = Language.get(LIKELY_SUBTAGS[tag], normalize=False)
                result = result.update(self)
                self._filled = result
                return result

        raise RuntimeError(
            "Couldn't fill in likely values. This represents a problem with "
            "the LIKELY_SUBTAGS data."
        )

    # Support an old, wordier name for the method
    fill_likely_values = maximize

    def match_score(self, supported: 'Language') -> int:
        """
        DEPRECATED: use .distance() instead, which uses newer data and is _lower_
        for better matching languages.
        """
        warnings.warn(
            "`match_score` is deprecated because it's based on deprecated CLDR info. "
            "Use `distance` instead, which is _lower_ for better matching languages. ",
            DeprecationWarning,
        )
        return 100 - min(self.distance(supported), 100)

    def distance(self, supported: 'Language') -> int:
        """
        Suppose that `self` is the language that the user desires, and
        `supported` is a language that is actually supported.

        This method returns a number from 0 to 134 measuring the 'distance'
        between the languages (lower numbers are better). This is not a
        symmetric relation.

        The language distance is not really about the linguistic similarity or
        history of the languages; instead, it's based largely on sociopolitical
        factors, indicating which language speakers are likely to know which
        other languages in the present world. Much of the heuristic is about
        finding a widespread 'world language' like English, Chinese, French, or
        Russian that speakers of a more localized language will accept.

        A version that works on language tags, as strings, is in the function
        `tag_distance`. See that function for copious examples.
        """
        if supported == self:
            return 0

        # CLDR has realized that these matching rules are undermined when the
        # unspecified language 'und' gets maximized to 'en-Latn-US', so this case
        # is specifically not maximized:
        if self.language is None and self.script is None and self.territory is None:
            desired_triple = ('und', 'Zzzz', 'ZZ')
        else:
            desired_complete = self.prefer_macrolanguage().maximize()
            desired_triple = (
                desired_complete.language,
                desired_complete.script,
                desired_complete.territory,
            )

        if (
            supported.language is None
            and supported.script is None
            and supported.territory is None
        ):
            supported_triple = ('und', 'Zzzz', 'ZZ')
        else:
            supported_complete = supported.prefer_macrolanguage().maximize()
            supported_triple = (
                supported_complete.language,
                supported_complete.script,
                supported_complete.territory,
            )

        return tuple_distance_cached(desired_triple, supported_triple)

    def is_valid(self) -> bool:
        """
        Checks whether the language, script, territory, and variants
        (if present) are all tags that have meanings assigned by IANA.
        For example, 'ja' (Japanese) is a valid tag, and 'jp' is not.

        The data is current as of CLDR 40.

        >>> Language.get('ja').is_valid()
        True
        >>> Language.get('jp').is_valid()
        False
        >>> Language.get('en-001').is_valid()
        True
        >>> Language.get('en-000').is_valid()
        False
        >>> Language.get('en-Latn').is_valid()
        True
        >>> Language.get('en-Latnx').is_valid()
        False
        >>> Language.get('und').is_valid()
        True
        >>> Language.get('en-GB-oxendict').is_valid()
        True
        >>> Language.get('en-GB-oxenfree').is_valid()
        False
        >>> Language.get('x-heptapod').is_valid()
        True

        Some scripts are, confusingly, not included in CLDR's 'validity' pattern.
        If a script appears in the IANA registry, we consider it valid.

        >>> Language.get('ur-Aran').is_valid()
        True
        >>> Language.get('cu-Cyrs').is_valid()
        True

        A language tag with multiple extlangs will parse, but is not valid.
        The only allowed example is 'zh-min-nan', which normalizes to the
        language 'nan'.

        >>> Language.get('zh-min-nan').is_valid()
        True
        >>> Language.get('sgn-ase-bfi').is_valid()
        False

        These examples check that duplicate tags are not valid:

        >>> Language.get('de-1901').is_valid()
        True
        >>> Language.get('de-1901-1901').is_valid()
        False
        >>> Language.get('en-a-bbb-c-ddd').is_valid()
        True
        >>> Language.get('en-a-bbb-a-ddd').is_valid()
        False

        Of course, you should be prepared to catch a failure to parse the
        language code at all:

        >>> Language.get('C').is_valid()
        Traceback (most recent call last):
        ...
        langcodes.tag_parser.LanguageTagError: Expected a language code, got 'c'
        """
        if self.extlangs is not None:
            # An erratum to BCP 47 says that tags with more than one extlang are
            # invalid.
            if len(self.extlangs) > 1:
                return False

        subtags = [self.language, self.script, self.territory]
        checked_subtags = []
        if self.variants is not None:
            subtags.extend(self.variants)
        for subtag in subtags:
            if subtag is not None:
                checked_subtags.append(subtag)
                if not subtag.startswith('x-') and not VALIDITY.match(subtag):
                    if subtag not in ALL_SCRIPTS:
                        return False

        # We check extensions for validity by ensuring that there aren't
        # two extensions introduced by the same letter. For example, you can't
        # have two 'u-' extensions.
        if self.extensions:
            checked_subtags.extend([extension[:2] for extension in self.extensions])
        if len(set(checked_subtags)) != len(checked_subtags):
            return False
        return True

    def has_name_data(self) -> bool:
        """
        Return True when we can name languages in this language. Requires
        `language_data` to be installed.

        This is true when the language, or one of its 'broader' versions, is in
        the list of CLDR target languages.

        >>> Language.get('fr').has_name_data()
        True
        >>> Language.get('so').has_name_data()
        True
        >>> Language.get('enc').has_name_data()
        False
        >>> Language.get('und').has_name_data()
        False
        """
        try:
            from language_data.name_data import LANGUAGES_WITH_NAME_DATA
        except ImportError:
            print(LANGUAGE_NAME_IMPORT_MESSAGE, file=sys.stdout)
            raise

        matches = set(self.broader_tags()) & LANGUAGES_WITH_NAME_DATA
        return bool(matches)

    # These methods help to show what the language tag means in natural
    # language. They actually apply the language-matching algorithm to find
    # the right language to name things in.

    def _get_name(
        self, attribute: str, language: Union[str, 'Language'], max_distance: int
    ) -> str:
        try:
            from language_data.names import code_to_names
        except ImportError:
            print(LANGUAGE_NAME_IMPORT_MESSAGE, file=sys.stdout)
            raise

        assert attribute in self.ATTRIBUTES
        if isinstance(language, str):
            language = Language.get(language)

        attr_value = getattr(self, attribute)
        if attr_value is None:
            if attribute == 'language':
                attr_value = 'und'
            else:
                return None
        names = code_to_names(attr_value)

        result = self._best_name(names, language, max_distance)
        if result is not None:
            return result
        else:
            # Construct a string like "Unknown language [zzz]"
            placeholder = None
            if attribute == 'language':
                placeholder = 'und'
            elif attribute == 'script':
                placeholder = 'Zzzz'
            elif attribute == 'territory':
                placeholder = 'ZZ'

            unknown_name = None
            if placeholder is not None:
                names = code_to_names(placeholder)
                unknown_name = self._best_name(names, language, max_distance)
            if unknown_name is None:
                unknown_name = 'Unknown language subtag'
            return f'{unknown_name} [{attr_value}]'

    def _best_name(
        self, names: Mapping[str, str], language: 'Language', max_distance: int
    ):
        matchable_languages = set(language.broader_tags())
        possible_languages = [
            key for key in sorted(names.keys()) if key in matchable_languages
        ]

        target_language, score = closest_match(
            language, possible_languages, max_distance
        )
        if target_language in names:
            return names[target_language]
        else:
            return names.get(DEFAULT_LANGUAGE)

    def language_name(
        self,
        language: Union[str, 'Language'] = DEFAULT_LANGUAGE,
        max_distance: int = 25,
    ) -> str:
        """
        Give the name of the language (not the entire tag, just the language part)
        in a natural language. The target language can be given as a string or
        another Language object.

        By default, things are named in English:

        >>> Language.get('fr').language_name()
        'French'
        >>> Language.get('el').language_name()
        'Greek'

        But you can ask for language names in numerous other languages:

        >>> Language.get('fr').language_name('fr')
        'français'
        >>> Language.get('el').language_name('fr')
        'grec'

        Why does everyone get Slovak and Slovenian confused? Let's ask them.

        >>> Language.get('sl').language_name('sl')
        'slovenščina'
        >>> Language.get('sk').language_name('sk')
        'slovenčina'
        >>> Language.get('sl').language_name('sk')
        'slovinčina'
        >>> Language.get('sk').language_name('sl')
        'slovaščina'
        """
        return self._get_name('language', language, max_distance)

    def display_name(
        self,
        language: Union[str, 'Language'] = DEFAULT_LANGUAGE,
        max_distance: int = 25,
    ) -> str:
        """
        It's often helpful to be able to describe a language code in a way that a user
        (or you) can understand, instead of in inscrutable short codes. The
        `display_name` method lets you describe a Language object *in a language*.

        The `.display_name(language, min_score)` method will look up the name of the
        language. The names come from the IANA language tag registry, which is only in
        English, plus CLDR, which names languages in many commonly-used languages.

        The default language for naming things is English:

            >>> Language.make(language='fr').display_name()
            'French'

            >>> Language.make().display_name()
            'Unknown language'

            >>> Language.get('zh-Hans').display_name()
            'Chinese (Simplified)'

            >>> Language.get('en-US').display_name()
            'English (United States)'

        But you can ask for language names in numerous other languages:

            >>> Language.get('fr').display_name('fr')
            'français'

            >>> Language.get('fr').display_name('es')
            'francés'

            >>> Language.make().display_name('es')
            'lengua desconocida'

            >>> Language.get('zh-Hans').display_name('de')
            'Chinesisch (Vereinfacht)'

            >>> Language.get('en-US').display_name('zh-Hans')
            '英语（美国）'
        """
        reduced = self.simplify_script()
        language = Language.get(language)
        language_name = reduced.language_name(language, max_distance)
        extra_parts = []

        if reduced.script is not None:
            extra_parts.append(reduced.script_name(language, max_distance))
        if reduced.territory is not None:
            extra_parts.append(reduced.territory_name(language, max_distance))

        if extra_parts:
            clarification = language._display_separator().join(extra_parts)
            pattern = language._display_pattern()
            return pattern.format(language_name, clarification)
        else:
            return language_name

    def _display_pattern(self) -> str:
        """
        Get the pattern, according to CLDR, that should be used for clarifying
        details of a language code.
        """
        # Technically we are supposed to look up this pattern in each language.
        # Practically, it's the same in every language except Chinese, where the
        # parentheses are full-width.
        if self._disp_pattern is not None:
            return self._disp_pattern
        if self.distance(Language.get('zh')) <= 25:
            self._disp_pattern = "{0}（{1}）"
        else:
            self._disp_pattern = "{0} ({1})"
        return self._disp_pattern

    def _display_separator(self) -> str:
        """
        Get the symbol that should be used to separate multiple clarifying
        details -- such as a comma in English, or an ideographic comma in
        Japanese.

        Requires that `language_data` is installed.
        """
        try:
            from language_data.names import DISPLAY_SEPARATORS
        except ImportError:
            print(LANGUAGE_NAME_IMPORT_MESSAGE, file=sys.stdout)
            raise

        if self._disp_separator is not None:
            return self._disp_separator
        matched, _dist = closest_match(self, DISPLAY_SEPARATORS.keys())
        self._disp_separator = DISPLAY_SEPARATORS[matched]
        return self._disp_separator

    def autonym(self, max_distance: int = 9) -> str:
        """
        Give the display name of this language *in* this language.
        Requires that `language_data` is installed.

        >>> Language.get('fr').autonym()
        'français'
        >>> Language.get('es').autonym()
        'español'
        >>> Language.get('ja').autonym()
        '日本語'

        This uses the `display_name()` method, so it can include the name of a
        script or territory when appropriate.

        >>> Language.get('en-AU').autonym()
        'English (Australia)'
        >>> Language.get('sr-Latn').autonym()
        'srpski (latinica)'
        >>> Language.get('sr-Cyrl').autonym()
        'српски (ћирилица)'
        >>> Language.get('pa').autonym()
        'ਪੰਜਾਬੀ'
        >>> Language.get('pa-Arab').autonym()
        'پنجابی (عربی)'

        This only works for language codes that CLDR has locale data for. You
        can't ask for the autonym of 'ja-Latn' and get 'nihongo (rōmaji)'.
        """
        lang = self.prefer_macrolanguage()
        return lang.display_name(language=lang, max_distance=max_distance)

    def script_name(
        self,
        language: Union[str, 'Language'] = DEFAULT_LANGUAGE,
        max_distance: int = 25,
    ) -> str:
        """
        Describe the script part of the language tag in a natural language.
        Requires that `language_data` is installed.
        """
        return self._get_name('script', language, max_distance)

    def territory_name(
        self,
        language: Union[str, 'Language'] = DEFAULT_LANGUAGE,
        max_distance: int = 25,
    ) -> str:
        """
        Describe the territory part of the language tag in a natural language.
        Requires that `language_data` is installed.
        """
        return self._get_name('territory', language, max_distance)

    def region_name(
        self,
        language: Union[str, 'Language'] = DEFAULT_LANGUAGE,
        max_distance: int = 25,
    ) -> str:
        warnings.warn(
            "`region_name` has been renamed to `territory_name` for consistency",
            DeprecationWarning,
        )
        return self.territory_name(language, max_distance)

    @property
    def region(self):
        warnings.warn(
            "The `region` property has been renamed to `territory` for consistency",
            DeprecationWarning,
        )
        return self.territory

    def variant_names(
        self,
        language: Union[str, 'Language'] = DEFAULT_LANGUAGE,
        max_distance: int = 25,
    ) -> Sequence[str]:
        """
        Deprecated in version 3.0.

        We don't store names for variants anymore, so this just returns the list
        of variant codes, such as ['oxendict'] for en-GB-oxendict.
        """
        warnings.warn(
            "variant_names is deprecated and just returns the variant codes",
            DeprecationWarning,
        )
        return self.variants or []

    def describe(
        self,
        language: Union[str, 'Language'] = DEFAULT_LANGUAGE,
        max_distance: int = 25,
    ) -> dict:
        """
        Return a dictionary that describes a given language tag in a specified
        natural language. Requires that `language_data` is installed.

        See `language_name` and related methods for more specific versions of this.

        The desired `language` will in fact be matched against the available
        options using the matching technique that this module provides. We can
        illustrate many aspects of this by asking for a description of Shavian
        script (a phonetic script for English devised by author George Bernard
        Shaw), and where you might find it, in various languages.

        >>> shaw = Language.make(script='Shaw').maximize()
        >>> shaw.describe('en')
        {'language': 'English', 'script': 'Shavian', 'territory': 'United Kingdom'}

        >>> shaw.describe('fr')
        {'language': 'anglais', 'script': 'shavien', 'territory': 'Royaume-Uni'}

        >>> shaw.describe('es')
        {'language': 'inglés', 'script': 'shaviano', 'territory': 'Reino Unido'}

        >>> shaw.describe('pt')
        {'language': 'inglês', 'script': 'shaviano', 'territory': 'Reino Unido'}

        >>> shaw.describe('uk')
        {'language': 'англійська', 'script': 'шоу', 'territory': 'Велика Британія'}

        >>> shaw.describe('arb')
        {'language': 'الإنجليزية', 'script': 'الشواني', 'territory': 'المملكة المتحدة'}

        >>> shaw.describe('th')
        {'language': 'อังกฤษ', 'script': 'ซอเวียน', 'territory': 'สหราชอาณาจักร'}

        >>> shaw.describe('zh-Hans')
        {'language': '英语', 'script': '萧伯纳式文', 'territory': '英国'}

        >>> shaw.describe('zh-Hant')
        {'language': '英文', 'script': '簫柏納字符', 'territory': '英國'}

        >>> shaw.describe('ja')
        {'language': '英語', 'script': 'ショー文字', 'territory': 'イギリス'}

        When we don't have a localization for the language, we fall back on English,
        because the IANA provides names for all known codes in English.

        >>> shaw.describe('lol')
        {'language': 'English', 'script': 'Shavian', 'territory': 'United Kingdom'}

        When the language tag itself is a valid tag but with no known meaning, we
        say so in the appropriate language.

        >>> Language.get('xyz-ZY').display_name()
        'Unknown language [xyz] (Unknown Region [ZY])'

        >>> Language.get('xyz-ZY').display_name('es')
        'lengua desconocida [xyz] (Región desconocida [ZY])'
        """
        names = {}
        if self.language:
            names['language'] = self.language_name(language, max_distance)
        if self.script:
            names['script'] = self.script_name(language, max_distance)
        if self.territory:
            names['territory'] = self.territory_name(language, max_distance)
        return names

    def speaking_population(self) -> int:
        """
        Get an estimate of how many people in the world speak this language,
        derived from CLDR data. Requires that `language_data` is installed.

        Only the language and territory codes will be considered. If a
        territory code is included, the population will count only the
        speakers of the language in that territory.

        Script subtags are disregarded, because it doesn't make sense to ask
        how many people speak in a particular writing script.

        >>> Language.get('es').speaking_population()
        487664083
        >>> Language.get('pt').speaking_population()
        237135429
        >>> Language.get('es-BR').speaking_population()
        76218
        >>> Language.get('pt-BR').speaking_population()
        192661560
        >>> Language.get('vo').speaking_population()
        0
        """
        try:
            from language_data.population_data import LANGUAGE_SPEAKING_POPULATION
        except ImportError:
            print(LANGUAGE_NAME_IMPORT_MESSAGE, file=sys.stdout)
            raise

        lang = self._filter_attributes(['language', 'territory'])
        return LANGUAGE_SPEAKING_POPULATION.get(str(lang), 0)

    def writing_population(self) -> int:
        """
        Get an estimate of how many people in the world read and write
        this language, derived from CLDR data. Requires that `language_data`
        is installed.

        For many languages that aren't typically written, this is an
        overestimate, according to CLDR -- the data often includes people who
        speak that language but write in a different language.

        Only the language, script, and territory codes will be considered.
        If a territory code is included, the population will count only the
        speakers of the language in that territory.

        >>> all = Language.get('zh').writing_population()
        >>> all
        1240326057

        >>> traditional = Language.get('zh-Hant').writing_population()
        >>> traditional
        37019589

        >>> simplified = Language.get('zh-Hans').writing_population()
        >>> all == traditional + simplified
        True

        >>> Language.get('zh-Hant-HK').writing_population()
        6439733
        >>> Language.get('zh-Hans-HK').writing_population()
        338933

        Note that if you want to get the total Chinese writing population
        of Hong Kong, you need to avoid normalization that would interpret
        'zh-HK' as 'zh-Hant-HK'.

        >>> Language.get('zh-HK', normalize=False).writing_population()
        6778666

        Unknown or unspecified language codes get a population of 0.

        >>> Language.get('xyz').writing_population()
        0

        >>> Language.get('und').writing_population()
        0
        """
        try:
            from language_data.population_data import LANGUAGE_WRITING_POPULATION
        except ImportError:
            print(LANGUAGE_NAME_IMPORT_MESSAGE, file=sys.stdout)
            raise

        lang = self._filter_attributes(['language', 'script', 'territory'])
        if str(lang) in LANGUAGE_WRITING_POPULATION:
            return LANGUAGE_WRITING_POPULATION[str(lang)]
        else:
            lang = lang.simplify_script()
            return LANGUAGE_WRITING_POPULATION.get(str(lang), 0)

    @staticmethod
    def find_name(
        tagtype: str, name: str, language: Optional[Union[str, 'Language']] = None
    ) -> 'Language':
        """
        Find the subtag of a particular `tagtype` that has the given `name`.
        Requires that `language_data` is installed.

        The default language, "und", will allow matching names in any language,
        so you can get the code 'fr' by looking up "French", "Français", or
        "francés".

        Occasionally, names are ambiguous in a way that can be resolved by
        specifying what name the language is supposed to be in. For example,
        there is a language named 'Malayo' in English, but it's different from
        the language named 'Malayo' in Spanish (which is Malay). Specifying the
        language will look up the name in a trie that is only in that language.

        In a previous version, we thought we were going to deprecate the
        `language` parameter, as there weren't significant cases of conflicts
        in names of things between languages. Well, we got more data, and
        conflicts in names are everywhere.

        Specifying the language that the name should be in is still not
        required, but it will help to make sure that names can be
        round-tripped.

        >>> Language.find_name('language', 'francés')
        Language.make(language='fr')

        >>> Language.find_name('territory', 'United Kingdom')
        Language.make(territory='GB')

        >>> Language.find_name('script', 'Arabic')
        Language.make(script='Arab')

        >>> Language.find_name('language', 'norsk bokmål')
        Language.make(language='nb')

        >>> Language.find_name('language', 'norsk')
        Language.make(language='no')

        >>> Language.find_name('language', 'norsk', 'en')
        Traceback (most recent call last):
            ...
        LookupError: Can't find any language named 'norsk'

        >>> Language.find_name('language', 'norsk', 'no')
        Language.make(language='no')

        >>> Language.find_name('language', 'malayo', 'en')
        Language.make(language='mbp')

        >>> Language.find_name('language', 'malayo', 'es')
        Language.make(language='ms')

        Some langauge names resolve to more than a language. For example,
        the name 'Brazilian Portuguese' resolves to a language and a territory,
        and 'Simplified Chinese' resolves to a language and a script. In these
        cases, a Language object with multiple subtags will be returned.

        >>> Language.find_name('language', 'Brazilian Portuguese', 'en')
        Language.make(language='pt', territory='BR')

        >>> Language.find_name('language', 'Simplified Chinese', 'en')
        Language.make(language='zh', script='Hans')

        A small amount of fuzzy matching is supported: if the name can be
        shortened to match a single language name, you get that language.
        This allows, for example, "Hakka dialect" to match "Hakka".

        >>> Language.find_name('language', 'Hakka dialect')
        Language.make(language='hak')
        """
        try:
            from language_data.names import name_to_code
        except ImportError:
            print(LANGUAGE_NAME_IMPORT_MESSAGE, file=sys.stdout)
            raise

        # No matter what form of language we got, normalize it to a single
        # language subtag
        if isinstance(language, Language):
            language = language.language
        elif isinstance(language, str):
            language = get(language).language
        if language is None:
            language = 'und'

        code = name_to_code(tagtype, name, language)
        if code is None:
            raise LookupError(f"Can't find any {tagtype} named {name!r}")
        if '-' in code:
            return Language.get(code)
        else:
            data = {tagtype: code}
            return Language.make(**data)

    @staticmethod
    def find(
        name: str, language: Optional[Union[str, 'Language']] = None
    ) -> 'Language':
        """
        A concise version of `find_name`, used to get a language tag by its
        name in a natural language. The language can be omitted in the large
        majority of cases, where the language name is not ambiguous.

        >>> Language.find('Türkçe')
        Language.make(language='tr')
        >>> Language.find('brazilian portuguese')
        Language.make(language='pt', territory='BR')
        >>> Language.find('simplified chinese')
        Language.make(language='zh', script='Hans')

        Some language names are ambiguous: for example, there is a language
        named 'Fala' in English (with code 'fax'), but 'Fala' is also the
        Kwasio word for French. In this case, specifying the language that
        the name is in is necessary for disambiguation.

        >>> Language.find('fala')
        Language.make(language='fr')
        >>> Language.find('fala', 'nmg')
        Language.make(language='fr')
        >>> Language.find('fala', 'en')
        Language.make(language='fax')
        """
        return Language.find_name('language', name, language)

    def to_dict(self) -> dict:
        """
        Get a dictionary of the attributes of this Language object, which
        can be useful for constructing a similar object.
        """
        if self._dict is not None:
            return self._dict

        result = {}
        for key in self.ATTRIBUTES:
            value = getattr(self, key)
            if value:
                result[key] = value
        self._dict = result
        return result

    def update(self, other: 'Language') -> 'Language':
        """
        Update this Language with the fields of another Language.
        """
        return Language.make(
            language=other.language or self.language,
            extlangs=other.extlangs or self.extlangs,
            script=other.script or self.script,
            territory=other.territory or self.territory,
            variants=other.variants or self.variants,
            extensions=other.extensions or self.extensions,
            private=other.private or self.private,
        )

    def update_dict(self, newdata: dict) -> 'Language':
        """
        Update the attributes of this Language from a dictionary.
        """
        return Language.make(
            language=newdata.get('language', self.language),
            extlangs=newdata.get('extlangs', self.extlangs),
            script=newdata.get('script', self.script),
            territory=newdata.get('territory', self.territory),
            variants=newdata.get('variants', self.variants),
            extensions=newdata.get('extensions', self.extensions),
            private=newdata.get('private', self.private),
        )

    @staticmethod
    def _filter_keys(d: dict, keys: Iterable[str]) -> dict:
        """
        Select a subset of keys from a dictionary.
        """
        return {key: d[key] for key in keys if key in d}

    def _filter_attributes(self, keyset: Iterable[str]) -> 'Language':
        """
        Return a copy of this object with a subset of its attributes set.
        """
        filtered = self._filter_keys(self.to_dict(), keyset)
        return Language.make(**filtered)

    def _searchable_form(self) -> 'Language':
        """
        Convert a parsed language tag so that the information it contains is in
        the best form for looking up information in the CLDR.
        """
        if self._searchable is not None:
            return self._searchable

        self._searchable = (
            self._filter_attributes({'language', 'script', 'territory'})
            .simplify_script()
            .prefer_macrolanguage()
        )
        return self._searchable

    def __eq__(self, other):
        if self is other:
            return True
        if not isinstance(other, Language):
            return False
        return self._str_tag == other._str_tag

    def __hash__(self) -> int:
        return hash(id(self))

    def __getitem__(self, key: str) -> Optional[Union[str, List[str]]]:
        if key in self.ATTRIBUTES:
            return getattr(self, key)
        else:
            raise KeyError(key)

    def __contains__(self, key: str) -> bool:
        return key in self.ATTRIBUTES and getattr(self, key)

    def __repr__(self) -> str:
        items = []
        for attr in self.ATTRIBUTES:
            if getattr(self, attr):
                value = getattr(self, attr)
                items.append(f'{attr}={value!r}')
        joined = ', '.join(items)
        return f"Language.make({joined})"

    def __str__(self) -> str:
        return self.to_tag()


# Make the get(), find(), and find_name() functions available at the top level
get = Language.get
find = Language.find
find_name = Language.find_name

# Make the Language object available under the old name LanguageData
LanguageData = Language


def standardize_tag(tag: Union[str, Language], macro: bool = False) -> str:
    """
    Standardize a language tag:

    - Replace deprecated values with their updated versions (if those exist)
    - Remove script tags that are redundant with the language
    - If *macro* is True, use a macrolanguage to represent the most common
      standardized language within that macrolanguage. For example, 'cmn'
      (Mandarin) becomes 'zh' (Chinese), and 'arb' (Modern Standard Arabic)
      becomes 'ar' (Arabic).
    - Format the result according to the conventions of BCP 47

    Macrolanguage replacement is not required by BCP 47, but it is required
    by the Unicode CLDR.

    >>> standardize_tag('en_US')
    'en-US'

    >>> standardize_tag('en-Latn')
    'en'

    >>> standardize_tag('en-uk')
    'en-GB'

    >>> standardize_tag('eng')
    'en'

    >>> standardize_tag('arb-Arab', macro=True)
    'ar'

    >>> standardize_tag('sh-QU')
    'sr-Latn-EU'

    >>> standardize_tag('sgn-US')
    'ase'

    >>> standardize_tag('zh-cmn-hans-cn')
    'zh-Hans-CN'

    >>> standardize_tag('zsm', macro=True)
    'ms'

    >>> standardize_tag('ja-latn-hepburn')
    'ja-Latn-hepburn'

    >>> standardize_tag('spa-latn-mx')
    'es-MX'

    If the tag can't be parsed according to BCP 47, this will raise a
    LanguageTagError (a subclass of ValueError):

    >>> standardize_tag('spa-mx-latn')
    Traceback (most recent call last):
        ...
    langcodes.tag_parser.LanguageTagError: This script subtag, 'latn', is out of place. Expected variant, extension, or end of string.
    """
    langdata = Language.get(tag, normalize=True)
    if macro:
        langdata = langdata.prefer_macrolanguage()

    return langdata.simplify_script().to_tag()


def tag_is_valid(tag: Union[str, Language]) -> bool:
    """
    Determines whether a string is a valid language tag. This is similar to
    Language.get(tag).is_valid(), but can return False in the case where
    the tag doesn't parse.

    >>> tag_is_valid('ja')
    True
    >>> tag_is_valid('jp')
    False
    >>> tag_is_valid('spa-Latn-MX')
    True
    >>> tag_is_valid('spa-MX-Latn')
    False
    >>> tag_is_valid('')
    False
    >>> tag_is_valid('C.UTF-8')
    False
    """
    try:
        langdata = Language.get(tag)
        return langdata.is_valid()
    except LanguageTagError:
        return False


def tag_match_score(
    desired: Union[str, Language], supported: Union[str, Language]
) -> int:
    """
    DEPRECATED: use .distance() instead, which uses newer data and is _lower_
    for better matching languages.

    Return a number from 0 to 100 indicating the strength of match between the
    language the user desires, D, and a supported language, S. Higher numbers
    are better. A reasonable cutoff for not messing with your users is to
    only accept scores of 75 or more.

    A score of 100 means the languages are the same, possibly after normalizing
    and filling in likely values.
    """
    warnings.warn(
        "tag_match_score is deprecated because it's based on deprecated CLDR info. "
        "Use tag_distance instead, which is _lower_ for better matching languages. ",
        DeprecationWarning,
    )
    desired_ld = Language.get(desired)
    supported_ld = Language.get(supported)
    return desired_ld.match_score(supported_ld)


def tag_distance(desired: Union[str, Language], supported: Union[str, Language]) -> int:
    """
    Tags that expand to the same thing when likely values are filled in get a
    distance of 0.

    >>> tag_distance('en', 'en')
    0
    >>> tag_distance('en', 'en-US')
    0
    >>> tag_distance('zh-Hant', 'zh-TW')
    0
    >>> tag_distance('ru-Cyrl', 'ru')
    0

    As a specific example, Serbo-Croatian is a politically contentious idea,
    but in CLDR, it's considered equivalent to Serbian in Latin characters.

    >>> tag_distance('sh', 'sr-Latn')
    0

    ... which is very similar to Croatian but sociopolitically not the same.

    >>> tag_distance('sh', 'hr')
    9

    Unicode reorganized its distinction between 'no' (Norwegian) and 'nb'
    (Norwegian Bokmål) in 2021. 'no' is preferred in most contexts, and the more
    specific 'nb' is a distance of 1 from it:

    >>> tag_distance('nb', 'no')
    1

    These distances can be asymmetrical: this data includes the fact that speakers
    of Swiss German (gsw) know High German (de), but not at all the other way around.

    The difference seems a little bit extreme, but the asymmetry is certainly
    there. And if your text is tagged as 'gsw', it must be that way for a
    reason.

    >>> tag_distance('gsw', 'de')
    8
    >>> tag_distance('de', 'gsw')
    84

    Unconnected languages get a distance of 80 to 134.

    >>> tag_distance('en', 'zh')
    134
    >>> tag_distance('es', 'fr')
    84
    >>> tag_distance('fr-CH', 'de-CH')
    80

    Different local variants of the same language get a distance from 3 to 5.
    >>> tag_distance('zh-HK', 'zh-MO')   # Chinese is similar in Hong Kong and Macao
    4
    >>> tag_distance('en-AU', 'en-GB')   # Australian English is similar to British English
    3
    >>> tag_distance('en-IN', 'en-GB')   # Indian English is also similar to British English
    3
    >>> tag_distance('es-PE', 'es-419')  # Peruvian Spanish is Latin American Spanish
    1
    >>> tag_distance('es-419', 'es-PE')  # but Latin American Spanish is not necessarily Peruvian
    4
    >>> tag_distance('es-ES', 'es-419')  # Spanish in Spain is further from Latin American Spanish
    5
    >>> tag_distance('en-US', 'en-GB')   # American and British English are somewhat different
    5
    >>> tag_distance('es-MX', 'es-ES')   # Mexican Spanish is different from Spanish Spanish
    5
    >>> # European Portuguese is different from the most common form (Brazilian Portuguese)
    >>> tag_distance('pt', 'pt-PT')
    5

    >>> # Serbian has two scripts, and people might prefer one but understand both
    >>> tag_distance('sr-Latn', 'sr-Cyrl')
    5

    A distance of 10 is used for matching a specific language to its
    more-commonly-used macrolanguage tag.

    >>> tag_distance('arz', 'ar')  # Egyptian Arabic to Modern Standard Arabic
    10
    >>> tag_distance('wuu', 'zh')  # Wu Chinese to (Mandarin) Chinese
    10

    Higher distances can arrive due to particularly contentious differences in
    the script for writing the language, where people who understand one script
    can learn the other but may not be happy with it. This specifically applies
    to Chinese.

    >>> tag_distance('zh-TW', 'zh-CN')
    54
    >>> tag_distance('zh-Hans', 'zh-Hant')
    54
    >>> tag_distance('zh-CN', 'zh-HK')
    54
    >>> tag_distance('zh-CN', 'zh-TW')
    54
    >>> tag_distance('zh-Hant', 'zh-Hans')
    54

    This distance range also applies to the differences between Norwegian
    Bokmål, Nynorsk, and Danish.

    >>> tag_distance('no', 'da')
    12
    >>> tag_distance('no', 'nn')
    20

    Differences of 20 to 50 can represent substantially different languages,
    in cases where speakers of the first may understand the second for demographic
    reasons.

    >>> tag_distance('eu', 'es')  # Basque to Spanish
    20
    >>> tag_distance('af', 'nl')  # Afrikaans to Dutch
    24
    >>> tag_distance('mr', 'hi')  # Marathi to Hindi
    30
    >>> tag_distance('ms', 'id')  # Malay to Indonesian
    34
    >>> tag_distance('mg', 'fr')  # Malagasy to French
    34
    >>> tag_distance('ta', 'en')  # Tamil to English
    44

    A complex example is the tag 'yue' for Cantonese. Written Chinese is usually
    presumed to be Mandarin Chinese, but colloquial Cantonese can be written as
    well. (Some things could not be written any other way, such as Cantonese
    song lyrics.)

    The difference between Cantonese and Mandarin also implies script and
    territory differences by default, adding to the distance.

    >>> tag_distance('yue', 'zh')
    64

    When the supported script is a different one than desired, this is usually
    a major difference with score of 50 or more.

    >>> tag_distance('ja', 'ja-Latn-US-hepburn')
    54

    >>> # You can read the Shavian script, right?
    >>> tag_distance('en', 'en-Shaw')
    54
    """
    desired_obj = Language.get(desired)
    supported_obj = Language.get(supported)
    return desired_obj.distance(supported_obj)


def best_match(
    desired_language: Union[str, Language],
    supported_languages: Sequence[str],
    min_score: int = 75,
) -> Tuple[str, int]:
    """
    DEPRECATED: use .closest_match() instead. This function emulates the old
    matching behavior by subtracting the language distance from 100.

    You have software that supports any of the `supported_languages`. You want
    to use `desired_language`. This function lets you choose the right language,
    even if there isn't an exact match.

    Returns:

    - The best-matching language code, which will be one of the
      `supported_languages` or 'und'
    - The score of the match, from 0 to 100; higher is better.

    `min_score` sets the minimum match score. If all languages match with a lower
    score than that, the result will be 'und' with a score of 0.
    """
    max_distance = 100 - min_score
    supported, distance = closest_match(
        desired_language, supported_languages, max_distance
    )
    score = max(0, 100 - distance)
    return supported, score


def closest_match(
    desired_language: Union[str, Language],
    supported_languages: Sequence[str],
    max_distance: int = 25,
) -> Tuple[str, int]:
    """
    You have software that supports any of the `supported_languages`. You want
    to use `desired_language`. This function lets you choose the right language,
    even if there isn't an exact match.

    Returns:

    - The best-matching language code, which will be one of the
      `supported_languages` or 'und' for no match
    - The distance of the match, which is 0 for a perfect match and increases
      from there (see `tag_distance`)

    `max_distance` sets the maximum match distance. If all matches are farther
    than that, the result will be 'und' with a distance of 1000. The default
    value is 25, and raising it can cause data to be processed in significantly
    the wrong language. The documentation for `tag_distance` describes the
    distance values in more detail.

    When there is a tie for the best matching language, the first one in the
    tie will be used.

    >>> closest_match('fr', ['de', 'en', 'fr'])
    ('fr', 0)

    >>> closest_match('pt', ['pt-BR', 'pt-PT'])
    ('pt-BR', 0)

    >>> closest_match('en-AU', ['en-GB', 'en-US'])
    ('en-GB', 3)

    >>> closest_match('af', ['en', 'nl', 'zu'])
    ('nl', 24)

    >>> closest_match('ja', ['ja-Latn-hepburn', 'en'])
    ('und', 1000)
    """
    desired_language = str(desired_language)

    # Quickly return if the desired language is directly supported
    if desired_language in supported_languages:
        return desired_language, 0

    # Reduce the desired language to a standard form that could also match
    desired_language = standardize_tag(desired_language)
    if desired_language in supported_languages:
        return desired_language, 0

    match_distances = [
        (supported, tag_distance(desired_language, supported))
        for supported in supported_languages
    ]
    match_distances = [
        (supported, distance)
        for (supported, distance) in match_distances
        if distance <= max_distance
    ] + [('und', 1000)]

    match_distances.sort(key=itemgetter(1))
    return match_distances[0]


def closest_supported_match(
    desired_language: Union[str, Language],
    supported_languages: Sequence[str],
    max_distance: int = 25,
) -> Optional[str]:
    """
    Wraps `closest_match` with a simpler return type. Returns the language
    tag of the closest match if there is one, or None if there is not.

    >>> closest_supported_match('fr', ['de', 'en', 'fr'])
    'fr'

    >>> closest_supported_match('pt', ['pt-BR', 'pt-PT'])
    'pt-BR'

    >>> closest_supported_match('en-AU', ['en-GB', 'en-US'])
    'en-GB'

    >>> closest_supported_match('und', ['en', 'und'])
    'und'

    >>> closest_supported_match('af', ['en', 'nl', 'zu'])
    'nl'

    >>> print(closest_supported_match('af', ['en', 'nl', 'zu'], max_distance=10))
    None
    """
    code, distance = closest_match(desired_language, supported_languages, max_distance)
    if distance == 1000:
        return None
    else:
        return code