dateparser 1.2.1__py3-none-any.whl

dateparser_scripts/order_languages.py

@@ -0,0 +1,217 @@
+import json
+import os
+from collections import OrderedDict
+
+import regex as re
+import requests
+from parsel import Selector
+
+from dateparser_scripts.utils import get_raw_data
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+# Languages with insufficient translation data are excluded
+avoid_languages = {"cu", "kkj", "nds", "prg", "tk", "vai", "vai-Latn", "vai-Vaii", "vo"}
+
+
+def _get_language_locale_dict():
+    cldr_dates_full_dir = "../raw_data/cldr_dates_full/main/"
+    available_locale_names = os.listdir(cldr_dates_full_dir)
+    available_language_names = [
+        shortname
+        for shortname in available_locale_names
+        if not re.search(r"-[A-Z0-9]+$", shortname)
+    ]
+    available_language_names.remove("root")
+    language_locale_dict = {}
+    for language_name in available_language_names:
+        language_locale_dict[language_name] = []
+        for locale_name in available_locale_names:
+            if re.match(language_name + "-[A-Z0-9]+$", locale_name):
+                language_locale_dict[language_name].append(locale_name)
+
+    for language in avoid_languages:
+        if language in language_locale_dict:
+            del language_locale_dict[language]
+    return language_locale_dict
+
+
+def _get_language_order(language_locale_dict):
+    def get_most_common_locales():
+        # Order from https://w3techs.com/technologies/overview/content_language
+        # Last updated on 03.10.2022
+        old_common_locales = [
+            "en",
+            "ru",
+            "es",
+            "de",
+            "tr",
+            "fr",
+            "fa",
+            "ja",
+            "zh",
+            "vi",
+            "it",
+            "nl",
+            "pt",
+            "ar",
+            "pl",
+            "id",
+            "ko",
+            "uk",
+            "th",
+            "he",
+            "cs",
+            "sv",
+            "ro",
+            "el",
+            "da",
+            "hu",
+            "fi",
+            "sr",
+            "sk",
+            "bg",
+            "nb",
+            "hr",
+            "lt",
+            "no",
+            "hi",
+            "sl",
+            "ca",
+            "et",
+        ]
+
+        response = requests.get(
+            "https://w3techs.com/technologies/overview/content_language"
+        )
+        sel = Selector(text=response.text)
+        if response.ok:
+            try:
+                bars = sel.xpath("//table[@class='bars']//a/@href").getall()
+                if not bars:
+                    raise ValueError("No bars found")
+                new_most_common_locales = [
+                    i.replace("https://w3techs.com/technologies/details/cl", "").strip(
+                        "-"
+                    )
+                    for i in bars
+                ]
+                if new_most_common_locales[0] != "en":
+                    raise ValueError("English is not the first language")
+            except Exception as e:
+                print(e)
+                print("The website could have changed, please update the code")
+                return old_common_locales
+        else:
+            return old_common_locales
+        return new_most_common_locales
+
+    territory_info_file = "../raw_data/cldr_core/supplemental/territoryInfo.json"
+    with open(territory_info_file) as f:
+        territory_content = json.load(f)
+    territory_info_data = territory_content["supplemental"]["territoryInfo"]
+
+    language_population_dict = {}
+    for territory in territory_info_data:
+        population = int(territory_info_data[territory]["_population"])
+        try:
+            lang_dict = territory_info_data[territory]["languagePopulation"]
+            for language in lang_dict:
+                language_population = (
+                    float(lang_dict[language]["_populationPercent"]) * population
+                )
+                if language in language_population_dict:
+                    language_population_dict[language] += language_population
+                else:
+                    language_population_dict[language] = language_population
+        except Exception:
+            pass
+
+    most_common_locales = get_most_common_locales()
+    language_order_with_duplicates = most_common_locales + sorted(
+        language_population_dict.keys(),
+        key=lambda x: (language_population_dict[x], x),
+        reverse=True,
+    )
+    language_order = sorted(
+        set(language_order_with_duplicates),
+        key=lambda x: language_order_with_duplicates.index(x),
+    )
+
+    for index in range(0, len(language_order)):
+        language_order[index] = re.sub(r"_", r"-", language_order[index])
+
+    cldr_languages = language_locale_dict.keys()
+    supplementary_date_directory = (
+        "../dateparser_data/supplementary_language_data/date_translation_data"
+    )
+    supplementary_languages = [x[:-5] for x in os.listdir(supplementary_date_directory)]
+    available_languages = set(cldr_languages).union(set(supplementary_languages))
+    language_order = [
+        shortname for shortname in language_order if shortname in available_languages
+    ]
+    absent_languages = set(available_languages) - set(language_order)
+    remaining_languages = []
+    for language in absent_languages:
+        parent_language = re.sub(r"-\w+", "", language)
+        if parent_language in language_order:
+            language_order.insert(language_order.index(parent_language) + 1, language)
+        else:
+            remaining_languages.append(language)
+    language_order = language_order + sorted(remaining_languages)
+    language_order = list(map(str, language_order))
+    return language_order
+
+
+def generate_language_map(language_order):
+    data = {}
+    for lang in sorted(language_order):
+        if "-" not in lang:
+            data[lang] = [lang]
+        else:
+            data[lang.split("-")[0]].append(lang)
+    return data
+
+
+def main():
+    get_raw_data()
+    language_locale_dict = _get_language_locale_dict()
+    language_order = _get_language_order(language_locale_dict)
+
+    parent_directory = "../dateparser/data/"
+    filename = "../dateparser/data/languages_info.py"
+    if not os.path.isdir(parent_directory):
+        os.mkdir(parent_directory)
+    language_order_string = "language_order = " + json.dumps(
+        language_order, separators=(",", ": "), indent=4
+    )
+
+    complete_language_locale_dict = OrderedDict()
+    for key in language_order:
+        if key in language_locale_dict.keys():
+            complete_language_locale_dict[key] = sorted(language_locale_dict[key])
+        else:
+            complete_language_locale_dict[key] = []
+
+    language_locale_dict_string = "language_locale_dict = " + json.dumps(
+        complete_language_locale_dict, separators=(",", ": "), indent=4
+    )
+    language_map_data = generate_language_map(language_order)
+    language_map_data_string = "language_map = " + json.dumps(
+        language_map_data, separators=(",", ": "), indent=4
+    )
+
+    languages_info_string = (
+        language_order_string
+        + "\n\n"
+        + language_map_data_string
+        + "\n\n"
+        + language_locale_dict_string
+        + "\n"
+    )
+    with open(filename, "w") as f:
+        f.write(languages_info_string)
+
+
+if __name__ == "__main__":
+    main()

dateparser_scripts/update_supported_languages_and_locales.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+import os
+from tempfile import TemporaryFile
+
+from dateparser.data.languages_info import language_locale_dict
+
+
+def to_string(data):
+    result = ""
+    language_column_width = 18
+    for language in sorted(data):
+        result += language
+        locales = data[language]
+        if locales:
+            result += " " * (language_column_width - len(language))
+            result += ", ".join("'{}'".format(locale) for locale in sorted(locales))
+        result += "\n"
+    return result
+
+
+def main():
+    readme_path = os.path.join(
+        os.path.dirname(__file__), "..", "docs", "supported_locales.rst"
+    )
+    new_data = to_string(language_locale_dict)
+    temporary_file = TemporaryFile("w+")
+    with open(readme_path) as readme_file:
+        delimiter = "============    ================================================================\n"
+        delimiters_seen = 0
+        is_inside_table = False
+        for line in readme_file:
+            if line == delimiter:
+                delimiters_seen += 1
+                is_inside_table = delimiters_seen == 2
+            elif is_inside_table:
+                continue
+            temporary_file.write(line)
+            if is_inside_table:
+                temporary_file.write(new_data)
+    temporary_file.seek(0)
+    with open(readme_path, "w") as readme_file:
+        readme_file.write(temporary_file.read())
+    temporary_file.close()
+
+
+if __name__ == "__main__":
+    main()

dateparser_scripts/utils.py

@@ -0,0 +1,73 @@
+import os
+import shutil
+from collections import OrderedDict
+
+from git import Repo
+
+
+def get_raw_data():
+    cldr_version = "31.0.1"
+    raw_data_directory = "../raw_data"
+
+    cldr_data = {
+        "dates_full": {
+            "url": "https://github.com/unicode-cldr/cldr-dates-full.git",
+            "dir": "{}/cldr_dates_full/".format(raw_data_directory),
+        },
+        "core": {
+            "url": "https://github.com/unicode-cldr/cldr-core.git",
+            "dir": "{}/cldr_core/".format(raw_data_directory),
+        },
+        "rbnf": {
+            "url": "https://github.com/unicode-cldr/cldr-rbnf.git",
+            "dir": "{}/cldr_rbnf/".format(raw_data_directory),
+        },
+    }
+
+    if os.path.isdir(raw_data_directory):
+        # remove current raw data
+        shutil.rmtree(raw_data_directory)
+    os.mkdir(raw_data_directory)
+
+    for name, data in cldr_data.items():
+        print('Clonning "{}" from: {}'.format(name, data["url"]))
+        repo = Repo.clone_from(data["url"], data["dir"], branch="master")
+        repo.git.co(cldr_version)
+
+
+def get_dict_difference(parent_dict, child_dict):
+    difference_dict = OrderedDict()
+    for key, child_value in child_dict.items():
+        parent_value = parent_dict.get(key)
+        child_specific_value = None
+        if not parent_value:
+            child_specific_value = child_value
+        elif isinstance(child_value, list):
+            child_specific_value = sorted(set(child_value) - set(parent_value))
+        elif isinstance(child_value, dict):
+            child_specific_value = get_dict_difference(parent_value, child_value)
+        elif child_value != parent_value:
+            child_specific_value = child_value
+        if child_specific_value:
+            difference_dict[key] = child_specific_value
+    return difference_dict
+
+
+def combine_dicts(primary_dict, supplementary_dict):
+    combined_dict = OrderedDict()
+    for key, value in primary_dict.items():
+        if key in supplementary_dict:
+            if isinstance(value, list):
+                combined_dict[key] = value + supplementary_dict[key]
+            elif isinstance(value, dict):
+                combined_dict[key] = combine_dicts(value, supplementary_dict[key])
+            else:
+                combined_dict[key] = supplementary_dict[key]
+        else:
+            combined_dict[key] = primary_dict[key]
+    remaining_keys = [
+        key for key in supplementary_dict.keys() if key not in primary_dict.keys()
+    ]
+    for key in remaining_keys:
+        combined_dict[key] = supplementary_dict[key]
+    return combined_dict

dateparser_scripts/write_complete_data.py

@@ -0,0 +1,129 @@
+import json
+import os
+import shutil
+from collections import OrderedDict
+
+import regex as re
+from ruamel.yaml import RoundTripLoader
+
+from dateparser_scripts.order_languages import avoid_languages
+from dateparser_scripts.utils import combine_dicts
+
+cldr_date_directory = "../dateparser_data/cldr_language_data/date_translation_data/"
+supplementary_directory = "../dateparser_data/supplementary_language_data/"
+supplementary_date_directory = (
+    "../dateparser_data/supplementary_language_data/date_translation_data/"
+)
+translation_data_directory = "../dateparser/data/"
+date_translation_directory = "../dateparser/data/date_translation_data/"
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+cldr_languages = list(
+    set(map(lambda x: x[:-5], os.listdir(cldr_date_directory))) - avoid_languages
+)
+supplementary_languages = [x[:-5] for x in os.listdir(supplementary_date_directory)]
+all_languages = set(cldr_languages).union(set(supplementary_languages))
+
+RELATIVE_PATTERN = re.compile(r"\{0\}")
+
+
+def _modify_relative_data(relative_data):
+    modified_relative_data = OrderedDict()
+    for key, value in relative_data.items():
+        for i, string in enumerate(value):
+            string = RELATIVE_PATTERN.sub(r"(\\d+[.,]?\\d*)", string)
+            value[i] = string
+        modified_relative_data[key] = value
+    return modified_relative_data
+
+
+def _modify_data(language_data):
+    relative_data = language_data.get("relative-type-regex", {})
+    relative_data = _modify_relative_data(relative_data)
+    locale_specific_data = language_data.get("locale_specific", {})
+    for _, info in locale_specific_data.items():
+        locale_relative_data = info.get("relative-type-regex", {})
+        locale_relative_data = _modify_relative_data(locale_relative_data)
+
+
+def _get_complete_date_translation_data(language):
+    cldr_data = {}
+    supplementary_data = {}
+    if language in cldr_languages:
+        with open(cldr_date_directory + language + ".json") as f:
+            cldr_data = json.load(f, object_pairs_hook=OrderedDict)
+    if language in supplementary_languages:
+        with open(supplementary_date_directory + language + ".yaml") as g:
+            supplementary_data = OrderedDict(RoundTripLoader(g).get_data())
+    complete_data = combine_dicts(cldr_data, supplementary_data)
+    if "name" not in complete_data:
+        complete_data["name"] = language
+    return complete_data
+
+
+def _write_file(filename, text, mode, in_memory, in_memory_result):
+    if in_memory:
+        in_memory_result[filename] = text
+    else:
+        with open(filename, mode) as out:
+            out.write(text)
+
+
+def write_complete_data(in_memory=False):
+    """
+    This function is responsible of generating the needed py files from the
+    CLDR files (JSON format) and supplementary language data (YAML format).
+
+    Use it with in_memory=True to avoid writing real files and getting a
+    dictionary containing the file names and their content (used when testing).
+    """
+    in_memory_result = {}
+
+    if not in_memory:
+        if not os.path.isdir(translation_data_directory):
+            os.mkdir(translation_data_directory)
+        if os.path.isdir(date_translation_directory):
+            shutil.rmtree(date_translation_directory)
+        os.mkdir(date_translation_directory)
+
+    with open(supplementary_directory + "base_data.yaml") as f:
+        base_data = RoundTripLoader(f).get_data()
+
+    for language in all_languages:
+        date_translation_data = _get_complete_date_translation_data(language)
+        date_translation_data = combine_dicts(date_translation_data, base_data)
+        _modify_data(date_translation_data)
+        translation_data = json.dumps(
+            date_translation_data, indent=4, separators=(",", ": "), ensure_ascii=False
+        )
+        out_text = ("info = " + translation_data + "\n").encode("utf-8")
+        _write_file(
+            date_translation_directory + language + ".py",
+            out_text,
+            "wb",
+            in_memory,
+            in_memory_result,
+        )
+
+    init_text = (
+        "from dateparser.data import date_translation_data\n"
+        "from .languages_info import language_order, language_locale_dict\n"
+    )
+
+    _write_file(
+        translation_data_directory + "__init__.py",
+        init_text,
+        "w",
+        False,
+        in_memory_result,
+    )
+    _write_file(
+        date_translation_directory + "__init__.py", "", "w", False, in_memory_result
+    )
+
+    return in_memory_result
+
+
+if __name__ == "__main__":
+    write_complete_data()