dateparser 1.2.1__py3-none-any.whl

dateparser/conf.py

@@ -0,0 +1,267 @@
+import hashlib
+from datetime import datetime
+from functools import wraps
+
+from dateparser.data.languages_info import language_order
+
+from .parser import date_order_chart
+from .utils import registry
+
+
+@registry
+class Settings:
+    """Control and configure default parsing behavior of dateparser.
+    Currently, supported settings are:
+
+    * `DATE_ORDER`
+    * `PREFER_LOCALE_DATE_ORDER`
+    * `TIMEZONE`
+    * `TO_TIMEZONE`
+    * `RETURN_AS_TIMEZONE_AWARE`
+    * `PREFER_MONTH_OF_YEAR`
+    * `PREFER_DAY_OF_MONTH`
+    * `PREFER_DATES_FROM`
+    * `RELATIVE_BASE`
+    * `STRICT_PARSING`
+    * `REQUIRE_PARTS`
+    * `SKIP_TOKENS`
+    * `NORMALIZE`
+    * `RETURN_TIME_AS_PERIOD`
+    * `PARSERS`
+    * `DEFAULT_LANGUAGES`
+    * `LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD`
+    * `CACHE_SIZE_LIMIT`
+    """
+
+    _default = True
+    _pyfile_data = None
+    _mod_settings = dict()
+
+    def __init__(self, settings=None):
+        if settings:
+            self._updateall(settings.items())
+        else:
+            self._updateall(self._get_settings_from_pyfile().items())
+
+    @classmethod
+    def get_key(cls, settings=None):
+        if not settings:
+            return "default"
+
+        keys = sorted(["%s-%s" % (key, str(settings[key])) for key in settings])
+        return hashlib.md5("".join(keys).encode("utf-8")).hexdigest()
+
+    @classmethod
+    def _get_settings_from_pyfile(cls):
+        if not cls._pyfile_data:
+            from dateparser_data import settings
+
+            cls._pyfile_data = settings.settings
+        return cls._pyfile_data
+
+    def _updateall(self, iterable):
+        for key, value in iterable:
+            setattr(self, key, value)
+
+    def replace(self, mod_settings=None, **kwds):
+        for k, v in kwds.items():
+            if v is None:
+                raise TypeError('Invalid {{"{}": {}}}'.format(k, v))
+
+        for x in self._get_settings_from_pyfile().keys():
+            kwds.setdefault(x, getattr(self, x))
+
+        kwds["_default"] = False
+        if mod_settings:
+            kwds["_mod_settings"] = mod_settings
+
+        return self.__class__(settings=kwds)
+
+
+settings = Settings()
+
+
+def apply_settings(f):
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        mod_settings = kwargs.get("settings")
+        kwargs["settings"] = mod_settings or settings
+
+        if isinstance(kwargs["settings"], dict):
+            kwargs["settings"] = settings.replace(
+                mod_settings=mod_settings, **kwargs["settings"]
+            )
+
+        if not isinstance(kwargs["settings"], Settings):
+            raise TypeError(
+                "settings can only be either dict or instance of Settings class"
+            )
+
+        return f(*args, **kwargs)
+
+    return wrapper
+
+
+class SettingValidationError(ValueError):
+    pass
+
+
+def _check_repeated_values(setting_name, setting_value):
+    if len(setting_value) != len(set(setting_value)):
+        raise SettingValidationError(
+            'There are repeated values in the "{}" setting'.format(setting_name)
+        )
+    return
+
+
+def _check_require_part(setting_name, setting_value):
+    """Returns `True` if the provided list of parts contains valid values"""
+    invalid_values = set(setting_value) - {"day", "month", "year"}
+    if invalid_values:
+        raise SettingValidationError(
+            '"{}" setting contains invalid values: {}'.format(
+                setting_name, ", ".join(invalid_values)
+            )
+        )
+    _check_repeated_values(setting_name, setting_value)
+
+
+def _check_parsers(setting_name, setting_value):
+    """Returns `True` if the provided list of parsers contains valid values"""
+    existing_parsers = [
+        "timestamp",
+        "relative-time",
+        "custom-formats",
+        "absolute-time",
+        "no-spaces-time",
+        "negative-timestamp",
+    ]  # FIXME: Extract the list of existing parsers from another place (#798)
+    unknown_parsers = set(setting_value) - set(existing_parsers)
+    if unknown_parsers:
+        raise SettingValidationError(
+            'Found unknown parsers in the "{}" setting: {}'.format(
+                setting_name, ", ".join(unknown_parsers)
+            )
+        )
+    _check_repeated_values(setting_name, setting_value)
+
+
+def _check_default_languages(setting_name, setting_value):
+    unsupported_languages = set(setting_value) - set(language_order)
+    if unsupported_languages:
+        raise SettingValidationError(
+            "Found invalid languages in the '{}' setting: {}".format(
+                setting_name, ", ".join(map(repr, unsupported_languages))
+            )
+        )
+    _check_repeated_values(setting_name, setting_value)
+
+
+def _check_between_0_and_1(setting_name, setting_value):
+    is_valid = 0 <= setting_value <= 1
+    if not is_valid:
+        raise SettingValidationError(
+            "{} is not a valid value for {}. It can take values between 0 and "
+            "1.".format(
+                setting_value,
+                setting_name,
+            )
+        )
+
+
+def check_settings(settings):
+    """
+    Check if provided settings are valid, if not it raises `SettingValidationError`.
+    Only checks for the modified settings.
+    """
+    settings_values = {
+        "DATE_ORDER": {
+            "values": tuple(date_order_chart.keys()),
+            "type": str,
+        },
+        "TIMEZONE": {
+            # we don't check invalid Timezones as they raise an error
+            "type": str,
+        },
+        "TO_TIMEZONE": {
+            # It defaults to None, but it's not allowed to use it directly
+            # "values" can take unlimited options
+            "type": str
+        },
+        "RETURN_AS_TIMEZONE_AWARE": {
+            # It defaults to 'default', but it's not allowed to use it directly
+            "type": bool
+        },
+        "PREFER_MONTH_OF_YEAR": {"values": ("current", "first", "last"), "type": str},
+        "PREFER_DAY_OF_MONTH": {"values": ("current", "first", "last"), "type": str},
+        "PREFER_DATES_FROM": {
+            "values": ("current_period", "past", "future"),
+            "type": str,
+        },
+        "RELATIVE_BASE": {
+            # "values" can take unlimited options
+            "type": datetime
+        },
+        "STRICT_PARSING": {"type": bool},
+        "REQUIRE_PARTS": {
+            # "values" covered by the 'extra_check'
+            "type": list,
+            "extra_check": _check_require_part,
+        },
+        "SKIP_TOKENS": {
+            # "values" can take unlimited options
+            "type": list,
+        },
+        "NORMALIZE": {"type": bool},
+        "RETURN_TIME_AS_PERIOD": {"type": bool},
+        "PARSERS": {
+            # "values" covered by the 'extra_check'
+            "type": list,
+            "extra_check": _check_parsers,
+        },
+        "FUZZY": {"type": bool},
+        "PREFER_LOCALE_DATE_ORDER": {"type": bool},
+        "DEFAULT_LANGUAGES": {"type": list, "extra_check": _check_default_languages},
+        "LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD": {
+            "type": float,
+            "extra_check": _check_between_0_and_1,
+        },
+        "CACHE_SIZE_LIMIT": {
+            "type": int,
+        },
+    }
+
+    modified_settings = settings._mod_settings  # check only modified settings
+
+    # check settings keys:
+    for setting in modified_settings:
+        if setting not in settings_values:
+            raise SettingValidationError('"{}" is not a valid setting'.format(setting))
+
+    for setting_name, setting_value in modified_settings.items():
+        setting_type = type(setting_value)
+        setting_props = settings_values[setting_name]
+
+        # check type:
+        if not isinstance(setting_value, setting_props["type"]):
+            raise SettingValidationError(
+                '"{}" must be "{}", not "{}".'.format(
+                    setting_name, setting_props["type"].__name__, setting_type.__name__
+                )
+            )
+
+        # check values:
+        if setting_props.get("values") and setting_value not in setting_props["values"]:
+            raise SettingValidationError(
+                '"{}" is not a valid value for "{}", it should be: "{}" or "{}"'.format(
+                    setting_value,
+                    setting_name,
+                    '", "'.join(setting_props["values"][:-1]),
+                    setting_props["values"][-1],
+                )
+            )
+
+        # specific checks
+        extra_check = setting_props.get("extra_check")
+        if extra_check:
+            extra_check(setting_name, setting_value)

dateparser/custom_language_detection/fasttext.py

@@ -0,0 +1,43 @@
+import os
+
+import fasttext
+
+from dateparser_cli.exceptions import FastTextModelNotFoundException
+from dateparser_cli.fasttext_manager import fasttext_downloader
+from dateparser_cli.utils import create_data_model_home, dateparser_model_home
+
+_supported_models = ["large.bin", "small.bin"]
+_DEFAULT_MODEL = "small"
+
+
+class _FastTextCache:
+    model = None
+
+
+def _load_fasttext_model():
+    if _FastTextCache.model:
+        return _FastTextCache.model
+    create_data_model_home()
+    downloaded_models = [
+        file for file in os.listdir(dateparser_model_home) if file in _supported_models
+    ]
+    if not downloaded_models:
+        fasttext_downloader(_DEFAULT_MODEL)
+        return _load_fasttext_model()
+    model_path = os.path.join(dateparser_model_home, downloaded_models[0])
+    if not os.path.isfile(model_path):
+        raise FastTextModelNotFoundException("Fasttext model file not found")
+    _FastTextCache.model = fasttext.load_model(model_path)
+    return _FastTextCache.model
+
+
+def detect_languages(text, confidence_threshold):
+    _language_parser = _load_fasttext_model()
+    text = text.replace("\n", " ").replace("\r", "")
+    language_codes = []
+    parser_data = _language_parser.predict(text)
+    for idx, language_probability in enumerate(parser_data[1]):
+        if language_probability > confidence_threshold:
+            language_code = parser_data[0][idx].replace("__label__", "")
+            language_codes.append(language_code)
+    return language_codes

dateparser/custom_language_detection/langdetect.py

@@ -0,0 +1,37 @@
+import langdetect
+
+# The below _Factory is set to prevent setting global state of the library
+# but still get consistent results.
+# Refer : https://github.com/Mimino666/langdetect
+
+
+class _Factory:
+    data = None
+
+
+def _init_factory():
+    if _Factory.data is None:
+        _Factory.data = langdetect.detector_factory.DetectorFactory()
+        _Factory.data.load_profile(langdetect.detector_factory.PROFILES_DIRECTORY)
+        _Factory.data.seed = 0
+
+
+def _get_language_probablities(text):
+    _init_factory()
+    detector = _Factory.data.create()
+    detector.append(text)
+    return detector.get_probabilities()
+
+
+def detect_languages(text, confidence_threshold):
+    language_codes = []
+    try:
+        parser_data = _get_language_probablities(text)
+        for language_candidate in parser_data:
+            if language_candidate.prob > confidence_threshold:
+                language_codes.append(language_candidate.lang)
+    except langdetect.lang_detect_exception.LangDetectException:
+        # This exception can be produced with empty strings or inputs without letters like `10-10-2021`.
+        # As this could be really common, we ignore them.
+        pass
+    return language_codes

dateparser/custom_language_detection/language_mapping.py

@@ -0,0 +1,18 @@
+from dateparser.data.languages_info import language_map
+
+
+def map_languages(language_codes):
+    """
+    Returns the candidates from the supported languages codes.
+    :param language_codes:
+        A list of language codes, e.g. ['en', 'es'] in ISO 639 Standard.
+    :type language_codes: list
+    :return: Returns list[str] representing supported languages
+    :rtype: list[str]
+    """
+    return [
+        language_code
+        for language in language_codes
+        if language in language_map
+        for language_code in language_map[language]
+    ]

dateparser/data/__init__.py

@@ -0,0 +1,2 @@
+from dateparser.data import date_translation_data
+from .languages_info import language_order, language_locale_dict

dateparser/data/date_translation_data/af.py

@@ -0,0 +1,242 @@
+info = {
+    "name": "af",
+    "date_order": "YMD",
+    "january": [
+        "jan",
+        "januarie"
+    ],
+    "february": [
+        "feb",
+        "februarie"
+    ],
+    "march": [
+        "maart",
+        "mrt"
+    ],
+    "april": [
+        "apr",
+        "april"
+    ],
+    "may": [
+        "mei"
+    ],
+    "june": [
+        "jun",
+        "junie"
+    ],
+    "july": [
+        "jul",
+        "julie"
+    ],
+    "august": [
+        "aug",
+        "augustus"
+    ],
+    "september": [
+        "sep",
+        "september"
+    ],
+    "october": [
+        "okt",
+        "oktober"
+    ],
+    "november": [
+        "nov",
+        "november"
+    ],
+    "december": [
+        "des",
+        "desember"
+    ],
+    "monday": [
+        "ma",
+        "maandag"
+    ],
+    "tuesday": [
+        "di",
+        "dinsdag"
+    ],
+    "wednesday": [
+        "wo",
+        "woensdag"
+    ],
+    "thursday": [
+        "do",
+        "donderdag"
+    ],
+    "friday": [
+        "vr",
+        "vrydag"
+    ],
+    "saturday": [
+        "sa",
+        "saterdag"
+    ],
+    "sunday": [
+        "so",
+        "sondag"
+    ],
+    "am": [
+        "vm"
+    ],
+    "pm": [
+        "nm"
+    ],
+    "year": [
+        "j",
+        "jaar"
+    ],
+    "month": [
+        "maand",
+        "md"
+    ],
+    "week": [
+        "week",
+        "wk"
+    ],
+    "day": [
+        "d",
+        "dag"
+    ],
+    "hour": [
+        "u",
+        "uur"
+    ],
+    "minute": [
+        "m",
+        "min",
+        "minuut"
+    ],
+    "second": [
+        "s",
+        "sek",
+        "sekonde"
+    ],
+    "relative-type": {
+        "0 day ago": [
+            "vandag"
+        ],
+        "0 hour ago": [
+            "hierdie uur"
+        ],
+        "0 minute ago": [
+            "hierdie minuut"
+        ],
+        "0 month ago": [
+            "vandeesmaand"
+        ],
+        "0 second ago": [
+            "nou"
+        ],
+        "0 week ago": [
+            "vandeesweek"
+        ],
+        "0 year ago": [
+            "hierdie jaar"
+        ],
+        "1 day ago": [
+            "gister"
+        ],
+        "1 month ago": [
+            "verlede maand"
+        ],
+        "1 week ago": [
+            "verlede week"
+        ],
+        "1 year ago": [
+            "verlede jaar"
+        ],
+        "in 1 day": [
+            "môre"
+        ],
+        "in 1 month": [
+            "volgende maand"
+        ],
+        "in 1 week": [
+            "volgende week"
+        ],
+        "in 1 year": [
+            "volgende jaar"
+        ]
+    },
+    "relative-type-regex": {
+        "\\1 day ago": [
+            "(\\d+[.,]?\\d*) dae gelede",
+            "(\\d+[.,]?\\d*) dag gelede"
+        ],
+        "\\1 hour ago": [
+            "(\\d+[.,]?\\d*) uur gelede"
+        ],
+        "\\1 minute ago": [
+            "(\\d+[.,]?\\d*) min gelede",
+            "(\\d+[.,]?\\d*) minute gelede",
+            "(\\d+[.,]?\\d*) minuut gelede"
+        ],
+        "\\1 month ago": [
+            "(\\d+[.,]?\\d*) maand gelede",
+            "(\\d+[.,]?\\d*) maande gelede",
+            "(\\d+[.,]?\\d*) md gelede"
+        ],
+        "\\1 second ago": [
+            "(\\d+[.,]?\\d*) sek gelede",
+            "(\\d+[.,]?\\d*) sekonde gelede",
+            "(\\d+[.,]?\\d*) sekondes gelede"
+        ],
+        "\\1 week ago": [
+            "(\\d+[.,]?\\d*) w gelede",
+            "(\\d+[.,]?\\d*) week gelede",
+            "(\\d+[.,]?\\d*) weke gelede"
+        ],
+        "\\1 year ago": [
+            "(\\d+[.,]?\\d*) jaar gelede"
+        ],
+        "in \\1 day": [
+            "oor (\\d+[.,]?\\d*) dae",
+            "oor (\\d+[.,]?\\d*) dag",
+            "oor (\\d+[.,]?\\d*) minuut"
+        ],
+        "in \\1 hour": [
+            "oor (\\d+[.,]?\\d*) uur"
+        ],
+        "in \\1 minute": [
+            "oor (\\d+[.,]?\\d*) min",
+            "oor (\\d+[.,]?\\d*) minuut"
+        ],
+        "in \\1 month": [
+            "oor (\\d+[.,]?\\d*) md",
+            "oor (\\d+[.,]?\\d*) minuut"
+        ],
+        "in \\1 second": [
+            "oor (\\d+[.,]?\\d*) sek",
+            "oor (\\d+[.,]?\\d*) sekonde",
+            "oor (\\d+[.,]?\\d*) sekondes"
+        ],
+        "in \\1 week": [
+            "oor (\\d+[.,]?\\d*) w",
+            "oor (\\d+[.,]?\\d*) week",
+            "oor (\\d+[.,]?\\d*) weke"
+        ],
+        "in \\1 year": [
+            "oor (\\d+[.,]?\\d*) jaar"
+        ]
+    },
+    "locale_specific": {
+        "af-NA": {
+            "name": "af-NA"
+        }
+    },
+    "skip": [
+        " ",
+        "'",
+        ",",
+        "-",
+        ".",
+        "/",
+        ";",
+        "@",
+        "[",
+        "]",
+        "|",
+        "，"
+    ]
+}