dateparser 1.2.1__py3-none-any.whl

dateparser/search/__init__.py

@@ -0,0 +1,71 @@
+from dateparser.search.search import DateSearchWithDetection
+
+_search_with_detection = DateSearchWithDetection()
+
+
+def search_dates(
+    text,
+    languages=None,
+    settings=None,
+    add_detected_language=False,
+    detect_languages_function=None,
+):
+    """Find all substrings of the given string which represent date and/or time and parse them.
+
+    :param text:
+        A string in a natural language which may contain date and/or time expressions.
+    :type text: str
+
+    :param languages:
+        A list of two letters language codes.e.g. ['en', 'es']. If languages are given, it will
+        not attempt to detect the language.
+    :type languages: list
+
+    :param settings:
+        Configure customized behavior using settings defined in :mod:`dateparser.conf.Settings`.
+    :type settings: dict
+
+    :param add_detected_language:
+        Indicates if we want the detected language returned in the tuple.
+    :type add_detected_language: bool
+
+    :param detect_languages_function:
+        A function for language detection that takes as input a `text` and a `confidence_threshold`,
+        and returns a list of detected language codes.
+        Note: detect_languages_function is only uses if `languages` are not provided.
+    :type detect_languages_function: function
+
+    :return: Returns list of tuples containing:
+        substrings representing date and/or time, corresponding :mod:`datetime.datetime`
+        object and detected language if *add_detected_language* is True.
+        Returns None if no dates that can be parsed are found.
+    :rtype: list
+    :raises: ValueError - Unknown Language
+
+    >>> from dateparser.search import search_dates
+    >>> search_dates('The first artificial Earth satellite was launched on 4 October 1957.')
+    [('on 4 October 1957', datetime.datetime(1957, 10, 4, 0, 0))]
+
+    >>> search_dates('The first artificial Earth satellite was launched on 4 October 1957.',
+    >>>              add_detected_language=True)
+    [('on 4 October 1957', datetime.datetime(1957, 10, 4, 0, 0), 'en')]
+
+    >>> search_dates("The client arrived to the office for the first time in March 3rd, 2004 "
+    >>>              "and got serviced, after a couple of months, on May 6th 2004, the customer "
+    >>>              "returned indicating a defect on the part")
+    [('in March 3rd, 2004 and', datetime.datetime(2004, 3, 3, 0, 0)),
+     ('on May 6th 2004', datetime.datetime(2004, 5, 6, 0, 0))]
+
+    """
+    result = _search_with_detection.search_dates(
+        text=text,
+        languages=languages,
+        settings=settings,
+        detect_languages_function=detect_languages_function,
+    )
+    dates = result.get("Dates")
+    if dates:
+        if add_detected_language:
+            language = result.get("Language")
+            dates = [date + (language,) for date in dates]
+        return dates

dateparser/search/detection.py

@@ -0,0 +1,78 @@
+from functools import wraps
+
+
+def _restore_languages_on_generator_exit(method):
+    @wraps(method)
+    def wrapped(self, *args, **kwargs):
+        stored_languages = self.languages[:]
+        for language in method(self, *args, **kwargs):
+            yield language
+        else:
+            self.languages[:] = stored_languages
+
+    return wrapped
+
+
+class BaseLanguageDetector:
+    def __init__(self, languages):
+        self.languages = languages[:]
+
+    @_restore_languages_on_generator_exit
+    def iterate_applicable_languages(self, date_string, settings=None, modify=False):
+        languages = self.languages if modify else self.languages[:]
+        yield from self._filter_languages(date_string, languages, settings)
+
+    @staticmethod
+    def _filter_languages(date_string, languages, settings=None):
+        while languages:
+            language = languages[0]
+            if language.is_applicable(
+                date_string, strip_timezone=False, settings=settings
+            ):
+                yield language
+            elif language.is_applicable(
+                date_string, strip_timezone=True, settings=settings
+            ):
+                yield language
+
+            languages.pop(0)
+
+
+class AutoDetectLanguage(BaseLanguageDetector):
+    def __init__(self, languages, allow_redetection=False):
+        super().__init__(languages=languages[:])
+        self.language_pool = languages[:]
+        self.allow_redetection = allow_redetection
+
+    @_restore_languages_on_generator_exit
+    def iterate_applicable_languages(self, date_string, modify=False, settings=None):
+        languages = self.languages if modify else self.languages[:]
+        initial_languages = languages[:]
+        yield from self._filter_languages(date_string, languages, settings=settings)
+
+        if not self.allow_redetection:
+            return
+
+        # Try languages that was not tried before with this date_string
+        languages = [
+            language
+            for language in self.language_pool
+            if language not in initial_languages
+        ]
+        if modify:
+            self.languages = languages
+
+        yield from self._filter_languages(date_string, languages, settings=settings)
+
+
+class ExactLanguages(BaseLanguageDetector):
+    def __init__(self, languages):
+        if languages is None:
+            raise ValueError("language cannot be None for ExactLanguages")
+        super().__init__(languages=languages)
+
+    @_restore_languages_on_generator_exit
+    def iterate_applicable_languages(self, date_string, modify=False, settings=None):
+        yield from super().iterate_applicable_languages(
+            date_string, modify=False, settings=settings
+        )

dateparser/search/search.py

@@ -0,0 +1,297 @@
+from collections.abc import Set
+
+import regex as re
+
+from dateparser.conf import Settings, apply_settings, check_settings
+from dateparser.custom_language_detection.language_mapping import map_languages
+from dateparser.date import DateDataParser
+from dateparser.languages.loader import LocaleDataLoader
+from dateparser.search.text_detection import FullTextLanguageDetector
+
+RELATIVE_REG = re.compile("(ago|in|from now|tomorrow|today|yesterday)")
+
+
+def date_is_relative(translation):
+    return re.search(RELATIVE_REG, translation) is not None
+
+
+class _ExactLanguageSearch:
+    def __init__(self, loader):
+        self.loader = loader
+        self.language = None
+
+    def get_current_language(self, shortname):
+        if self.language is None or self.language.shortname != shortname:
+            self.language = self.loader.get_locale(shortname)
+
+    def search(self, shortname, text, settings):
+        self.get_current_language(shortname)
+        result = self.language.translate_search(text, settings=settings)
+        return result
+
+    @staticmethod
+    def set_relative_base(substring, already_parsed):
+        if len(already_parsed) == 0:
+            return substring, None
+
+        i = len(already_parsed) - 1
+        while already_parsed[i][1]:
+            i -= 1
+            if i == -1:
+                return substring, None
+        relative_base = already_parsed[i][0]["date_obj"]
+        return substring, relative_base
+
+    def choose_best_split(self, possible_parsed_splits, possible_substrings_splits):
+        rating = []
+        for i in range(len(possible_parsed_splits)):
+            num_substrings = len(possible_substrings_splits[i])
+            num_substrings_without_digits = 0
+            not_parsed = 0
+            for j, item in enumerate(possible_parsed_splits[i]):
+                if item[0]["date_obj"] is None:
+                    not_parsed += 1
+                if not any(char.isdigit() for char in possible_substrings_splits[i][j]):
+                    num_substrings_without_digits += 1
+            rating.append(
+                [
+                    num_substrings,
+                    0
+                    if not_parsed == 0
+                    else (float(not_parsed) / float(num_substrings)),
+                    0
+                    if num_substrings_without_digits == 0
+                    else (float(num_substrings_without_digits) / float(num_substrings)),
+                ]
+            )
+            best_index, best_rating = min(
+                enumerate(rating), key=lambda p: (p[1][1], p[1][0], p[1][2])
+            )
+        return (
+            possible_parsed_splits[best_index],
+            possible_substrings_splits[best_index],
+        )
+
+    def split_by(self, item, original, splitter):
+        if item.count(splitter) <= 2:
+            return [[item.split(splitter), original.split(splitter)]]
+
+        item_all_split = item.split(splitter)
+        original_all_split = original.split(splitter)
+        all_possible_splits = [[item_all_split, original_all_split]]
+        for i in range(2, 4):
+            item_partially_split = []
+            original_partially_split = []
+            for j in range(0, len(item_all_split), i):
+                item_join = splitter.join(item_all_split[j : j + i])
+                original_join = splitter.join(original_all_split[j : j + i])
+                item_partially_split.append(item_join)
+                original_partially_split.append(original_join)
+            all_possible_splits.append([item_partially_split, original_partially_split])
+        return all_possible_splits
+
+    def split_if_not_parsed(self, item, original):
+        splitters = [",", "،", "——", "—", "–", ".", " "]
+        possible_splits = []
+        for splitter in splitters:
+            if splitter in item and item.count(splitter) == original.count(splitter):
+                possible_splits.extend(self.split_by(item, original, splitter))
+        return possible_splits
+
+    def parse_item(self, parser, item, translated_item, parsed, need_relative_base):
+        relative_base = None
+        item = item.replace("ngày", "")
+        item = item.replace("am", "")
+        parsed_item = parser.get_date_data(item)
+        is_relative = date_is_relative(translated_item)
+
+        if need_relative_base:
+            item, relative_base = self.set_relative_base(item, parsed)
+
+        if relative_base:
+            parser._settings.RELATIVE_BASE = relative_base
+            parsed_item = parser.get_date_data(item)
+        return parsed_item, is_relative
+
+    def parse_found_objects(self, parser, to_parse, original, translated, settings):
+        parsed = []
+        substrings = []
+        need_relative_base = True
+        if settings.RELATIVE_BASE:
+            need_relative_base = False
+        for i, item in enumerate(to_parse):
+            if len(item) <= 2:
+                continue
+
+            parsed_item, is_relative = self.parse_item(
+                parser, item, translated[i], parsed, need_relative_base
+            )
+            if parsed_item["date_obj"]:
+                parsed.append((parsed_item, is_relative))
+                substrings.append(original[i].strip(" .,:()[]-'"))
+                continue
+
+            possible_splits = self.split_if_not_parsed(item, original[i])
+            if not possible_splits:
+                continue
+
+            possible_parsed = []
+            possible_substrings = []
+            for split_translated, split_original in possible_splits:
+                current_parsed = []
+                current_substrings = []
+                if split_translated:
+                    for j, jtem in enumerate(split_translated):
+                        if len(jtem) <= 2:
+                            continue
+                        parsed_jtem, is_relative_jtem = self.parse_item(
+                            parser,
+                            jtem,
+                            split_translated[j],
+                            current_parsed,
+                            need_relative_base,
+                        )
+                        current_parsed.append((parsed_jtem, is_relative_jtem))
+                        current_substrings.append(split_original[j].strip(" .,:()[]-"))
+                possible_parsed.append(current_parsed)
+                possible_substrings.append(current_substrings)
+            parsed_best, substrings_best = self.choose_best_split(
+                possible_parsed, possible_substrings
+            )
+            for k in range(len(parsed_best)):
+                if parsed_best[k][0]["date_obj"]:
+                    parsed.append(parsed_best[k])
+                    substrings.append(substrings_best[k])
+        return parsed, substrings
+
+    def search_parse(self, shortname, text, settings):
+        translated, original = self.search(shortname, text, settings)
+        bad_translate_with_search = [
+            "vi",
+            "hu",
+        ]  # splitting done by spaces and some dictionary items contain spaces
+        if shortname not in bad_translate_with_search:
+            languages = ["en"]
+            to_parse = translated
+        else:
+            languages = [shortname]
+            to_parse = original
+
+        parser = DateDataParser(languages=languages, settings=settings)
+        parsed, substrings = self.parse_found_objects(
+            parser=parser,
+            to_parse=to_parse,
+            original=original,
+            translated=translated,
+            settings=settings,
+        )
+        parser._settings = Settings()
+        return list(zip(substrings, [i[0]["date_obj"] for i in parsed]))
+
+
+class DateSearchWithDetection:
+    """
+    Class which executes language detection of string in a natural language, translation of a given string,
+    search of substrings which represent date and/or time and parsing of these substrings.
+
+    """
+
+    def __init__(self):
+        self.loader = LocaleDataLoader()
+        self.available_language_map = self.loader.get_locale_map()
+        self.search = _ExactLanguageSearch(self.loader)
+
+    @apply_settings
+    def detect_language(
+        self, text, languages, settings=None, detect_languages_function=None
+    ):
+        if detect_languages_function and not languages:
+            detected_languages = detect_languages_function(
+                text,
+                confidence_threshold=settings.LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD,
+            )
+            detected_languages = (
+                map_languages(detected_languages) or settings.DEFAULT_LANGUAGES
+            )
+            return detected_languages[0] if detected_languages else None
+
+        if isinstance(languages, (list, tuple, Set)):
+            if all([language in self.available_language_map for language in languages]):
+                languages = [
+                    self.available_language_map[language] for language in languages
+                ]
+            else:
+                unsupported_languages = set(languages) - set(
+                    self.available_language_map.keys()
+                )
+                raise ValueError(
+                    "Unknown language(s): %s"
+                    % ", ".join(map(repr, unsupported_languages))
+                )
+        elif languages is not None:
+            raise TypeError(
+                "languages argument must be a list (%r given)" % type(languages)
+            )
+
+        if languages:
+            self.language_detector = FullTextLanguageDetector(languages=languages)
+        else:
+            self.language_detector = FullTextLanguageDetector(
+                list(self.available_language_map.values())
+            )
+
+        detected_language = self.language_detector._best_language(text) or (
+            settings.DEFAULT_LANGUAGES[0] if settings.DEFAULT_LANGUAGES else None
+        )
+        return detected_language
+
+    @apply_settings
+    def search_dates(
+        self, text, languages=None, settings=None, detect_languages_function=None
+    ):
+        """
+        Find all substrings of the given string which represent date and/or time and parse them.
+
+        :param text:
+            A string in a natural language which may contain date and/or time expressions.
+        :type text: str
+
+        :param languages:
+            A list of two letters language codes.e.g. ['en', 'es']. If languages are given, it will not attempt
+            to detect the language.
+        :type languages: list
+
+        :param settings:
+               Configure customized behavior using settings defined in :mod:`dateparser.conf.Settings`.
+        :type settings: dict
+
+        :param detect_languages_function:
+               A function for language detection that takes as input a `text` and a `confidence_threshold`,
+               returns a list of detected language codes.
+        :type detect_languages_function: function
+
+        :return: a dict mapping keys to two letter language code and a list of tuples of pairs:
+                substring representing date expressions and corresponding :mod:`datetime.datetime` object.
+            For example:
+            {'Language': 'en', 'Dates': [('on 4 October 1957', datetime.datetime(1957, 10, 4, 0, 0))]}
+            If language of the string isn't recognised returns:
+            {'Language': None, 'Dates': None}
+        :raises: ValueError - Unknown Language
+        """
+
+        check_settings(settings)
+
+        language_shortname = self.detect_language(
+            text=text,
+            languages=languages,
+            settings=settings,
+            detect_languages_function=detect_languages_function,
+        )
+        if not language_shortname:
+            return {"Language": None, "Dates": None}
+        return {
+            "Language": language_shortname,
+            "Dates": self.search.search_parse(
+                language_shortname, text, settings=settings
+            ),
+        }

dateparser/search/text_detection.py

@@ -0,0 +1,89 @@
+from dateparser.conf import apply_settings
+from dateparser.search.detection import BaseLanguageDetector
+from dateparser.utils import normalize_unicode
+
+
+class FullTextLanguageDetector(BaseLanguageDetector):
+    def __init__(self, languages):
+        super(BaseLanguageDetector, self).__init__()
+        self.languages = languages[:]
+        self.language_unique_chars = []
+        self.language_chars = []
+
+    def get_unique_characters(self, settings):
+        settings = settings.replace(NORMALIZE=False)
+
+        for language in self.languages:
+            chars = language.get_wordchars_for_detection(settings=settings)
+            self.language_chars.append(chars)
+
+        for char_set in self.language_chars:
+            unique_chars = char_set
+            for other_char_set in self.language_chars:
+                if other_char_set != char_set:
+                    unique_chars = unique_chars - other_char_set
+            self.language_unique_chars.append(unique_chars)
+
+    def character_check(self, date_string, settings):
+        date_string_set = set(date_string.lower())
+        symbol_set = {
+            "0",
+            "1",
+            "2",
+            "3",
+            "4",
+            "5",
+            "6",
+            "7",
+            "8",
+            "9",
+            " ",
+            "/",
+            "-",
+            ")",
+            "(",
+            ".",
+            ":",
+            "\\",
+            ",",
+            "'",
+        }
+        if date_string_set & symbol_set == date_string_set:
+            self.languages = [self.languages[0]]
+            return
+        self.get_unique_characters(settings=settings)
+        for i in range(len(self.languages)):
+            for char in self.language_unique_chars[i]:
+                if char.lower() in date_string.lower():
+                    self.languages = [self.languages[i]]
+                    return
+        indices_to_pop = []
+        for i in range(len(self.languages)):
+            if len(date_string_set & self.language_chars[i]) == 0:
+                indices_to_pop.append(i)
+        self.languages = [
+            i for j, i in enumerate(self.languages) if j not in indices_to_pop
+        ]
+
+    @apply_settings
+    def _best_language(self, date_string, settings=None):
+        self.character_check(date_string, settings)
+        date_string = normalize_unicode(date_string.lower())
+        if len(self.languages) == 1:
+            return self.languages[0].shortname
+        applicable_languages = []
+        for language in self.languages:
+            num_words = language.count_applicability(
+                date_string, strip_timezone=False, settings=settings
+            )
+            if num_words[0] > 0 or num_words[1] > 0:
+                applicable_languages.append((language.shortname, num_words))
+            else:
+                num_words = language.count_applicability(
+                    date_string, strip_timezone=True, settings=settings
+                )
+                if num_words[0] > 0 or num_words[1] > 0:
+                    applicable_languages.append((language.shortname, num_words))
+        if not applicable_languages:
+            return None
+        return max(applicable_languages, key=lambda p: (p[1][0], p[1][1]))[0]

dateparser/timezone_parser.py

@@ -0,0 +1,91 @@
+from datetime import datetime, timedelta, timezone, tzinfo
+
+import regex as re
+
+from .timezones import timezone_info_list
+
+
+class StaticTzInfo(tzinfo):
+    def __init__(self, name, offset):
+        self.__offset = offset
+        self.__name = name
+
+    def tzname(self, dt):
+        return self.__name
+
+    def utcoffset(self, dt):
+        return self.__offset
+
+    def dst(self, dt):
+        return timedelta(0)
+
+    def __repr__(self):
+        return "<%s '%s'>" % (self.__class__.__name__, self.__name)
+
+    def localize(self, dt, is_dst=False):
+        if dt.tzinfo is not None:
+            raise ValueError("Not naive datetime (tzinfo is already set)")
+        return dt.replace(tzinfo=self)
+
+    def __getinitargs__(self):
+        return self.__name, self.__offset
+
+
+def pop_tz_offset_from_string(date_string, as_offset=True):
+    if _search_regex_ignorecase.search(date_string):
+        for name, info in _tz_offsets:
+            timezone_re = info["regex"]
+            timezone_match = timezone_re.search(date_string)
+            if timezone_match:
+                start, stop = timezone_match.span()
+                date_string = date_string[: start + 1] + date_string[stop:]
+                return (
+                    date_string,
+                    StaticTzInfo(name, info["offset"]) if as_offset else name,
+                )
+    return date_string, None
+
+
+def word_is_tz(word):
+    return bool(_search_regex.match(word))
+
+
+def convert_to_local_tz(datetime_obj, datetime_tz_offset):
+    return datetime_obj - datetime_tz_offset + local_tz_offset
+
+
+def build_tz_offsets(search_regex_parts):
+    def get_offset(tz_obj, regex, repl="", replw=""):
+        return (
+            tz_obj[0],
+            {
+                "regex": re.compile(
+                    re.sub(repl, replw, regex % tz_obj[0]), re.IGNORECASE
+                ),
+                "offset": timedelta(seconds=tz_obj[1]),
+            },
+        )
+
+    for tz_info in timezone_info_list:
+        for regex in tz_info["regex_patterns"]:
+            for tz_obj in tz_info["timezones"]:
+                search_regex_parts.append(tz_obj[0])
+                yield get_offset(tz_obj, regex)
+
+                # alternate patterns
+                for replace, replacewith in tz_info.get("replace", []):
+                    search_regex_parts.append(re.sub(replace, replacewith, tz_obj[0]))
+                    yield get_offset(tz_obj, regex, repl=replace, replw=replacewith)
+
+
+def get_local_tz_offset():
+    offset = datetime.now() - datetime.now(tz=timezone.utc).replace(tzinfo=None)
+    offset = timedelta(days=offset.days, seconds=round(offset.seconds, -1))
+    return offset
+
+
+_search_regex_parts = []
+_tz_offsets = list(build_tz_offsets(_search_regex_parts))
+_search_regex = re.compile("|".join(_search_regex_parts))
+_search_regex_ignorecase = re.compile("|".join(_search_regex_parts), re.IGNORECASE)
+local_tz_offset = get_local_tz_offset()