spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/annotator/matcher/date_matcher.py

@@ -0,0 +1,303 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the DateMatcher."""
+
+from sparknlp.common import *
+
+
+class DateMatcherUtils(Params):
+    """Base class for DateMatcher Annotators
+    """
+    inputFormats = Param(Params._dummy(),
+                         "inputFormats",
+                         "input formats list of patterns to match",
+                         typeConverter=TypeConverters.toListString)
+
+    outputFormat = Param(Params._dummy(),
+                         "outputFormat",
+                         "desired output format for dates extracted",
+                         typeConverter=TypeConverters.toString)
+
+    readMonthFirst = Param(Params._dummy(),
+                           "readMonthFirst",
+                           "Whether to parse july 07/05/2015 or as 05/07/2015",
+                           typeConverter=TypeConverters.toBoolean
+                           )
+
+    defaultDayWhenMissing = Param(Params._dummy(),
+                                  "defaultDayWhenMissing",
+                                  "which day to set when it is missing from parsed input",
+                                  typeConverter=TypeConverters.toInt
+                                  )
+
+    anchorDateYear = Param(Params._dummy(),
+                           "anchorDateYear",
+                           "Add an anchor year for the relative dates such as a day after tomorrow. If not set it "
+                           "will use the current year. Example: 2021",
+                           typeConverter=TypeConverters.toInt
+                           )
+
+    anchorDateMonth = Param(Params._dummy(),
+                            "anchorDateMonth",
+                            "Add an anchor month for the relative dates such as a day after tomorrow. If not set it "
+                            "will use the current month. Example: 1 which means January",
+                            typeConverter=TypeConverters.toInt
+                            )
+
+    anchorDateDay = Param(Params._dummy(),
+                          "anchorDateDay",
+                          "Add an anchor day of the day for the relative dates such as a day after tomorrow. If not "
+                          "set it will use the current day. Example: 11",
+                          typeConverter=TypeConverters.toInt
+                          )
+
+    sourceLanguage = Param(Params._dummy(),
+                           "sourceLanguage",
+                           "source language for explicit translation",
+                           typeConverter=TypeConverters.toString)
+
+    relaxedFactoryStrategy = Param(Params._dummy(),
+                                   "relaxedFactoryStrategy",
+                                   "Matched Strategy to searches relaxed dates",
+                                   typeConverter=TypeConverters.toString)
+
+    aggressiveMatching = Param(Params._dummy(),
+                               "aggressiveMatching",
+                               "Whether to aggressively attempt to find date matches, even in ambiguous or less common formats",
+                               typeConverter=TypeConverters.toBoolean)
+
+    def setInputFormats(self, value):
+        """Sets input formats patterns to match in the documents.
+
+        Parameters
+        ----------
+        value : List[str]
+            Input formats regex patterns to match dates in documents
+        """
+        return self._set(inputFormats=value)
+
+    def setOutputFormat(self, value):
+        """Sets desired output format for extracted dates, by default yyyy/MM/dd.
+
+        Not all of the date information needs to be included. For example
+        ``"YYYY"`` is also a valid input.
+
+        Parameters
+        ----------
+        value : str
+            Desired output format for dates extracted.
+        """
+        return self._set(outputFormat=value)
+
+    def setReadMonthFirst(self, value):
+        """Sets whether to parse the date in mm/dd/yyyy format instead of
+        dd/mm/yyyy, by default True.
+
+        For example July 5th 2015, would be parsed as 07/05/2015 instead of
+        05/07/2015.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to parse the date in mm/dd/yyyy format instead of
+            dd/mm/yyyy.
+        """
+        return self._set(readMonthFirst=value)
+
+    def setDefaultDayWhenMissing(self, value):
+        """Sets which day to set when it is missing from parsed input,
+        by default 1.
+
+        Parameters
+        ----------
+        value : int
+            [description]
+        """
+        return self._set(defaultDayWhenMissing=value)
+
+    def setAnchorDateYear(self, value):
+        """Sets an anchor year for the relative dates such as a day after
+        tomorrow. If not set it will use the current year.
+
+        Example: 2021
+
+        Parameters
+        ----------
+        value : int
+            The anchor year for relative dates
+        """
+        return self._set(anchorDateYear=value)
+
+    def setAnchorDateMonth(self, value):
+        """Sets an anchor month for the relative dates such as a day after
+        tomorrow. If not set it will use the current month.
+
+        Example: 1 which means January
+
+        Parameters
+        ----------
+        value : int
+            The anchor month for relative dates
+        """
+        normalizedMonth = value - 1
+        return self._set(anchorDateMonth=normalizedMonth)
+
+    def setSourceLanguage(self, value):
+        return self._set(sourceLanguage=value)
+
+    def setAnchorDateDay(self, value):
+        """Sets an anchor day of the day for the relative dates such as a day
+        after tomorrow. If not set it will use the current day.
+
+        Example: 11
+
+        Parameters
+        ----------
+        value : int
+            The anchor day for relative dates
+        """
+        return self._set(anchorDateDay=value)
+
+    def setRelaxedFactoryStrategy(self, matchStrategy=MatchStrategy.MATCH_FIRST):
+        """ Sets matched strategy to search relaxed dates by ordered rules by more exhaustive to less Strategy.
+
+        Not all of the date information needs to be included. For example
+        ``"YYYY"`` is also a valid input.
+
+        Parameters
+        ----------
+        matchStrategy : MatchStrategy
+            Matched strategy to search relaxed dates by ordered rules by more exhaustive to less Strategy
+        """
+        return self._set(relaxedFactoryStrategy=matchStrategy)
+
+    def setAggressiveMatching(self, value):
+        """ Sets whether to aggressively attempt to find date matches, even in ambiguous or less common formats
+
+        Parameters
+        ----------
+        aggressiveMatching : Boolean
+            Whether to aggressively attempt to find date matches, even in ambiguous or less common formats
+        """
+        return self._set(aggressiveMatching=value)
+
+
+class DateMatcher(AnnotatorModel, DateMatcherUtils):
+    """Matches standard date formats into a provided format
+    Reads from different forms of date and time expressions and converts them
+    to a provided date format.
+
+    Extracts only **one** date per document. Use with sentence detector to find
+    matches in each sentence.
+    To extract multiple dates from a document, please use the
+    :class:`.MultiDateMatcher`.
+
+    Reads the following kind of dates::
+
+        "1978-01-28", "1984/04/02,1/02/1980", "2/28/79",
+        "The 31st of April in the year 2008", "Fri, 21 Nov 1997", "Jan 21,
+        ‘97", "Sun", "Nov 21", "jan 1st", "next thursday", "last wednesday",
+        "today", "tomorrow", "yesterday", "next week", "next month",
+        "next year", "day after", "the day before", "0600h", "06:00 hours",
+        "6pm", "5:30 a.m.", "at 5", "12:59", "23:59", "1988/11/23 6pm",
+        "next week at 7.30", "5 am tomorrow"
+
+    For example ``"The 31st of April in the year 2008"`` will be converted into
+    ``2008/04/31``.
+
+    Pretrained pipelines are available for this module, see
+    `Pipelines <https://sparknlp.org/docs/en/pipelines>`__.
+
+    For extended examples of usage, see the
+    `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/MultiDateMatcherMultiLanguage_en.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``DATE``
+    ====================== ======================
+
+    Parameters
+    ----------
+    dateFormat
+        Desired format for dates extracted, by default yyyy/MM/dd.
+    readMonthFirst
+        Whether to parse the date in mm/dd/yyyy format instead of dd/mm/yyyy,
+        by default True.
+    defaultDayWhenMissing
+        Which day to set when it is missing from parsed input, by default 1.
+    anchorDateYear
+        Add an anchor year for the relative dates such as a day after tomorrow.
+        If not set it will use the current year. Example: 2021
+    anchorDateMonth
+        Add an anchor month for the relative dates such as a day after tomorrow.
+        If not set it will use the current month. Example: 1 which means January
+    anchorDateDay
+        Add an anchor day of the day for the relative dates such as a day after
+        tomorrow. If not set it will use the current day. Example: 11
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> date = DateMatcher() \\
+    ...     .setInputCols("document") \\
+    ...     .setOutputCol("date") \\
+    ...     .setAnchorDateYear(2020) \\
+    ...     .setAnchorDateMonth(1) \\
+    ...     .setAnchorDateDay(11) \\
+    ...     .setOutputFormat("yyyy/MM/dd")
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     date
+    ... ])
+    >>> data = spark.createDataFrame([["Fri, 21 Nov 1997"], ["next week at 7.30"], ["see you a day after"]]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("date").show(truncate=False)
+    +-------------------------------------------------+
+    |date                                             |
+    +-------------------------------------------------+
+    |[[date, 5, 15, 1997/11/21, [sentence -> 0], []]] |
+    |[[date, 0, 8, 2020/01/18, [sentence -> 0], []]]  |
+    |[[date, 10, 18, 2020/01/12, [sentence -> 0], []]]|
+    +-------------------------------------------------+
+
+    See Also
+    --------
+    MultiDateMatcher : for matching multiple dates in a document
+    """
+
+    name = "DateMatcher"
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.DATE
+
+    @keyword_only
+    def __init__(self):
+        super(DateMatcher, self).__init__(classname="com.johnsnowlabs.nlp.annotators.DateMatcher")
+        self._setDefault(
+            inputFormats=[""],
+            outputFormat="yyyy/MM/dd",
+            readMonthFirst=True,
+            defaultDayWhenMissing=1,
+            anchorDateYear=-1,
+            anchorDateMonth=-1,
+            anchorDateDay=-1
+        )

sparknlp/annotator/matcher/multi_date_matcher.py

@@ -0,0 +1,109 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for MultiDateMatcher."""
+
+from sparknlp.common import *
+from sparknlp.annotator.matcher.date_matcher import DateMatcherUtils
+
+
+class MultiDateMatcher(AnnotatorModel, DateMatcherUtils):
+    """Matches standard date formats into a provided format.
+
+    Reads the following kind of dates::
+
+        "1978-01-28", "1984/04/02,1/02/1980", "2/28/79",
+        "The 31st of April in the year 2008", "Fri, 21 Nov 1997", "Jan 21,
+        ‘97", "Sun", "Nov 21", "jan 1st", "next thursday", "last wednesday",
+        "today", "tomorrow", "yesterday", "next week", "next month",
+        "next year", "day after", "the day before", "0600h", "06:00 hours",
+        "6pm", "5:30 a.m.", "at 5", "12:59", "23:59", "1988/11/23 6pm",
+        "next week at 7.30", "5 am tomorrow"
+
+    For example ``"The 31st of April in the year 2008"`` will be converted into
+    ``2008/04/31``.
+
+    For extended examples of usage, see the `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/MultiDateMatcherMultiLanguage_en.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``DATE``
+    ====================== ======================
+
+    Parameters
+    ----------
+    dateFormat
+        Desired format for dates extracted, by default yyyy/MM/dd.
+    readMonthFirst
+        Whether to parse the date in mm/dd/yyyy format instead of dd/mm/yyyy,
+        by default True.
+    defaultDayWhenMissing
+        Which day to set when it is missing from parsed input, by default 1.
+    anchorDateYear
+        Add an anchor year for the relative dates such as a day after tomorrow.
+        If not set it will use the current year. Example: 2021
+    anchorDateMonth
+        Add an anchor month for the relative dates such as a day after tomorrow.
+        If not set it will use the current month. Example: 1 which means January
+    anchorDateDay
+        Add an anchor day of the day for the relative dates such as a day after
+        tomorrow. If not set it will use the current day. Example: 11
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> date = MultiDateMatcher() \\
+    ...     .setInputCols("document") \\
+    ...     .setOutputCol("date") \\
+    ...     .setAnchorDateYear(2020) \\
+    ...     .setAnchorDateMonth(1) \\
+    ...     .setAnchorDateDay(11) \\
+    ...     .setOutputFormat("yyyy/MM/dd")
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     date
+    ... ])
+    >>> data = spark.createDataFrame([["I saw him yesterday and he told me that he will visit us next week"]]) \\
+    ...     .toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("explode(date) as dates").show(truncate=False)
+    +-----------------------------------------------+
+    |dates                                          |
+    +-----------------------------------------------+
+    |[date, 57, 65, 2020/01/18, [sentence -> 0], []]|
+    |[date, 10, 18, 2020/01/10, [sentence -> 0], []]|
+    +-----------------------------------------------+
+    """
+
+    name = "MultiDateMatcher"
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.DATE
+
+    @keyword_only
+    def __init__(self):
+        super(MultiDateMatcher, self).__init__(classname="com.johnsnowlabs.nlp.annotators.MultiDateMatcher")
+        self._setDefault(
+            inputFormats=[""],
+            outputFormat="yyyy/MM/dd",
+            readMonthFirst=True,
+            defaultDayWhenMissing=1
+        )

sparknlp/annotator/matcher/regex_matcher.py

@@ -0,0 +1,221 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the RegexMatcher."""
+
+from sparknlp.common import *
+
+
+class RegexMatcher(AnnotatorApproach):
+    """Uses rules to match a set of regular expressions and associate them with a
+    provided identifier.
+
+    A rule consists of a regex pattern and an identifier, delimited by a character of
+    choice. An example could be `"\\d{4}\\/\\d\\d\\/\\d\\d,date"` which will match
+    strings like `"1970/01/01"` to the identifier `"date"`.
+
+    Rules must be provided by either :meth:`.setRules` (followed by
+    :meth:`.setDelimiter`) or an external file.
+
+    To use an external file, a dictionary of predefined regular expressions must be
+    provided with :meth:`.setExternalRules`. The dictionary can be set in the form of a
+    delimited text file.
+
+    Pretrained pipelines are available for this module, see `Pipelines
+    <https://sparknlp.org/docs/en/pipelines>`__.
+
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/regex-matcher/Matching_Text_with_RegexMatcher.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``CHUNK``
+    ====================== ======================
+
+    Parameters
+    ----------
+    strategy
+        Can be either MATCH_FIRST|MATCH_ALL|MATCH_COMPLETE, by default
+        "MATCH_ALL"
+    rules
+        Regex rules to match the identifier with
+    delimiter
+        Delimiter for rules provided with setRules
+    externalRules
+        external resource to rules, needs 'delimiter' in options
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+
+    In this example, the ``rules.txt`` has the form of::
+
+        the\\s\\w+, followed by 'the'
+        ceremonies, ceremony
+
+    where each regex is separated by the identifier ``","``
+
+    >>> documentAssembler = DocumentAssembler().setInputCol("text").setOutputCol("document")
+    >>> sentence = SentenceDetector().setInputCols(["document"]).setOutputCol("sentence")
+    >>> regexMatcher = RegexMatcher() \\
+    ...     .setExternalRules("src/test/resources/regex-matcher/rules.txt",  ",") \\
+    ...     .setInputCols(["sentence"]) \\
+    ...     .setOutputCol("regex") \\
+    ...     .setStrategy("MATCH_ALL")
+    >>> pipeline = Pipeline().setStages([documentAssembler, sentence, regexMatcher])
+    >>> data = spark.createDataFrame([[
+    ...     "My first sentence with the first rule. This is my second sentence with ceremonies rule."
+    ... ]]).toDF("text")
+    >>> results = pipeline.fit(data).transform(data)
+    >>> results.selectExpr("explode(regex) as result").show(truncate=False)
+    +--------------------------------------------------------------------------------------------+
+    |result                                                                                      |
+    +--------------------------------------------------------------------------------------------+
+    |[chunk, 23, 31, the first, [identifier -> followed by 'the', sentence -> 0, chunk -> 0], []]|
+    |[chunk, 71, 80, ceremonies, [identifier -> ceremony, sentence -> 1, chunk -> 0], []]        |
+    +--------------------------------------------------------------------------------------------+
+    """
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.CHUNK
+
+    strategy = Param(Params._dummy(),
+                     "strategy",
+                     "MATCH_FIRST|MATCH_ALL|MATCH_COMPLETE",
+                     typeConverter=TypeConverters.toString)
+    externalRules = Param(Params._dummy(),
+                          "externalRules",
+                          "external resource to rules, needs 'delimiter' in options",
+                          typeConverter=TypeConverters.identity)
+    rules = Param(Params._dummy(),
+                  "rules",
+                  "Regex rules to match the identifier with",
+                  typeConverter=TypeConverters.toListString)
+    delimiter = Param(Params._dummy(),
+                      "delimiter",
+                      "Delimiter for rules",
+                      typeConverter=TypeConverters.toString)
+
+    @keyword_only
+    def __init__(self):
+        super(RegexMatcher, self).__init__(classname="com.johnsnowlabs.nlp.annotators.RegexMatcher")
+        self._setDefault(
+            strategy="MATCH_ALL"
+        )
+
+    def setStrategy(self, value):
+        """Sets matching strategy, by default "MATCH_ALL".
+
+        Can be either MATCH_FIRST|MATCH_ALL|MATCH_COMPLETE.
+
+        Parameters
+        ----------
+        value : str
+            Matching Strategy
+        """
+        return self._set(strategy=value)
+
+    def setExternalRules(self, path, delimiter, read_as=ReadAs.TEXT, options={"format": "text"}):
+        """Sets external resource to rules, needs 'delimiter' in options.
+
+        Only one of either parameter `rules` or `externalRules` must be set.
+
+
+        Parameters
+        ----------
+        path : str
+            Path to the source files
+        delimiter : str
+            Delimiter for the dictionary file. Can also be set it `options`.
+        read_as : str, optional
+            How to read the file, by default ReadAs.TEXT
+        options : dict, optional
+            Options to read the resource, by default {"format": "text"}
+        """
+        opts = options.copy()
+        if "delimiter" not in opts:
+            opts["delimiter"] = delimiter
+        return self._set(externalRules=ExternalResource(path, read_as, opts))
+
+    def setRules(self, value):
+        """Sets the regex rules to match the identifier with.
+
+        The rules must consist of a regex pattern and an identifier for that pattern. The regex
+        pattern and the identifier must be delimited by a character that will also have to set with
+        `setDelimiter`.
+
+        Only one of either parameter `rules` or `externalRules` must be set.
+
+         Examples
+         --------
+        >>> regexMatcher = RegexMatcher() \\
+        ...     .setRules(["\\d{4}\\/\\d\\d\\/\\d\\d,date", "\\d{2}\\/\\d\\d\\/\\d\\d,short_date"]) \\
+        ...     .setDelimiter(",") \\
+        ...     .setInputCols(["sentence"]) \\
+        ...     .setOutputCol("regex") \\
+        ...     .setStrategy("MATCH_ALL")
+
+        Parameters
+        ----------
+        value : List[str]
+             List of rules
+        """
+        return self._set(rules=value)
+
+    def setDelimiter(self, value):
+        """Sets the delimiter for rules.
+
+        Parameters
+        ----------
+        value : str
+             Delimiter for the rules
+        """
+        return self._set(delimiter=value)
+
+    def _create_model(self, java_model):
+        return RegexMatcherModel(java_model=java_model)
+
+
+class RegexMatcherModel(AnnotatorModel):
+    """Instantiated model of the RegexMatcher.
+
+    This is the instantiated model of the :class:`.RegexMatcher`.
+    For training your own model, please see the documentation of that class.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``CHUNK``
+    ====================== ======================
+
+    Parameters
+    ----------
+    None
+    """
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.CHUNK
+
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.RegexMatcherModel", java_model=None):
+        super(RegexMatcherModel, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+
+    name = "RegexMatcherModel"