spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/annotator/token/recursive_tokenizer.py

@@ -0,0 +1,205 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the RecursiveTokenizer."""
+
+from sparknlp.common import *
+
+
+class RecursiveTokenizer(AnnotatorApproach):
+    """Tokenizes raw text recursively based on a handful of definable rules.
+
+    Unlike the Tokenizer, the RecursiveTokenizer operates based on these array
+    string parameters only:
+
+    - ``prefixes``: Strings that will be split when found at the beginning of
+      token.
+    - ``suffixes``: Strings that will be split when found at the end of token.
+    - ``infixes``: Strings that will be split when found at the middle of token.
+    - ``whitelist``: Whitelist of strings not to split
+
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/training/italian/Training_Context_Spell_Checker_Italian.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``TOKEN``
+    ====================== ======================
+
+    Parameters
+    ----------
+    prefixes
+        Strings to be considered independent tokens when found at the beginning
+        of a word, by default ["'", '"', '(', '[', '\\n']
+    suffixes
+        Strings to be considered independent tokens when found at the end of a
+        word, by default ['.', ':', '%', ',', ';', '?', "'", '"', ')', ']',
+        '\\n', '!', "'s"]
+    infixes
+        Strings to be considered independent tokens when found in the middle of
+        a word, by default ['\\n', '(', ')']
+    whitelist
+        Strings to be considered as single tokens , by default ["it\'s",
+        "that\'s", "there\'s", "he\'s", "she\'s", "what\'s", "let\'s", "who\'s",
+        "It\'s", "That\'s", "There\'s", "He\'s", "She\'s", "What\'s", "Let\'s",
+        "Who\'s"]
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> tokenizer = RecursiveTokenizer() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("token")
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     tokenizer
+    ... ])
+    >>> data = spark.createDataFrame([["One, after the Other, (and) again. PO, QAM,"]]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.select("token.result").show(truncate=False)
+    +------------------------------------------------------------------+
+    |result                                                            |
+    +------------------------------------------------------------------+
+    |[One, ,, after, the, Other, ,, (, and, ), again, ., PO, ,, QAM, ,]|
+    +------------------------------------------------------------------+
+    """
+    name = 'RecursiveTokenizer'
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.TOKEN
+
+    prefixes = Param(Params._dummy(),
+                     "prefixes",
+                     "strings to be considered independent tokens when found at the beginning of a word",
+                     typeConverter=TypeConverters.toListString)
+
+    suffixes = Param(Params._dummy(),
+                     "suffixes",
+                     "strings to be considered independent tokens when found at the end of a word",
+                     typeConverter=TypeConverters.toListString)
+
+    infixes = Param(Params._dummy(),
+                    "infixes",
+                    "strings to be considered independent tokens when found in the middle of a word",
+                    typeConverter=TypeConverters.toListString)
+
+    whitelist = Param(Params._dummy(),
+                      "whitelist",
+                      "strings to be considered as single tokens",
+                      typeConverter=TypeConverters.toListString)
+
+    def setPrefixes(self, p):
+        """Sets strings to be considered independent tokens when found at the
+        beginning of a word, by default ["'", '"', '(', '[', '\\n'].
+
+        Parameters
+        ----------
+        p : List[str]
+            Strings to be considered independent tokens when found at the
+            beginning of a word
+        """
+        return self._set(prefixes=p)
+
+    def setSuffixes(self, s):
+        """Sets strings to be considered independent tokens when found at the
+        end of a word, by default ['.', ':', '%', ',', ';', '?', "'", '"', ')',
+        ']', '\\n', '!', "'s"].
+
+        Parameters
+        ----------
+        s : List[str]
+            Strings to be considered independent tokens when found at the end of
+            a word
+        """
+        return self._set(suffixes=s)
+
+    def setInfixes(self, i):
+        """Sets strings to be considered independent tokens when found in the
+        middle of a word, by default ['\\n', '(', ')'].
+
+        Parameters
+        ----------
+        i : List[str]
+            Strings to be considered independent tokens when found in the middle
+            of a word
+
+        Returns
+        -------
+        [type]
+            [description]
+        """
+        return self._set(infixes=i)
+
+    def setWhitelist(self, w):
+        """Sets strings to be considered as single tokens, by default ["it\'s",
+        "that\'s", "there\'s", "he\'s", "she\'s", "what\'s", "let\'s", "who\'s",
+        "It\'s", "That\'s", "There\'s", "He\'s", "She\'s", "What\'s", "Let\'s",
+        "Who\'s"].
+
+        Parameters
+        ----------
+        w : List[str]
+            Strings to be considered as single tokens
+        """
+        return self._set(whitelist=w)
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.RecursiveTokenizer"):
+        super(RecursiveTokenizer, self).__init__(classname="com.johnsnowlabs.nlp.annotators.RecursiveTokenizer")
+        self._setDefault(
+            prefixes=["'", "\"", "(", "[", "\n"],
+            infixes=["\n", "(", ")"],
+            suffixes=[".", ":", "%", ",", ";", "?", "'", "\"", ")", "]", "\n", "!", "'s"],
+            whitelist=["it's", "that's", "there's", "he's", "she's", "what's", "let's", "who's", \
+                       "It's", "That's", "There's", "He's", "She's", "What's", "Let's", "Who's"]
+        )
+
+    def _create_model(self, java_model):
+        return RecursiveTokenizerModel(java_model=java_model)
+
+
+class RecursiveTokenizerModel(AnnotatorModel):
+    """Instantiated model of the RecursiveTokenizer.
+
+    This is the instantiated model of the :class:`.RecursiveTokenizer`.
+    For training your own model, please see the documentation of that class.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``TOKEN``
+    ====================== ======================
+
+    Parameters
+    ----------
+    None
+    """
+    name = 'RecursiveTokenizerModel'
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.TOKEN
+
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.RecursiveTokenizerModel", java_model=None):
+        super(RecursiveTokenizerModel, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )

sparknlp/annotator/token/regex_tokenizer.py

@@ -0,0 +1,208 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the RegexTokenizer."""
+
+
+from sparknlp.common import *
+
+
+class RegexTokenizer(AnnotatorModel):
+    """A tokenizer that splits text by a regex pattern.
+
+    The pattern needs to be set with :meth:`.setPattern` and this sets the
+    delimiting pattern or how the tokens should be split. By default this
+    pattern is ``\\s+`` which means that tokens should be split by 1 or more
+    whitespace characters.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``TOKEN``
+    ====================== ======================
+
+    Parameters
+    ----------
+    minLength
+        Set the minimum allowed length for each token, by default 1
+    maxLength
+        Set the maximum allowed length for each token
+    toLowercase
+        Indicates whether to convert all characters to lowercase before
+        tokenizing, by default False
+    pattern
+        Regex pattern used for tokenizing, by default ``\\s+``
+    positionalMask
+        Using a positional mask to guarantee the incremental progression of the
+        tokenization, by default False
+    trimWhitespace
+        Using a trimWhitespace flag to remove whitespaces from identified tokens,
+        by default False
+    preservePosition
+        Using a preservePosition flag to preserve initial indexes before eventual whitespaces removal in tokens,
+        by default True
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> regexTokenizer = RegexTokenizer() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("regexToken") \\
+    ...     .setToLowercase(True) \\
+    >>> pipeline = Pipeline().setStages([
+    ...       documentAssembler,
+    ...       regexTokenizer
+    ...     ])
+    >>> data = spark.createDataFrame([["This is my first sentence.\\nThis is my second."]]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("regexToken.result").show(truncate=False)
+    +-------------------------------------------------------+
+    |result                                                 |
+    +-------------------------------------------------------+
+    |[this, is, my, first, sentence., this, is, my, second.]|
+    +-------------------------------------------------------+
+    """
+
+    name = "RegexTokenizer"
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.TOKEN
+
+    @keyword_only
+    def __init__(self):
+        super(RegexTokenizer, self).__init__(classname="com.johnsnowlabs.nlp.annotators.RegexTokenizer")
+        self._setDefault(
+            inputCols=["document"],
+            outputCol="regexToken",
+            toLowercase=False,
+            minLength=1,
+            pattern="\\s+",
+            positionalMask=False,
+            trimWhitespace=False,
+            preservePosition=True
+        )
+
+    minLength = Param(Params._dummy(),
+                      "minLength",
+                      "Set the minimum allowed length for each token",
+                      typeConverter=TypeConverters.toInt)
+
+    maxLength = Param(Params._dummy(),
+                      "maxLength",
+                      "Set the maximum allowed length for each token",
+                      typeConverter=TypeConverters.toInt)
+
+    toLowercase = Param(Params._dummy(),
+                        "toLowercase",
+                        "Indicates whether to convert all characters to lowercase before tokenizing.",
+                        typeConverter=TypeConverters.toBoolean)
+
+    pattern = Param(Params._dummy(),
+                    "pattern",
+                    "regex pattern used for tokenizing. Defaults \S+",
+                    typeConverter=TypeConverters.toString)
+
+    positionalMask = Param(Params._dummy(),
+                           "positionalMask",
+                           "Using a positional mask to guarantee the incremental progression of the tokenization.",
+                           typeConverter=TypeConverters.toBoolean)
+
+    trimWhitespace = Param(Params._dummy(),
+                           "trimWhitespace",
+                           "Indicates whether to use a trimWhitespaces flag to remove whitespaces from identified tokens.",
+                           typeConverter=TypeConverters.toBoolean)
+
+    preservePosition = Param(Params._dummy(),
+                             "preservePosition",
+                             "Indicates whether to use a preserve initial indexes before eventual whitespaces removal in tokens.",
+                             typeConverter=TypeConverters.toBoolean)
+
+    def setMinLength(self, value):
+        """Sets the minimum allowed length for each token, by default 1.
+
+        Parameters
+        ----------
+        value : int
+            Minimum allowed length for each token
+        """
+        return self._set(minLength=value)
+
+    def setMaxLength(self, value):
+        """Sets the maximum allowed length for each token.
+
+        Parameters
+        ----------
+        value : int
+            Maximum allowed length for each token
+        """
+        return self._set(maxLength=value)
+
+    def setToLowercase(self, value):
+        """Sets whether to convert all characters to lowercase before
+        tokenizing, by default False.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to convert all characters to lowercase before tokenizing
+        """
+        return self._set(toLowercase=value)
+
+    def setPattern(self, value):
+        """Sets the regex pattern used for tokenizing, by default ``\\s+``.
+
+        Parameters
+        ----------
+        value : str
+            Regex pattern used for tokenizing
+        """
+        return self._set(pattern=value)
+
+    def setPositionalMask(self, value):
+        """Sets whether to use a positional mask to guarantee the incremental
+        progression of the tokenization, by default False.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to use a positional mask
+        """
+        return self._set(positionalMask=value)
+
+    def setTrimWhitespace(self, value):
+        """Indicates whether to use a trimWhitespaces flag to remove whitespaces from identified tokens.
+
+        Parameters
+        ----------
+        value : bool
+            Indicates whether to use a trimWhitespaces flag, by default False.
+        """
+        return self._set(trimWhitespace=value)
+
+    def setPreservePosition(self, value):
+        """Indicates whether to use a preserve initial indexes before eventual whitespaces removal in tokens.
+
+        Parameters
+        ----------
+        value : bool
+            Indicates whether to use a preserve initial indexes, by default True.
+        """
+        return self._set(preservePosition=value)
+