spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/annotator/pos/perceptron.py

@@ -0,0 +1,263 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the Perceptron Annotator."""
+
+from sparknlp.common import *
+
+
+class PerceptronApproach(AnnotatorApproach):
+    """Trains an averaged Perceptron model to tag words part-of-speech. Sets a
+    POS tag to each word within a sentence.
+
+    For pretrained models please see the :class:`.PerceptronModel`.
+
+    The training data needs to be in a Spark DataFrame, where the column needs
+    to consist of Annotations of type ``POS``. The `Annotation` needs to have
+    member ``result`` set to the POS tag and have a ``"word"`` mapping to its
+    word inside of member ``metadata``. This DataFrame for training can easily
+    created by the helper class :class:`.POS`.
+
+
+    >>> POS().readDataset(spark, datasetPath) \\
+    ...     .selectExpr("explode(tags) as tags").show(truncate=False)
+    +---------------------------------------------+
+    |tags                                         |
+    +---------------------------------------------+
+    |[pos, 0, 5, NNP, [word -> Pierre], []]       |
+    |[pos, 7, 12, NNP, [word -> Vinken], []]      |
+    |[pos, 14, 14, ,, [word -> ,], []]            |
+    |[pos, 31, 34, MD, [word -> will], []]        |
+    |[pos, 36, 39, VB, [word -> join], []]        |
+    |[pos, 41, 43, DT, [word -> the], []]         |
+    |[pos, 45, 49, NN, [word -> board], []]       |
+                            ...
+
+
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/training/french/Train-Perceptron-French.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``TOKEN, DOCUMENT``    ``POS``
+    ====================== ======================
+
+    Parameters
+    ----------
+    posCol
+        Column name for Array of POS tags that match tokens
+    nIterations
+        Number of iterations in training, converges to better accuracy, by
+        default 5
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from sparknlp.training import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> sentence = SentenceDetector() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("sentence")
+    >>> tokenizer = Tokenizer() \\
+    ...     .setInputCols(["sentence"]) \\
+    ...     .setOutputCol("token")
+    >>> datasetPath = "src/test/resources/anc-pos-corpus-small/test-training.txt"
+    >>> trainingPerceptronDF = POS().readDataset(spark, datasetPath)
+    >>> trainedPos = PerceptronApproach() \\
+    ...     .setInputCols(["document", "token"]) \\
+    ...     .setOutputCol("pos") \\
+    ...     .setPosColumn("tags") \\
+    ...     .fit(trainingPerceptronDF)
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     sentence,
+    ...     tokenizer,
+    ...     trainedPos
+    ... ])
+    >>> data = spark.createDataFrame([["To be or not to be, is this the question?"]]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("pos.result").show(truncate=False)
+    +--------------------------------------------------+
+    |result                                            |
+    +--------------------------------------------------+
+    |[NNP, NNP, CD, JJ, NNP, NNP, ,, MD, VB, DT, CD, .]|
+    +--------------------------------------------------+
+    """
+
+    inputAnnotatorTypes = [AnnotatorType.TOKEN, AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.POS
+
+    posCol = Param(Params._dummy(),
+                   "posCol",
+                   "column of Array of POS tags that match tokens",
+                   typeConverter=TypeConverters.toString)
+
+    nIterations = Param(Params._dummy(),
+                        "nIterations",
+                        "Number of iterations in training, converges to better accuracy",
+                        typeConverter=TypeConverters.toInt)
+
+    @keyword_only
+    def __init__(self):
+        super(PerceptronApproach, self).__init__(
+            classname="com.johnsnowlabs.nlp.annotators.pos.perceptron.PerceptronApproach")
+        self._setDefault(
+            nIterations=5
+        )
+
+    def setPosColumn(self, value):
+        """Sets column name for Array of POS tags that match tokens.
+
+        Parameters
+        ----------
+        value : str
+            Name of column for Array of POS tags
+        """
+        return self._set(posCol=value)
+
+    def setIterations(self, value):
+        """Sets number of iterations in training, by default 5.
+
+        Parameters
+        ----------
+        value : int
+            Number of iterations in training
+        """
+        return self._set(nIterations=value)
+
+    def getNIterations(self):
+        """Gets number of iterations in training, by default 5.
+
+        Returns
+        -------
+        int
+            Number of iterations in training
+        """
+        return self.getOrDefault(self.nIterations)
+
+    def _create_model(self, java_model):
+        return PerceptronModel(java_model=java_model)
+
+
+class PerceptronModel(AnnotatorModel):
+    """Averaged Perceptron model to tag words part-of-speech. Sets a POS tag to
+    each word within a sentence.
+
+    This is the instantiated model of the :class:`.PerceptronApproach`. For
+    training your own model, please see the documentation of that class.
+
+    Pretrained models can be loaded with :meth:`.pretrained` of the companion
+    object:
+
+    >>> posTagger = PerceptronModel.pretrained() \\
+    ...     .setInputCols(["document", "token"]) \\
+    ...     .setOutputCol("pos")
+
+
+    The default model is ``"pos_anc"``, if no name is provided.
+
+    For available pretrained models please see the `Models Hub
+    <https://sparknlp.org/models?task=Part+of+Speech+Tagging>`__.
+    Additionally, pretrained pipelines are available for this module, see
+    `Pipelines <https://sparknlp.org/docs/en/pipelines>`__.
+
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/training/french/Train-Perceptron-French.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``TOKEN, DOCUMENT``    ``POS``
+    ====================== ======================
+
+    Parameters
+    ----------
+    None
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> tokenizer = Tokenizer() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("token")
+    >>> posTagger = PerceptronModel.pretrained() \\
+    ...     .setInputCols(["document", "token"]) \\
+    ...     .setOutputCol("pos")
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     tokenizer,
+    ...     posTagger
+    ... ])
+    >>> data = spark.createDataFrame([["Peter Pipers employees are picking pecks of pickled peppers"]]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("explode(pos) as pos").show(truncate=False)
+    +-------------------------------------------+
+    |pos                                        |
+    +-------------------------------------------+
+    |[pos, 0, 4, NNP, [word -> Peter], []]      |
+    |[pos, 6, 11, NNP, [word -> Pipers], []]    |
+    |[pos, 13, 21, NNS, [word -> employees], []]|
+    |[pos, 23, 25, VBP, [word -> are], []]      |
+    |[pos, 27, 33, VBG, [word -> picking], []]  |
+    |[pos, 35, 39, NNS, [word -> pecks], []]    |
+    |[pos, 41, 42, IN, [word -> of], []]        |
+    |[pos, 44, 50, JJ, [word -> pickled], []]   |
+    |[pos, 52, 58, NNS, [word -> peppers], []]  |
+    +-------------------------------------------+
+    """
+    name = "PerceptronModel"
+
+    inputAnnotatorTypes = [AnnotatorType.TOKEN, AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.POS
+
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.pos.perceptron.PerceptronModel", java_model=None):
+        super(PerceptronModel, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+
+    @staticmethod
+    def pretrained(name="pos_anc", lang="en", remote_loc=None):
+        """Downloads and loads a pretrained model.
+
+        Parameters
+        ----------
+        name : str, optional
+            Name of the pretrained model, by default "pos_anc"
+        lang : str, optional
+            Language of the pretrained model, by default "en"
+        remote_loc : str, optional
+            Optional remote address of the resource, by default None. Will use
+            Spark NLPs repositories otherwise.
+
+        Returns
+        -------
+        PerceptronModel
+            The restored model
+        """
+        from sparknlp.pretrained import ResourceDownloader
+        return ResourceDownloader.downloadModel(PerceptronModel, name, lang, remote_loc)

sparknlp/annotator/sentence/__init__.py

@@ -0,0 +1,17 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Module of annotators for sentence detection."""
+from sparknlp.annotator.sentence.sentence_detector import *
+from sparknlp.annotator.sentence.sentence_detector_dl import *

sparknlp/annotator/sentence/sentence_detector.py

@@ -0,0 +1,290 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the SentenceDetector."""
+
+from sparknlp.common import *
+
+
+class SentenceDetectorParams:
+    """Base class for SentenceDetector parameters
+    """
+
+    useAbbreviations = Param(Params._dummy(),
+                             "useAbbreviations",
+                             "whether to apply abbreviations at sentence detection",
+                             typeConverter=TypeConverters.toBoolean)
+
+    customBounds = Param(Params._dummy(),
+                         "customBounds",
+                         "characters used to explicitly mark sentence bounds",
+                         typeConverter=TypeConverters.toListString)
+
+    useCustomBoundsOnly = Param(Params._dummy(),
+                                "useCustomBoundsOnly",
+                                "Only utilize custom bounds in sentence detection",
+                                typeConverter=TypeConverters.toBoolean)
+
+    customBoundsStrategy = Param(Params._dummy(),
+                                 "customBoundsStrategy",
+                                 "How to return matched custom bounds",
+                                 typeConverter=TypeConverters.toString)
+
+    explodeSentences = Param(Params._dummy(),
+                             "explodeSentences",
+                             "whether to explode each sentence into a different row, for better parallelization. Defaults to false.",
+                             typeConverter=TypeConverters.toBoolean)
+
+    splitLength = Param(Params._dummy(),
+                        "splitLength",
+                        "length at which sentences will be forcibly split.",
+                        typeConverter=TypeConverters.toInt)
+
+    minLength = Param(Params._dummy(),
+                      "minLength",
+                      "Set the minimum allowed length for each sentence.",
+                      typeConverter=TypeConverters.toInt)
+
+    maxLength = Param(Params._dummy(),
+                      "maxLength",
+                      "Set the maximum allowed length for each sentence",
+                      typeConverter=TypeConverters.toInt)
+
+
+class SentenceDetector(AnnotatorModel, SentenceDetectorParams):
+    """Annotator that detects sentence boundaries using regular expressions.
+
+    The following characters are checked as sentence boundaries:
+
+    1. Lists ("(i), (ii)", "(a), (b)", "1., 2.")
+    2. Numbers
+    3. Abbreviations
+    4. Punctuations
+    5. Multiple Periods
+    6. Geo-Locations/Coordinates ("N°. 1026.253.553.")
+    7. Ellipsis ("...")
+    8. In-between punctuations
+    9. Quotation marks
+    10. Exclamation Points
+    11. Basic Breakers (".", ";")
+
+    For the explicit regular expressions used for detection, refer to source of
+    `PragmaticContentFormatter <https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/main/scala/com/johnsnowlabs/nlp/annotators/sbd/pragmatic/PragmaticContentFormatter.scala>`__.
+
+    To add additional custom bounds, the parameter ``customBounds`` can be set with an array:
+
+    >>> sentence = SentenceDetector() \\
+    >>>     .setInputCols(["document"]) \\
+    >>>     .setOutputCol("sentence") \\
+    >>>     .setCustomBounds(["\\n\\n"])
+
+    If only the custom bounds should be used, then the parameter ``useCustomBoundsOnly`` should be set to ``true``.
+
+    Each extracted sentence can be returned in an Array or exploded to separate rows,
+    if ``explodeSentences`` is set to ``true``.
+
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/sentence-detection/SentenceDetector_advanced_examples.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``DOCUMENT``
+    ====================== ======================
+
+    Parameters
+    ----------
+    useAbbreviations
+        whether to apply abbreviations at sentence detection, by default True
+    customBounds
+        characters used to explicitly mark sentence bounds, by default []
+    useCustomBoundsOnly
+        Only utilize custom bounds in sentence detection, by default False
+    customBoundsStrategy
+        Sets how to return matched custom bounds, by default "none".
+
+        Will have no effect if no custom bounds are used.
+        Possible values are:
+
+        - "none" - Will not return the matched bound
+        - "prepend" - Prepends a sentence break to the match
+        - "append" - Appends a sentence break to the match
+    explodeSentences
+        whether to explode each sentence into a different row, for better
+        parallelization, by default False
+    splitLength
+        length at which sentences will be forcibly split
+    minLength
+        Set the minimum allowed length for each sentence, by default 0
+    maxLength
+        Set the maximum allowed length for each sentence, by default 99999
+    detectLists
+        whether detect lists during sentence detection, by default True
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> sentence = SentenceDetector() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("sentence")
+    ...     .setCustomBounds(["\\n\\n"])
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     sentence
+    ... ])
+    >>> data = spark.createDataFrame([["This is my first sentence. This my second.\\n\\nHow about a third?"]]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("explode(sentence) as sentences").show(truncate=False)
+    +------------------------------------------------------------------+
+    |sentences                                                         |
+    +------------------------------------------------------------------+
+    |[document, 0, 25, This is my first sentence., [sentence -> 0], []]|
+    |[document, 27, 41, This my second., [sentence -> 1], []]          |
+    |[document, 43, 60, How about a third?, [sentence -> 2], []]       |
+    +------------------------------------------------------------------+
+    """
+
+    name = 'SentenceDetector'
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.DOCUMENT
+
+    # this one is exclusive to this detector
+    detectLists = Param(Params._dummy(),
+                        "detectLists",
+                        "whether detect lists during sentence detection",
+                        typeConverter=TypeConverters.toBoolean)
+
+    def setCustomBounds(self, value):
+        """Sets characters used to explicitly mark sentence bounds, by default
+        [].
+
+        Parameters
+        ----------
+        value : List[str]
+            Characters used to explicitly mark sentence bounds
+        """
+        return self._set(customBounds=value)
+
+    def setCustomBoundsStrategy(self, value):
+        """Sets how to return matched custom bounds, by default "none".
+
+        Will have no effect if no custom bounds are used.
+        Possible values are:
+
+        - "none" - Will not return the matched bound
+        - "prepend" - Prepends a sentence break to the match
+        - "append" - Appends a sentence break to the match
+
+        Parameters
+        ----------
+        value : str
+            Strategy to use
+        """
+        return self._set(customBoundsStrategy=value)
+
+    def setUseAbbreviations(self, value):
+        """Sets whether to apply abbreviations at sentence detection, by default
+        True
+
+        Parameters
+        ----------
+        value : bool
+            Whether to apply abbreviations at sentence detection
+        """
+        return self._set(useAbbreviations=value)
+
+    def setDetectLists(self, value):
+        """Sets whether detect lists during sentence detection, by default True
+
+        Parameters
+        ----------
+        value : bool
+            Whether detect lists during sentence detection
+        """
+        return self._set(detectLists=value)
+
+    def setUseCustomBoundsOnly(self, value):
+        """Sets whether to only utilize custom bounds in sentence detection, by
+        default False.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to only utilize custom bounds
+        """
+        return self._set(useCustomBoundsOnly=value)
+
+    def setExplodeSentences(self, value):
+        """Sets whether to explode each sentence into a different row, for
+        better parallelization, by default False.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to explode each sentence into a different row
+        """
+        return self._set(explodeSentences=value)
+
+    def setSplitLength(self, value):
+        """Sets length at which sentences will be forcibly split.
+
+        Parameters
+        ----------
+        value : int
+            Length at which sentences will be forcibly split.
+        """
+        return self._set(splitLength=value)
+
+    def setMinLength(self, value):
+        """Sets minimum allowed length for each sentence, by default 0
+
+        Parameters
+        ----------
+        value : int
+            Minimum allowed length for each sentence
+        """
+        return self._set(minLength=value)
+
+    def setMaxLength(self, value):
+        """Sets the maximum allowed length for each sentence, by default
+        99999
+
+        Parameters
+        ----------
+        value : int
+            Maximum allowed length for each sentence
+        """
+        return self._set(maxLength=value)
+
+    @keyword_only
+    def __init__(self):
+        super(SentenceDetector, self).__init__(
+            classname="com.johnsnowlabs.nlp.annotators.sbd.pragmatic.SentenceDetector")
+        self._setDefault(
+            useAbbreviations=True,
+            detectLists=True,
+            useCustomBoundsOnly=False,
+            customBounds=[],
+            customBoundsStrategy="none",
+            explodeSentences=False,
+            minLength=0,
+            maxLength=99999
+        )