spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/base/token_assembler.py

@@ -0,0 +1,124 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the TokenAssembler."""
+
+from pyspark import keyword_only
+from pyspark.ml.param import TypeConverters, Params, Param
+
+from sparknlp.common.annotator_type import AnnotatorType
+from sparknlp.internal import AnnotatorTransformer
+
+from sparknlp.common import AnnotatorProperties
+
+
+class TokenAssembler(AnnotatorTransformer, AnnotatorProperties):
+    """This transformer reconstructs a ``DOCUMENT`` type annotation from tokens,
+    usually after these have been normalized, lemmatized, normalized, spell
+    checked, etc, in order to use this document annotation in further
+    annotators. Requires ``DOCUMENT`` and ``TOKEN`` type annotations as input.
+
+    For more extended examples on document pre-processing see the
+    `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/token-assembler/Assembling_Tokens_to_Documents.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT, TOKEN``    ``DOCUMENT``
+    ====================== ======================
+
+    Parameters
+    ----------
+    preservePosition
+        Whether to preserve the actual position of the tokens or reduce them to
+        one space
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+
+    First, the text is tokenized and cleaned
+
+    >>> documentAssembler = DocumentAssembler() \\
+    ...    .setInputCol("text") \\
+    ...    .setOutputCol("document")
+    >>> sentenceDetector = SentenceDetector() \\
+    ...    .setInputCols(["document"]) \\
+    ...    .setOutputCol("sentences")
+    >>> tokenizer = Tokenizer() \\
+    ...    .setInputCols(["sentences"]) \\
+    ...    .setOutputCol("token")
+    >>> normalizer = Normalizer() \\
+    ...    .setInputCols(["token"]) \\
+    ...    .setOutputCol("normalized") \\
+    ...    .setLowercase(False)
+    >>> stopwordsCleaner = StopWordsCleaner() \\
+    ...    .setInputCols(["normalized"]) \\
+    ...    .setOutputCol("cleanTokens") \\
+    ...    .setCaseSensitive(False)
+
+    Then the TokenAssembler turns the cleaned tokens into a ``DOCUMENT`` type
+    structure.
+
+    >>> tokenAssembler = TokenAssembler() \\
+    ...    .setInputCols(["sentences", "cleanTokens"]) \\
+    ...    .setOutputCol("cleanText")
+    >>> data = spark.createDataFrame([["Spark NLP is an open-source text processing library for advanced natural language processing."]]) \\
+    ...    .toDF("text")
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     sentenceDetector,
+    ...     tokenizer,
+    ...     normalizer,
+    ...     stopwordsCleaner,
+    ...     tokenAssembler
+    ... ]).fit(data)
+    >>> result = pipeline.transform(data)
+    >>> result.select("cleanText").show(truncate=False)
+    +---------------------------------------------------------------------------------------------------------------------------+
+    |cleanText                                                                                                                  |
+    +---------------------------------------------------------------------------------------------------------------------------+
+    |[[document, 0, 80, Spark NLP opensource text processing library advanced natural language processing, [sentence -> 0], []]]|
+    +---------------------------------------------------------------------------------------------------------------------------+
+    """
+
+    name = "TokenAssembler"
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT, AnnotatorType.TOKEN]
+
+    outputAnnotatorType = AnnotatorType.DOCUMENT
+
+    preservePosition = Param(Params._dummy(), "preservePosition", "whether to preserve the actual position of the tokens or reduce them to one space", typeConverter=TypeConverters.toBoolean)
+
+    @keyword_only
+    def __init__(self):
+        super(TokenAssembler, self).__init__(classname="com.johnsnowlabs.nlp.TokenAssembler")
+
+    @keyword_only
+    def setParams(self):
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    def setPreservePosition(self, value):
+        """Sets whether to preserve the actual position of the tokens or reduce
+        them to one space.
+
+        Parameters
+        ----------
+        value : str
+            Name of the Id Column
+        """
+        return self._set(preservePosition=value)

sparknlp/common/__init__.py

@@ -0,0 +1,26 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Module of common base classes for Spark NLP annotators."""
+from sparknlp.common.annotator_approach import *
+from sparknlp.common.annotator_model import *
+from sparknlp.common.annotator_properties import *
+from sparknlp.common.coverage_result import *
+from sparknlp.common.properties import *
+from sparknlp.common.read_as import *
+from sparknlp.common.recursive_annotator_approach import *
+from sparknlp.common.storage import *
+from sparknlp.common.utils import *
+from sparknlp.common.annotator_type import *
+from sparknlp.common.match_strategy import *
+from sparknlp.common.completion_post_processing import *

sparknlp/common/annotator_approach.py

@@ -0,0 +1,41 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains the base classes for Annotator Approaches."""
+
+from pyspark import keyword_only
+from pyspark.ml.util import JavaMLWritable
+from pyspark.ml.wrapper import JavaEstimator
+
+import sparknlp.internal as _internal
+from sparknlp.common.annotator_properties import AnnotatorProperties
+
+
+class AnnotatorApproach(JavaEstimator, JavaMLWritable, _internal.AnnotatorJavaMLReadable, AnnotatorProperties,
+                        _internal.ParamsGettersSetters):
+
+    @keyword_only
+    def __init__(self, classname):
+        _internal.ParamsGettersSetters.__init__(self)
+        self.__class__._java_class_name = classname
+        self._java_obj = self._new_java_obj(classname, self.uid)
+        self._setDefault(lazyAnnotator=False)
+
+    def _create_model(self, java_model):
+        raise NotImplementedError('Please implement _create_model in %s' % self)
+
+    def __init_subclass__(cls, **kwargs):
+        for required in ('inputAnnotatorTypes', 'outputAnnotatorType'):
+            if not getattr(cls, required):
+                raise TypeError(f"Can't instantiate class {cls.__name__} without {required} attribute defined")
+        return super().__init_subclass__(**kwargs)

sparknlp/common/annotator_model.py

@@ -0,0 +1,47 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains the base classes for Annotator Models."""
+
+from pyspark import keyword_only
+from pyspark.ml.util import JavaMLWritable
+from pyspark.ml.wrapper import JavaModel
+
+import sparknlp.internal as _internal
+from sparknlp.common import AnnotatorProperties
+
+
+class AnnotatorModel(JavaModel, _internal.AnnotatorJavaMLReadable, JavaMLWritable, AnnotatorProperties,
+                     _internal.ParamsGettersSetters):
+
+    @keyword_only
+    def setParams(self):
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    @keyword_only
+    def __init__(self, classname, java_model=None):
+        super(AnnotatorModel, self).__init__(java_model=java_model)
+        if classname and not java_model:
+            self.__class__._java_class_name = classname
+            self._java_obj = self._new_java_obj(classname, self.uid)
+        if java_model is not None:
+            self._transfer_params_from_java()
+        self._setDefault(lazyAnnotator=False)
+
+    def __init_subclass__(cls, **kwargs):
+        for required in ('inputAnnotatorTypes', 'outputAnnotatorType'):
+            if not getattr(cls, required):
+                raise TypeError(f"Can't instantiate class {cls.__name__} without {required} attribute defined")
+
+        return super().__init_subclass__(**kwargs)

sparknlp/common/annotator_properties.py

@@ -0,0 +1,114 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains the base classes for Annotator properties."""
+
+from pyspark.ml.param import TypeConverters, Params, Param
+
+
+class AnnotatorProperties(Params):
+
+    inputAnnotatorTypes = []
+    optionalInputAnnotatorTypes = []
+
+    outputAnnotatorType = None
+
+    inputCols = Param(Params._dummy(),
+                      "inputCols",
+                      "previous annotations columns, if renamed",
+                      typeConverter=TypeConverters.toListString)
+
+    outputCol = Param(Params._dummy(),
+                      "outputCol",
+                      "output annotation column. can be left default.",
+                      typeConverter=TypeConverters.toString)
+
+    lazyAnnotator = Param(Params._dummy(),
+                          "lazyAnnotator",
+                          "Whether this AnnotatorModel acts as lazy in RecursivePipelines",
+                          typeConverter=TypeConverters.toBoolean
+                          )
+
+    def setInputCols(self, *value):
+        """Sets column names of input annotations.
+
+        Parameters
+        ----------
+        *value : List[str]
+            Input columns for the annotator
+        """
+        if type(value[0]) == str or type(value[0]) == list:
+            self.inputColsValidation(value)
+            if len(value) == 1 and type(value[0]) == list:
+                return self._set(inputCols=value[0])
+            else:
+                return self._set(inputCols=list(value))
+        else:
+            raise TypeError("InputCols datatype not supported. It must be either str or list")
+
+    def inputColsValidation(self, value):
+        actual_columns = len(value)
+        if type(value[0]) == list:
+            actual_columns = len(value[0])
+
+        expected_columns = len(self.inputAnnotatorTypes)
+
+        if len(self.optionalInputAnnotatorTypes) == 0:
+            if actual_columns != expected_columns:
+                raise TypeError(
+                    f"setInputCols in {self.uid} expecting {expected_columns} columns. "
+                    f"Provided column amount: {actual_columns}. "
+                    f"Which should be columns from the following annotators: {self.inputAnnotatorTypes}")
+        else:
+            expected_columns = expected_columns + len(self.optionalInputAnnotatorTypes)
+            if not (actual_columns == len(self.inputAnnotatorTypes) or actual_columns == expected_columns):
+                raise TypeError(
+                    f"setInputCols in {self.uid} expecting at least {len(self.inputAnnotatorTypes)} columns. "
+                    f"Provided column amount: {actual_columns}. "
+                    f"Which should be columns from at least the following annotators: {self.inputAnnotatorTypes}")
+
+    def getInputCols(self):
+        """Gets current column names of input annotations."""
+        return self.getOrDefault(self.inputCols)
+
+    def setOutputCol(self, value):
+        """Sets output column name of annotations.
+
+        Parameters
+        ----------
+        value : str
+            Name of output column
+        """
+        return self._set(outputCol=value)
+
+    def getOutputCol(self):
+        """Gets output column name of annotations."""
+        return self.getOrDefault(self.outputCol)
+
+    def setLazyAnnotator(self, value):
+        """Sets whether Annotator should be evaluated lazily in a
+        RecursivePipeline.
+
+        Parameters
+        ----------
+        value : bool
+            Whether Annotator should be evaluated lazily in a
+            RecursivePipeline
+        """
+        return self._set(lazyAnnotator=value)
+
+    def getLazyAnnotator(self):
+        """Gets whether Annotator should be evaluated lazily in a
+        RecursivePipeline.
+        """
+        return self.getOrDefault(self.lazyAnnotator)

sparknlp/common/annotator_type.py

@@ -0,0 +1,38 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+class AnnotatorType(object):
+    AUDIO = "audio"
+    DOCUMENT = "document"
+    IMAGE = "image"
+    TOKEN = "token"
+    WORDPIECE = "wordpiece"
+    WORD_EMBEDDINGS = "word_embeddings"
+    SENTENCE_EMBEDDINGS = "sentence_embeddings"
+    CATEGORY = "category"
+    DATE = "date"
+    ENTITY = "entity"
+    SENTIMENT = "sentiment"
+    POS = "pos"
+    CHUNK = "chunk"
+    NAMED_ENTITY = "named_entity"
+    NEGEX = "negex"
+    DEPENDENCY = "dependency"
+    LABELED_DEPENDENCY = "labeled_dependency"
+    LANGUAGE = "language"
+    NODE = "node"
+    TABLE = "table"
+    DUMMY = "dummy"
+    DOC_SIMILARITY_RANKINGS = "doc_similarity_rankings"

sparknlp/common/completion_post_processing.py

@@ -0,0 +1,37 @@
+#  Copyright 2017-2025 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from pyspark.ml.param import Param, Params, TypeConverters
+
+
+class CompletionPostProcessing:
+    removeThinkingTag = Param(
+        Params._dummy(),
+        "removeThinkingTag",
+        "Set a thinking tag (e.g. think) to be removed from output. Will match <TAG>...</TAG>",
+        typeConverter=TypeConverters.toString,
+    )
+
+    def setRemoveThinkingTag(self, value: str):
+        """Set a thinking tag (e.g. `think`) to be removed from output.
+        Will produce the regex: `(?s)<$TAG>.+?</$TAG>`
+        """
+        self._set(removeThinkingTag=value)
+        return self
+
+    def getRemoveThinkingTag(self):
+        """Get the thinking tag to be removed from output."""
+        value = None
+        if self.removeThinkingTag in self._paramMap:
+            value = self._paramMap[self.removeThinkingTag]
+        return value

sparknlp/common/coverage_result.py

@@ -0,0 +1,22 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains utilities for coverage analysis."""
+
+
+class CoverageResult:
+    def __init__(self, cov_obj):
+        self.covered = cov_obj.covered()
+        self.total = cov_obj.total()
+        self.percentage = cov_obj.percentage()
+

sparknlp/common/match_strategy.py

@@ -0,0 +1,33 @@
+#  Copyright 2017-2023 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Allowed strategies for RuleFactory applications regarding replacement"""
+
+
+class MatchStrategy(object):
+    """Object that contains constants for how for matched strategies used in RuleFactory.
+
+    Possible values are:
+
+    ================================== ===============================================================================
+    Value                              Description
+    ================================== ===============================================================================
+    ``MatchStrategy.MATCH_ALL``        This strategy matches all occurrences of all rules in the given text.
+    ``MatchStrategy.MATCH_FIRST``      This strategy matches only the first occurrence of each rule in the given text.
+    ``MatchStrategy.MATCH_COMPLETE``   This strategy matches only the first occurrence of each rule in the given text.
+    ================================== ===============================================================================
+    """
+    MATCH_ALL = "MATCH_ALL"
+    MATCH_FIRST = "MATCH_FIRST"
+    MATCH_COMPLETE = "MATCH_COMPLETE"