spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/base/doc2_chunk.py

@@ -0,0 +1,169 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for Doc2Chunk."""
+
+from pyspark import keyword_only
+from pyspark.ml.param import TypeConverters, Params, Param
+
+from sparknlp.internal import AnnotatorTransformer
+
+from sparknlp.common import AnnotatorProperties, AnnotatorType
+
+
+class Doc2Chunk(AnnotatorTransformer, AnnotatorProperties):
+    """Converts ``DOCUMENT`` type annotations into ``CHUNK`` type with the
+    contents of a ``chunkCol``.
+
+    Chunk text must be contained within input ``DOCUMENT``. May be either
+    ``StringType`` or ``ArrayType[StringType]`` (using setIsArray). Useful for
+    annotators that require a CHUNK type input.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``CHUNK``
+    ====================== ======================
+
+    Parameters
+    ----------
+    chunkCol
+        Column that contains the string. Must be part of DOCUMENT
+    startCol
+        Column that has a reference of where the chunk begins
+    startColByTokenIndex
+        Whether start column is prepended by whitespace tokens
+    isArray
+        Whether the chunkCol is an array of strings, by default False
+    failOnMissing
+        Whether to fail the job if a chunk is not found within document.
+        Return empty otherwise
+    lowerCase
+        Whether to lower case for matching case
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.common import *
+    >>> from sparknlp.annotator import *
+    >>> from sparknlp.training import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler().setInputCol("text").setOutputCol("document")
+    >>> chunkAssembler = Doc2Chunk() \\
+    ...     .setInputCols("document") \\
+    ...     .setChunkCol("target") \\
+    ...     .setOutputCol("chunk") \\
+    ...     .setIsArray(True)
+    >>> data = spark.createDataFrame([[
+    ...     "Spark NLP is an open-source text processing library for advanced natural language processing.",
+    ...     ["Spark NLP", "text processing library", "natural language processing"]
+    ... ]]).toDF("text", "target")
+    >>> pipeline = Pipeline().setStages([documentAssembler, chunkAssembler]).fit(data)
+    >>> result = pipeline.transform(data)
+    >>> result.selectExpr("chunk.result", "chunk.annotatorType").show(truncate=False)
+    +-----------------------------------------------------------------+---------------------+
+    |result                                                           |annotatorType        |
+    +-----------------------------------------------------------------+---------------------+
+    |[Spark NLP, text processing library, natural language processing]|[chunk, chunk, chunk]|
+    +-----------------------------------------------------------------+---------------------+
+
+    See Also
+    --------
+    Chunk2Doc : for converting `CHUNK` annotations to `DOCUMENT`
+    """
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.CHUNK
+
+    chunkCol = Param(Params._dummy(), "chunkCol", "column that contains string. Must be part of DOCUMENT", typeConverter=TypeConverters.toString)
+    startCol = Param(Params._dummy(), "startCol", "column that has a reference of where chunk begins", typeConverter=TypeConverters.toString)
+    startColByTokenIndex = Param(Params._dummy(), "startColByTokenIndex", "whether start col is by whitespace tokens", typeConverter=TypeConverters.toBoolean)
+    isArray = Param(Params._dummy(), "isArray", "whether the chunkCol is an array of strings", typeConverter=TypeConverters.toBoolean)
+    failOnMissing = Param(Params._dummy(), "failOnMissing", "whether to fail the job if a chunk is not found within document. return empty otherwise", typeConverter=TypeConverters.toBoolean)
+    lowerCase = Param(Params._dummy(), "lowerCase", "whether to lower case for matching case", typeConverter=TypeConverters.toBoolean)
+    name = "Doc2Chunk"
+
+    @keyword_only
+    def __init__(self):
+        super(Doc2Chunk, self).__init__(classname="com.johnsnowlabs.nlp.Doc2Chunk")
+        self._setDefault(
+            isArray=False
+        )
+
+    @keyword_only
+    def setParams(self):
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    def setChunkCol(self, value):
+        """Sets column that contains the string. Must be part of DOCUMENT.
+
+        Parameters
+        ----------
+        value : str
+            Name of the Chunk Column
+        """
+        return self._set(chunkCol=value)
+
+    def setIsArray(self, value):
+        """Sets whether the chunkCol is an array of strings.
+
+        Parameters
+        ----------
+        value : bool
+            Whether the chunkCol is an array of strings
+        """
+        return self._set(isArray=value)
+
+    def setStartCol(self, value):
+        """Sets column that has a reference of where chunk begins.
+
+        Parameters
+        ----------
+        value : str
+            Name of the reference column
+        """
+        return self._set(startCol=value)
+
+    def setStartColByTokenIndex(self, value):
+        """Sets whether start column is prepended by whitespace tokens.
+
+        Parameters
+        ----------
+        value : bool
+            whether start column is prepended by whitespace tokens
+        """
+        return self._set(startColByTokenIndex=value)
+
+    def setFailOnMissing(self, value):
+        """Sets whether to fail the job if a chunk is not found within document.
+        Return empty otherwise.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to fail job on missing chunks
+        """
+        return self._set(failOnMissing=value)
+
+    def setLowerCase(self, value):
+        """Sets whether to lower case for matching case.
+
+        Parameters
+        ----------
+        value : bool
+            Name of the Id Column
+        """
+        return self._set(lowerCase=value)
+

sparknlp/base/document_assembler.py

@@ -0,0 +1,164 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the DocumentAssembler."""
+
+from pyspark import keyword_only
+from pyspark.ml.param import TypeConverters, Params, Param
+
+from sparknlp.common import AnnotatorType
+from sparknlp.internal import AnnotatorTransformer
+
+
+class DocumentAssembler(AnnotatorTransformer):
+    """Prepares data into a format that is processable by Spark NLP.
+
+    This is the entry point for every Spark NLP pipeline. The
+    `DocumentAssembler` reads ``String`` columns. Additionally,
+    :meth:`.setCleanupMode` can be used to pre-process the
+    text (Default: ``disabled``). For possible options please refer the
+    parameters section.
+
+    For more extended examples on document pre-processing see the
+    `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/document-assembler/Loading_Documents_With_DocumentAssembler.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``NONE``               ``DOCUMENT``
+    ====================== ======================
+
+    Parameters
+    ----------
+    inputCol
+        Input column name
+    outputCol
+        Output column name
+    idCol
+        Name of String type column for row id.
+    metadataCol
+        Name of Map type column with metadata information
+    cleanupMode
+        How to cleanup the document , by default disabled.
+        Possible values: ``disabled, inplace, inplace_full, shrink, shrink_full,
+        each, each_full, delete_full``
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from pyspark.ml import Pipeline
+    >>> data = spark.createDataFrame([["Spark NLP is an open-source text processing library."]]).toDF("text")
+    >>> documentAssembler = DocumentAssembler().setInputCol("text").setOutputCol("document")
+    >>> result = documentAssembler.transform(data)
+    >>> result.select("document").show(truncate=False)
+    +----------------------------------------------------------------------------------------------+
+    |document                                                                                      |
+    +----------------------------------------------------------------------------------------------+
+    |[[document, 0, 51, Spark NLP is an open-source text processing library., [sentence -> 0], []]]|
+    +----------------------------------------------------------------------------------------------+
+    >>> result.select("document").printSchema()
+    root
+    |-- document: array (nullable = True)
+    |    |-- element: struct (containsNull = True)
+    |    |    |-- annotatorType: string (nullable = True)
+    |    |    |-- begin: integer (nullable = False)
+    |    |    |-- end: integer (nullable = False)
+    |    |    |-- result: string (nullable = True)
+    |    |    |-- metadata: map (nullable = True)
+    |    |    |    |-- key: string
+    |    |    |    |-- value: string (valueContainsNull = True)
+    |    |    |-- embeddings: array (nullable = True)
+    |    |    |    |-- element: float (containsNull = False)
+    """
+
+    outputAnnotatorType = AnnotatorType.DOCUMENT
+
+    inputCol = Param(Params._dummy(), "inputCol", "input column name", typeConverter=TypeConverters.toString)
+    outputCol = Param(Params._dummy(), "outputCol", "output column name", typeConverter=TypeConverters.toString)
+    idCol = Param(Params._dummy(), "idCol", "column for setting an id to such string in row", typeConverter=TypeConverters.toString)
+    metadataCol = Param(Params._dummy(), "metadataCol", "String to String map column to use as metadata", typeConverter=TypeConverters.toString)
+    cleanupMode = Param(Params._dummy(), "cleanupMode", "possible values: disabled, inplace, inplace_full, shrink, shrink_full, each, each_full, delete_full", typeConverter=TypeConverters.toString)
+    name = 'DocumentAssembler'
+
+    @keyword_only
+    def __init__(self):
+        super(DocumentAssembler, self).__init__(classname="com.johnsnowlabs.nlp.DocumentAssembler")
+        self._setDefault(outputCol="document", cleanupMode='disabled')
+
+    @keyword_only
+    def setParams(self):
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    def setInputCol(self, value):
+        """Sets input column name.
+
+        Parameters
+        ----------
+        value : str
+            Name of the input column
+        """
+        return self._set(inputCol=value)
+
+    def setOutputCol(self, value):
+        """Sets output column name.
+
+        Parameters
+        ----------
+        value : str
+            Name of the Output Column
+        """
+        return self._set(outputCol=value)
+
+    def setIdCol(self, value):
+        """Sets name of string type column for row id.
+
+        Parameters
+        ----------
+        value : str
+            Name of the Id Column
+        """
+        return self._set(idCol=value)
+
+    def setMetadataCol(self, value):
+        """Sets name for Map type column with metadata information.
+
+        Parameters
+        ----------
+        value : str
+            Name of the metadata column
+        """
+        return self._set(metadataCol=value)
+
+    def setCleanupMode(self, value):
+        """Sets how to cleanup the document, by default disabled.
+        Possible values: ``disabled, inplace, inplace_full, shrink, shrink_full,
+        each, each_full, delete_full``
+
+        Parameters
+        ----------
+        value : str
+            Cleanup mode
+        """
+        if value.strip().lower() not in ['disabled', 'inplace', 'inplace_full', 'shrink', 'shrink_full', 'each', 'each_full', 'delete_full']:
+            raise Exception("Cleanup mode possible values: disabled, inplace, inplace_full, shrink, shrink_full, each, each_full, delete_full")
+        return self._set(cleanupMode=value)
+
+    def getOutputCol(self):
+        """Gets output column name of annotations."""
+        return self.getOrDefault(self.outputCol)
+
+    # def getInputCol(self):
+    #     """Gets current column names of input annotations."""
+    #     return self.getOrDefault(self.inputCol)

sparknlp/base/embeddings_finisher.py

@@ -0,0 +1,201 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the EmbeddingsFinisher."""
+
+from pyspark import keyword_only
+from pyspark.ml.param import TypeConverters, Params, Param
+from sparknlp.internal import AnnotatorTransformer
+
+
+class EmbeddingsFinisher(AnnotatorTransformer):
+    """Extracts embeddings from Annotations into a more easily usable form.
+
+    This is useful for example:
+
+    - WordEmbeddings,
+    - Transformer based embeddings such as BertEmbeddings,
+    - SentenceEmbeddings and
+    - ChunkEmbeddings, etc.
+
+    By using ``EmbeddingsFinisher`` you can easily transform your embeddings
+    into array of floats or vectors which are compatible with Spark ML functions
+    such as LDA, K-mean, Random Forest classifier or any other functions that
+    require a ``featureCol``.
+
+    For more extended examples see the
+    `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/text-similarity/Spark_NLP_Spark_ML_Text_Similarity.ipynb
+>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``EMBEDDINGS``         ``NONE``
+    ====================== ======================
+
+    Parameters
+    ----------
+    inputCols
+        Names of input annotation columns containing embeddings
+    outputCols
+        Names of finished output columns
+    cleanAnnotations
+        Whether to remove all the existing annotation columns, by default False
+    outputAsVector
+        Whether to output the embeddings as Vectors instead of arrays,
+        by default False
+
+    Examples
+    --------
+    First extract embeddings.
+
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...    .setInputCol("text") \\
+    ...    .setOutputCol("document")
+    >>> tokenizer = Tokenizer() \\
+    ...    .setInputCols("document") \\
+    ...    .setOutputCol("token")
+    >>> normalizer = Normalizer() \\
+    ...    .setInputCols("token") \\
+    ...    .setOutputCol("normalized")
+    >>> stopwordsCleaner = StopWordsCleaner() \\
+    ...    .setInputCols("normalized") \\
+    ...    .setOutputCol("cleanTokens") \\
+    ...    .setCaseSensitive(False)
+    >>> gloveEmbeddings = WordEmbeddingsModel.pretrained() \\
+    ...    .setInputCols("document", "cleanTokens") \\
+    ...    .setOutputCol("embeddings") \\
+    ...    .setCaseSensitive(False)
+    >>> embeddingsFinisher = EmbeddingsFinisher() \\
+    ...    .setInputCols("embeddings") \\
+    ...    .setOutputCols("finished_sentence_embeddings") \\
+    ...    .setOutputAsVector(True) \\
+    ...    .setCleanAnnotations(False)
+    >>> data = spark.createDataFrame([["Spark NLP is an open-source text processing library."]]) \\
+    ...    .toDF("text")
+    >>> pipeline = Pipeline().setStages([
+    ...    documentAssembler,
+    ...    tokenizer,
+    ...    normalizer,
+    ...    stopwordsCleaner,
+    ...    gloveEmbeddings,
+    ...    embeddingsFinisher
+    ... ]).fit(data)
+    >>> result = pipeline.transform(data)
+
+    Show results.
+
+    >>> resultWithSize = result.selectExpr("explode(finished_sentence_embeddings) as embeddings")
+    >>> resultWithSize.show(5, 80)
+    +--------------------------------------------------------------------------------+
+    |                                                                      embeddings|
+    +--------------------------------------------------------------------------------+
+    |[0.1619900017976761,0.045552998781204224,-0.03229299932718277,-0.685609996318...|
+    |[-0.42416998744010925,1.1378999948501587,-0.5717899799346924,-0.5078899860382...|
+    |[0.08621499687433243,-0.15772999823093414,-0.06067200005054474,0.395359992980...|
+    |[-0.4970499873161316,0.7164199948310852,0.40119001269340515,-0.05761000141501...|
+    |[-0.08170200139284134,0.7159299850463867,-0.20677000284194946,0.0295659992843...|
+    +--------------------------------------------------------------------------------+
+
+    See Also
+    --------
+    EmbeddingsFinisher : for finishing embeddings
+    """
+
+    inputCols = Param(Params._dummy(), "inputCols", "name of input annotation cols containing embeddings", typeConverter=TypeConverters.toListString)
+    outputCols = Param(Params._dummy(), "outputCols", "output EmbeddingsFinisher ouput cols", typeConverter=TypeConverters.toListString)
+    cleanAnnotations = Param(Params._dummy(), "cleanAnnotations", "whether to remove all the existing annotation columns", typeConverter=TypeConverters.toBoolean)
+    outputAsVector = Param(Params._dummy(), "outputAsVector", "if enabled it will output the embeddings as Vectors instead of arrays", typeConverter=TypeConverters.toBoolean)
+
+    name = "EmbeddingsFinisher"
+
+    @keyword_only
+    def __init__(self):
+        super(EmbeddingsFinisher, self).__init__(classname="com.johnsnowlabs.nlp.EmbeddingsFinisher")
+        self._setDefault(
+            cleanAnnotations=False,
+            outputAsVector=False,
+            outputCols=[]
+        )
+
+    @keyword_only
+    def setParams(self):
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    def setInputCols(self, *value):
+        """Sets name of input annotation columns containing embeddings.
+
+        Parameters
+        ----------
+        *value : str
+            Input columns for the annotator
+        """
+
+        if len(value) == 1 and type(value[0]) == list:
+            return self._set(inputCols=value[0])
+        else:
+            return self._set(inputCols=list(value))
+
+    def setOutputCols(self, *value):
+        """Sets names of finished output columns.
+
+        Parameters
+        ----------
+        *value : List[str]
+            Input columns for the annotator
+        """
+
+        if len(value) == 1 and type(value[0]) == list:
+            return self._set(outputCols=value[0])
+        else:
+            return self._set(outputCols=list(value))
+
+    def setCleanAnnotations(self, value):
+        """Sets whether to remove all the existing annotation columns, by default
+        False.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to remove all the existing annotation columns
+        """
+
+        return self._set(cleanAnnotations=value)
+
+    def setOutputAsVector(self, value):
+        """Sets whether to output the embeddings as Vectors instead of arrays,
+        by default False.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to output the embeddings as Vectors instead of arrays
+        """
+
+        return self._set(outputAsVector=value)
+
+    def getInputCols(self):
+        """Gets input columns name of annotations."""
+        return self.getOrDefault(self.inputCols)
+
+    def getOutputCols(self):
+        """Gets output columns name of annotations."""
+        if len(self.getOrDefault(self.outputCols)) == 0:
+            return ["finished_" + input_col for input_col in self.getInputCols()]
+        else:
+            return self.getOrDefault(self.outputCols)