spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/base/finisher.py

@@ -0,0 +1,217 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the Finisher."""
+
+from pyspark import keyword_only
+from pyspark.ml.param import TypeConverters, Params, Param
+from sparknlp.internal import AnnotatorTransformer
+
+
+class Finisher(AnnotatorTransformer):
+    """Converts annotation results into a format that easier to use.
+
+    It is useful to extract the results from Spark NLP Pipelines. The Finisher
+    outputs annotation(s) values into ``String``.
+
+    For more extended examples on document pre-processing see the
+    `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb
+>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``ANY``                ``NONE``
+    ====================== ======================
+
+    Parameters
+    ----------
+    inputCols
+        Input annotations
+    outputCols
+        Output finished annotation cols
+    valueSplitSymbol
+        Character separating values, by default #
+    annotationSplitSymbol
+        Character separating annotations, by default @
+    cleanAnnotations
+        Whether to remove annotation columns, by default True
+    includeMetadata
+        Whether to include annotation metadata, by default False
+    outputAsArray
+        Finisher generates an Array with the results instead of string, by
+        default True
+    parseEmbeddingsVectors
+        Whether to include embeddings vectors in the process, by default False
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from sparknlp.pretrained import PretrainedPipeline
+    >>> data = spark.createDataFrame([[1, "New York and New Jersey aren't that far apart actually."]]).toDF("id", "text")
+
+    Define pretrained pipeline that extracts Named Entities amongst other things
+    and apply the `Finisher` on it.
+
+    >>> pipeline = PretrainedPipeline("explain_document_dl")
+    >>> finisher = Finisher().setInputCols("entities").setOutputCols("output")
+    >>> explainResult = pipeline.transform(data)
+
+    Show results.
+
+    >>> explainResult.selectExpr("explode(entities)").show(truncate=False)
+    +------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |entities                                                                                                                                              |
+    +------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |[[chunk, 0, 7, New York, [entity -> LOC, sentence -> 0, chunk -> 0], []], [chunk, 13, 22, New Jersey, [entity -> LOC, sentence -> 0, chunk -> 1], []]]|
+    +------------------------------------------------------------------------------------------------------------------------------------------------------+
+    >>> result = finisher.transform(explainResult)
+    >>> result.select("output").show(truncate=False)
+    +----------------------+
+    |output                |
+    +----------------------+
+    |[New York, New Jersey]|
+    +----------------------+
+
+    See Also
+    --------
+    Finisher : for finishing Strings
+    """
+
+    inputCols = Param(Params._dummy(), "inputCols", "input annotations", typeConverter=TypeConverters.toListString)
+    outputCols = Param(Params._dummy(), "outputCols", "output finished annotation cols", typeConverter=TypeConverters.toListString)
+    valueSplitSymbol = Param(Params._dummy(), "valueSplitSymbol", "character separating annotations", typeConverter=TypeConverters.toString)
+    annotationSplitSymbol = Param(Params._dummy(), "annotationSplitSymbol", "character separating annotations", typeConverter=TypeConverters.toString)
+    cleanAnnotations = Param(Params._dummy(), "cleanAnnotations", "whether to remove annotation columns", typeConverter=TypeConverters.toBoolean)
+    includeMetadata = Param(Params._dummy(), "includeMetadata", "annotation metadata format", typeConverter=TypeConverters.toBoolean)
+    outputAsArray = Param(Params._dummy(), "outputAsArray", "finisher generates an Array with the results instead of string", typeConverter=TypeConverters.toBoolean)
+    parseEmbeddingsVectors = Param(Params._dummy(), "parseEmbeddingsVectors", "whether to include embeddings vectors in the process", typeConverter=TypeConverters.toBoolean)
+    name = "Finisher"
+
+    @keyword_only
+    def __init__(self):
+        super(Finisher, self).__init__(classname="com.johnsnowlabs.nlp.Finisher")
+        self._setDefault(
+            cleanAnnotations=True,
+            includeMetadata=False,
+            outputAsArray=True,
+            parseEmbeddingsVectors=False,
+            valueSplitSymbol="#",
+            annotationSplitSymbol="@",
+            outputCols=[]
+        )
+
+    @keyword_only
+    def setParams(self):
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    def setInputCols(self, *value):
+        """Sets column names of input annotations.
+
+        Parameters
+        ----------
+        *value : List[str]
+            Input columns for the annotator
+        """
+        if len(value) == 1 and type(value[0]) == list:
+            return self._set(inputCols=value[0])
+        else:
+            return self._set(inputCols=list(value))
+
+    def setOutputCols(self, *value):
+        """Sets column names of finished output annotations.
+
+        Parameters
+        ----------
+        *value : List[str]
+            List of output columns
+        """
+        if len(value) == 1 and type(value[0]) == list:
+            return self._set(outputCols=value[0])
+        else:
+            return self._set(outputCols=list(value))
+
+    def setValueSplitSymbol(self, value):
+        """Sets character separating values, by default #.
+
+        Parameters
+        ----------
+        value : str
+            Character to separate annotations
+        """
+        return self._set(valueSplitSymbol=value)
+
+    def setAnnotationSplitSymbol(self, value):
+        """Sets character separating annotations, by default @.
+
+        Parameters
+        ----------
+        value : str
+            ...
+        """
+        return self._set(annotationSplitSymbol=value)
+
+    def setCleanAnnotations(self, value):
+        """Sets whether to remove annotation columns, by default True.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to remove annotation columns
+        """
+        return self._set(cleanAnnotations=value)
+
+    def setIncludeMetadata(self, value):
+        """Sets whether to include annotation metadata.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to include annotation metadata
+        """
+        return self._set(includeMetadata=value)
+
+    def setOutputAsArray(self, value):
+        """Sets whether to generate an array with the results instead of a
+        string.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to generate an array with the results instead of a string
+        """
+        return self._set(outputAsArray=value)
+
+    def setParseEmbeddingsVectors(self, value):
+        """Sets whether to include embeddings vectors in the process.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to include embeddings vectors in the process
+        """
+        return self._set(parseEmbeddingsVectors=value)
+
+    def getInputCols(self):
+        """Gets input columns name of annotations."""
+        return self.getOrDefault(self.inputCols)
+
+    def getOutputCols(self):
+        """Gets output columns name of annotations."""
+        if len(self.getOrDefault(self.outputCols)) == 0:
+            return ["finished_" + input_col for input_col in self.getInputCols()]
+        else:
+            return self.getOrDefault(self.outputCols)

sparknlp/base/gguf_ranking_finisher.py

@@ -0,0 +1,234 @@
+#  Copyright 2017-2024 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the GGUFRankingFinisher."""
+
+from pyspark import keyword_only
+from pyspark.ml.param import TypeConverters, Params, Param
+from sparknlp.internal import AnnotatorTransformer
+
+
+class GGUFRankingFinisher(AnnotatorTransformer):
+    """Finisher for AutoGGUFReranker outputs that provides ranking capabilities
+    including top-k selection, sorting by relevance score, and score normalization.
+
+    This finisher processes the output of AutoGGUFReranker, which contains documents with
+    relevance scores in their metadata. It provides several options for post-processing:
+
+    - Top-k selection: Select only the top k documents by relevance score
+    - Score thresholding: Filter documents by minimum relevance score
+    - Min-max scaling: Normalize relevance scores to 0-1 range
+    - Sorting: Sort documents by relevance score in descending order
+    - Ranking: Add rank information to document metadata
+
+    The finisher preserves the document annotation structure while adding ranking information
+    to the metadata and optionally filtering/sorting the documents.
+
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/finisher/gguf_ranking_finisher_example.py>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``DOCUMENT``
+    ====================== ======================
+
+    Parameters
+    ----------
+    inputCols
+        Name of input annotation columns containing reranked documents
+    outputCol
+        Name of output annotation column containing ranked documents, by default "ranked_documents"
+    topK
+        Maximum number of top documents to return based on relevance score (-1 for no limit), by default -1
+    minRelevanceScore
+        Minimum relevance score threshold for filtering documents, by default Double.MinValue
+    minMaxScaling
+        Whether to apply min-max scaling to normalize relevance scores to 0-1 range, by default False
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> reranker = AutoGGUFReranker.pretrained() \\
+    ...     .setInputCols("document") \\
+    ...     .setOutputCol("reranked_documents") \\
+    ...     .setQuery("A man is eating pasta.")
+    >>> finisher = GGUFRankingFinisher() \\
+    ...     .setInputCols("reranked_documents") \\
+    ...     .setOutputCol("ranked_documents") \\
+    ...     .setTopK(3) \\
+    ...     .setMinMaxScaling(True)
+    >>> pipeline = Pipeline().setStages([documentAssembler, reranker, finisher])
+    >>> data = spark.createDataFrame([
+    ...     ("A man is eating food.",),
+    ...     ("A man is eating a piece of bread.",),
+    ...     ("The girl is carrying a baby.",),
+    ...     ("A man is riding a horse.",)
+    ... ], ["text"])
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.select("ranked_documents").show(truncate=False)
+    # Documents will be sorted by relevance with rank information in metadata
+    """
+
+    name = "GGUFRankingFinisher"
+
+    inputCols = Param(Params._dummy(),
+                     "inputCols",
+                     "Name of input annotation columns containing reranked documents",
+                     typeConverter=TypeConverters.toListString)
+
+    outputCol = Param(Params._dummy(),
+                     "outputCol", 
+                     "Name of output annotation column containing ranked documents",
+                     typeConverter=TypeConverters.toListString)
+
+    topK = Param(Params._dummy(),
+                 "topK",
+                 "Maximum number of top documents to return based on relevance score (-1 for no limit)",
+                 typeConverter=TypeConverters.toInt)
+
+    minRelevanceScore = Param(Params._dummy(),
+                             "minRelevanceScore",
+                             "Minimum relevance score threshold for filtering documents",
+                             typeConverter=TypeConverters.toFloat)
+
+    minMaxScaling = Param(Params._dummy(),
+                         "minMaxScaling",
+                         "Whether to apply min-max scaling to normalize relevance scores to 0-1 range",
+                         typeConverter=TypeConverters.toBoolean)
+
+    @keyword_only
+    def __init__(self):
+        super(GGUFRankingFinisher, self).__init__(
+            classname="com.johnsnowlabs.nlp.finisher.GGUFRankingFinisher")
+        self._setDefault(
+            topK=-1,
+            minRelevanceScore=float('-inf'),  # Equivalent to Double.MinValue
+            minMaxScaling=False,
+            outputCol=["ranked_documents"]
+        )
+
+    @keyword_only
+    def setParams(self):
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    def setInputCols(self, *value):
+        """Sets input annotation column names.
+
+        Parameters
+        ----------
+        value : List[str]
+            Input annotation column names containing reranked documents
+        """
+        if len(value) == 1 and isinstance(value[0], list):
+            return self._set(inputCols=value[0])
+        else:
+            return self._set(inputCols=list(value))
+
+    def getInputCols(self):
+        """Gets input annotation column names.
+
+        Returns
+        -------
+        List[str]
+            Input annotation column names
+        """
+        return self.getOrDefault(self.inputCols)
+
+    def setOutputCol(self, value):
+        """Sets output annotation column name.
+
+        Parameters
+        ----------
+        value : str
+            Output annotation column name
+        """
+        return self._set(outputCol=[value])
+
+    def getOutputCol(self):
+        """Gets output annotation column name.
+
+        Returns
+        -------
+        str
+            Output annotation column name
+        """
+        output_cols = self.getOrDefault(self.outputCol)
+        return output_cols[0] if output_cols else "ranked_documents"
+
+    def setTopK(self, value):
+        """Sets maximum number of top documents to return.
+
+        Parameters
+        ----------
+        value : int
+            Maximum number of top documents to return (-1 for no limit)
+        """
+        return self._set(topK=value)
+
+    def getTopK(self):
+        """Gets maximum number of top documents to return.
+
+        Returns
+        -------
+        int
+            Maximum number of top documents to return
+        """
+        return self.getOrDefault(self.topK)
+
+    def setMinRelevanceScore(self, value):
+        """Sets minimum relevance score threshold.
+
+        Parameters
+        ----------
+        value : float
+            Minimum relevance score threshold
+        """
+        return self._set(minRelevanceScore=value)
+
+    def getMinRelevanceScore(self):
+        """Gets minimum relevance score threshold.
+
+        Returns
+        -------
+        float
+            Minimum relevance score threshold
+        """
+        return self.getOrDefault(self.minRelevanceScore)
+
+    def setMinMaxScaling(self, value):
+        """Sets whether to apply min-max scaling.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to apply min-max scaling to normalize scores
+        """
+        return self._set(minMaxScaling=value)
+
+    def getMinMaxScaling(self):
+        """Gets whether to apply min-max scaling.
+
+        Returns
+        -------
+        bool
+            Whether min-max scaling is enabled
+        """
+        return self.getOrDefault(self.minMaxScaling)

sparknlp/base/graph_finisher.py

@@ -0,0 +1,125 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the GraphFinisher."""
+
+from pyspark import keyword_only
+from pyspark.ml.param import TypeConverters, Params, Param
+from sparknlp.internal import AnnotatorTransformer
+
+
+class GraphFinisher(AnnotatorTransformer):
+    """Helper class to convert the knowledge graph from GraphExtraction into a
+    generic format, such as RDF.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``NONE``               ``NONE``
+    ====================== ======================
+
+    Parameters
+    ----------
+
+    inputCol
+        Name of input annotation column
+    outputCol
+        Name of finisher output column
+    cleanAnnotations
+        Whether to remove all the existing annotation columns, by default True
+    outputAsArray
+        Whether to generate an Array with the results, by default True
+
+    Examples
+    --------
+    This is a continuation of the example of
+    :class:`.GraphExtraction`. To see how the graph is extracted, see the
+    documentation of that class.
+
+    >>> graphFinisher = GraphFinisher() \\
+    ...     .setInputCol("graph") \\
+    ...     .setOutputCol("graph_finished")
+    ...     .setOutputAsArray(False)
+    >>> finishedResult = graphFinisher.transform(result)
+    >>> finishedResult.select("text", "graph_finished").show(truncate=False)
+    +-----------------------------------------------------+-----------------------------------------------------------------------+
+    |text                                                 |graph_finished                                                         |
+    +-----------------------------------------------------+-----------------------------------------------------------------------+
+    |You and John prefer the morning flight through Denver|[[(prefer,nsubj,morning), (morning,flat,flight), (flight,flat,Denver)]]|
+    +-----------------------------------------------------+-----------------------------------------------------------------------+
+    """
+    inputCol = Param(Params._dummy(), "inputCol", "Name of input annotation col", typeConverter=TypeConverters.toString)
+    outputCol = Param(Params._dummy(), "outputCol", "Name of finisher output col", typeConverter=TypeConverters.toString)
+    cleanAnnotations = Param(Params._dummy(),
+                             "cleanAnnotations",
+                             "Whether to remove all the existing annotation columns",
+                             typeConverter=TypeConverters.toBoolean)
+    outputAsArray = Param(Params._dummy(), "outputAsArray", "Finisher generates an Array with the results",
+                          typeConverter=TypeConverters.toBoolean)
+
+    name = "GraphFinisher"
+
+    @keyword_only
+    def __init__(self):
+        super(GraphFinisher, self).__init__(classname="com.johnsnowlabs.nlp.GraphFinisher")
+        self._setDefault(
+            cleanAnnotations=True,
+            outputAsArray=True
+        )
+
+    @keyword_only
+    def setParams(self):
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    def setInputCol(self, value):
+        """Sets name of input annotation column.
+
+        Parameters
+        ----------
+        value : str
+            Name of input annotation column.
+        """
+        return self._set(inputCol=value)
+
+    def setOutputCol(self, value):
+        """Sets name of finisher output column.
+
+        Parameters
+        ----------
+        value : str
+            Name of finisher output column.
+        """
+        return self._set(outputCol=value)
+
+    def setCleanAnnotations(self, value):
+        """Sets whether to remove all the existing annotation columns, by
+        default True.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to remove all the existing annotation columns, by default True.
+        """
+        return self._set(cleanAnnotations=value)
+
+    def setOutputAsArray(self, value):
+        """Sets whether to generate an Array with the results, by default True.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to generate an Array with the results, by default True.
+        """
+        return self._set(outputAsArray=value)
+

sparknlp/base/has_recursive_fit.py

@@ -0,0 +1,24 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains HasRecursiveFit property for estimators."""
+
+from abc import ABC
+
+from sparknlp.internal import RecursiveEstimator
+
+
+class HasRecursiveFit(RecursiveEstimator, ABC):
+    """Properties for the implementation of the RecursivePipeline."""
+    pass
+

sparknlp/base/has_recursive_transform.py

@@ -0,0 +1,22 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains HasRecursiveFit property for Spark transformers."""
+
+from sparknlp.internal import RecursiveTransformer
+
+
+class HasRecursiveTransform(RecursiveTransformer):
+    """Properties for the implementation of the RecursivePipeline."""
+    pass
+