spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/common/read_as.py

@@ -0,0 +1,33 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains utility classes for reading resources."""
+
+
+class ReadAs(object):
+    """Object that contains constants for how to read Spark Resources.
+
+    Possible values are:
+
+    ================= =======================================
+    Value             Description
+    ================= =======================================
+    ``ReadAs.TEXT``   Read the resource as text.
+    ``ReadAs.SPARK``  Read the resource as a Spark DataFrame.
+    ``ReadAs.BINARY`` Read the resource as a binary file.
+    ================= =======================================
+    """
+    TEXT = "TEXT"
+    SPARK = "SPARK"
+    BINARY = "BINARY"
+

sparknlp/common/recursive_annotator_approach.py

@@ -0,0 +1,35 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains base classes for recursive AnnotatorApproaches."""
+
+from pyspark import keyword_only
+from pyspark.ml.util import JavaMLWritable
+
+import sparknlp.internal as _internal
+from sparknlp.common import AnnotatorProperties
+
+
+class RecursiveAnnotatorApproach(_internal.RecursiveEstimator, JavaMLWritable, _internal.AnnotatorJavaMLReadable,
+                                 AnnotatorProperties,
+                                 _internal.ParamsGettersSetters):
+    @keyword_only
+    def __init__(self, classname):
+        _internal.ParamsGettersSetters.__init__(self)
+        self.__class__._java_class_name = classname
+        self._java_obj = self._new_java_obj(classname, self.uid)
+        self._setDefault(lazyAnnotator=False)
+
+    def _create_model(self, java_model):
+        raise NotImplementedError('Please implement _create_model in %s' % self)
+

sparknlp/common/storage.py

@@ -0,0 +1,149 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains utility classes for handling storage."""
+
+from pyspark.ml.param import Param, Params, TypeConverters
+
+from sparknlp.common.utils import ExternalResource
+from sparknlp.common.properties import HasCaseSensitiveProperties
+import sparknlp.internal as _internal
+
+
+class HasStorageRef:
+    storageRef = Param(Params._dummy(), "storageRef",
+                       "unique reference name for identification",
+                       TypeConverters.toString)
+
+    def setStorageRef(self, value):
+        """Sets unique reference name for identification.
+
+        Parameters
+        ----------
+        value : str
+            Unique reference name for identification
+        """
+        return self._set(storageRef=value)
+
+    def getStorageRef(self):
+        """Gets unique reference name for identification.
+
+        Returns
+        -------
+        str
+            Unique reference name for identification
+        """
+        return self.getOrDefault("storageRef")
+
+
+class HasStorageOptions:
+    includeStorage = Param(Params._dummy(),
+                           "includeStorage",
+                           "whether to include indexed storage in trained model",
+                           typeConverter=TypeConverters.toBoolean)
+
+    enableInMemoryStorage = Param(Params._dummy(),
+                                  "enableInMemoryStorage",
+                                  "whether to load whole indexed storage in memory (in-memory lookup)",
+                                  typeConverter=TypeConverters.toBoolean)
+
+    def setIncludeStorage(self, value):
+        """Sets whether to include indexed storage in trained model.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to include indexed storage in trained model
+        """
+        return self._set(includeStorage=value)
+
+    def getIncludeStorage(self):
+        """Gets whether to include indexed storage in trained model.
+
+        Returns
+        -------
+        bool
+            Whether to include indexed storage in trained model
+        """
+        return self.getOrDefault("includeStorage")
+
+    def setEnableInMemoryStorage(self, value):
+        """Sets whether to load whole indexed storage in memory (in-memory lookup)
+
+        Parameters
+        ----------
+        value : bool
+            Whether to load whole indexed storage in memory (in-memory lookup)
+        """
+        return self._set(enableInMemoryStorage=value)
+
+    def getEnableInMemoryStorage(self):
+        return self.getOrDefault("enableInMemoryStorage")
+
+
+class HasStorageModel(HasStorageRef, HasCaseSensitiveProperties, HasStorageOptions):
+
+    def saveStorage(self, path, spark):
+        """Saves the current model to storage.
+
+        Parameters
+        ----------
+        path : str
+            Path for saving the model.
+        spark : :class:`pyspark.sql.SparkSession`
+            The current SparkSession
+        """
+        self._transfer_params_to_java()
+        self._java_obj.saveStorage(path, spark._jsparkSession, False)
+
+    @staticmethod
+    def loadStorage(path, spark, storage_ref):
+        raise NotImplementedError("AnnotatorModel with HasStorageModel did not implement 'loadStorage'")
+
+    @staticmethod
+    def loadStorages(path, spark, storage_ref, databases):
+        for database in databases:
+            _internal._StorageHelper(path, spark, database, storage_ref, within_storage=False)
+
+
+class HasStorage(HasStorageRef, HasCaseSensitiveProperties, HasStorageOptions):
+    storagePath = Param(Params._dummy(),
+                        "storagePath",
+                        "path to file",
+                        typeConverter=TypeConverters.identity)
+
+    def setStoragePath(self, path, read_as):
+        """Sets path to file.
+
+        Parameters
+        ----------
+        path : str
+            Path to file
+        read_as : str
+            How to interpret the file
+
+        Notes
+        -----
+        See :class:`ReadAs <sparknlp.common.ReadAs>` for reading options.
+        """
+        return self._set(storagePath=ExternalResource(path, read_as, {}))
+
+    def getStoragePath(self):
+        """Gets path to file.
+
+        Returns
+        -------
+        str
+            path to file
+        """
+        return self.getOrDefault("storagePath")

sparknlp/common/utils.py

@@ -0,0 +1,39 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains utilities for annotators."""
+
+from sparknlp.common.read_as import ReadAs
+import sparknlp.internal as _internal
+
+
+def ExternalResource(path, read_as=ReadAs.TEXT, options={}):
+    """Returns a representation fo an External Resource.
+
+    How the resource is read can be set with `read_as`.
+
+    Parameters
+    ----------
+    path : str
+        Path to the resource
+    read_as : str, optional
+        How to read the resource, by default ReadAs.TEXT
+    options : dict, optional
+        Options to read the resource, by default {}
+    """
+    return _internal._ExternalResource(path, read_as, options).apply()
+
+
+def RegexRule(rule, identifier):
+    return _internal._RegexRule(rule, identifier).apply()
+

sparknlp/functions.py

@@ -1,28 +1,295 @@
-from pyspark.sql.functions import udf
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Contains helper functions to assist in transforming Annotation results.
+"""
+
+from pyspark.sql.functions import udf, array
 from pyspark.sql.types import *
 from pyspark.sql import DataFrame
 from sparknlp.annotation import Annotation
 
 
 def map_annotations(f, output_type: DataType):
+    """Creates a Spark UDF to map over an Annotator's results.
+
+    Parameters
+    ----------
+    f : function
+        The function to be applied over the results
+    output_type : :class:`pyspark.sql.types.DataType`
+        Output type of the data
+
+    Returns
+    -------
+    :func:`pyspark.sql.functions.udf`
+        Spark UserDefinedFunction (udf)
+
+    Examples
+    --------
+    >>> from sparknlp.pretrained import PretrainedPipeline
+    >>> explain_document_pipeline = PretrainedPipeline("explain_document_dl")
+    >>> data = spark.createDataFrame([["U.N. official Ekeus heads for Baghdad."]]).toDF("text")
+    >>> result = explain_document_pipeline.transform(data)
+
+    The array type must be provided in order to tell Spark the expected output
+    type of our column. We are using an Annotation array here.
+
+    >>> from sparknlp.functions import *
+    >>> def nnp_tokens(annotations: List[Row]):
+    ...     return list(
+    ...         filter(lambda annotation: annotation.result == 'NNP', annotations)
+    ...     )
+    >>> result.select(
+    ...     map_annotations(nnp_tokens, Annotation.arrayType())('pos').alias("nnp")
+    ... ).selectExpr("explode(nnp) as nnp").show(truncate=False)
+    +-----------------------------------------+
+    |nnp                                      |
+    +-----------------------------------------+
+    |[pos, 0, 2, NNP, [word -> U.N], []]      |
+    |[pos, 14, 18, NNP, [word -> Epeus], []]  |
+    |[pos, 30, 36, NNP, [word -> Baghdad], []]|
+    +-----------------------------------------+
+    """
     return udf(
-        lambda content: f(content),
+        lambda content: [ Annotation.toRow(a) for a in f([Annotation.fromRow(r) for r in content])],
         output_type
     )
 
+def map_annotations_array(f, output_type: DataType):
+    """Creates a Spark UDF to map over an Annotator's array results.
+
+    Parameters
+    ----------
+    f : function
+        The function to be applied over the results
+    output_type : :class:`pyspark.sql.types.DataType`
+        Output type of the data
+
+    Returns
+    -------
+    :func:`pyspark.sql.functions.udf`
+        Spark UserDefinedFunction (udf)
+    """
+    return udf(
+        lambda cols: [Annotation.toRow(item) for item in f([Annotation.fromRow(r) for col in cols for r in col])],
+        output_type
+    )
 
 def map_annotations_strict(f):
+    """Creates a Spark UDF to map over an Annotator's results, for which the
+    return type is explicitly defined as a `Annotation.dataType()`.
+
+    Parameters
+    ----------
+    f : function
+        The function to be applied over the results
+
+    Returns
+    -------
+    :func:`pyspark.sql.functions.udf`
+        Spark UserDefinedFunction (udf)
+
+    Examples
+    --------
+    >>> from sparknlp.pretrained import PretrainedPipeline
+    >>> explain_document_pipeline = PretrainedPipeline("explain_document_dl")
+    >>> data = spark.createDataFrame([["U.N. official Ekeus heads for Baghdad."]]).toDF("text")
+    >>> result = explain_document_pipeline.transform(data)
+    >>> def nnp_tokens(annotations):
+    ...     return list(
+    ...         filter(lambda annotation: annotation.result == 'NNP', annotations)
+    ...     )
+    >>> result.select(
+    ...     map_annotations_strict(nnp_tokens)('pos').alias("nnp")
+    ... ).selectExpr("explode(nnp) as nnp").show(truncate=False)
+    +-----------------------------------------+
+    |nnp                                      |
+    +-----------------------------------------+
+    |[pos, 0, 2, NNP, [word -> U.N], []]      |
+    |[pos, 14, 18, NNP, [word -> Epeus], []]  |
+    |[pos, 30, 36, NNP, [word -> Baghdad], []]|
+    +-----------------------------------------+
+    """
     return udf(
-        lambda content: f(content),
+        lambda content: [ Annotation.toRow(a) for a in f([Annotation.fromRow(r) for r in content])],
         ArrayType(Annotation.dataType())
     )
 
 
-def map_annotations_col(dataframe: DataFrame, f, column, output_column, output_type):
-    return dataframe.withColumn(output_column, map_annotations(f, output_type)(column))
+def map_annotations_col(dataframe: DataFrame, f, column: str, output_column: str, annotatyon_type: str,
+                        output_type: DataType = Annotation.arrayType()):
+    """Creates a Spark UDF to map over a column of Annotation results.
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Input DataFrame
+    f : function
+        Function to apply to the column
+    column : str
+        Name of the input column
+    output_column : str
+        Name of the output column
+    annotatyon_type : str
+        Annotator type
+    output_type : DataType, optional
+        Output type, by default Annotation.arrayType()
+
+    Returns
+    -------
+    :class:`pyspark.sql.DataFrame`
+        Transformed DataFrame
+
+    Examples
+    --------
+    >>> from sparknlp.pretrained import PretrainedPipeline
+    >>> from sparknlp.functions import *
+    >>> explain_document_pipeline = PretrainedPipeline("explain_document_dl")
+    >>> data = spark.createDataFrame([["U.N. official Ekeus heads for Baghdad."]]).toDF("text")
+    >>> result = explain_document_pipeline.transform(data)
+    >>> chunks_df = map_annotations_col(
+    ...     result,
+    ...     lambda x: [
+    ...         Annotation("chunk", a.begin, a.end, a.result, a.metadata, a.embeddings)
+    ...         for a in x
+    ...     ],
+    ...     "pos",
+    ...     "pos_chunk",
+    ...     "chunk",
+    ... )
+    >>> chunks_df.selectExpr("explode(pos_chunk)").show()
+    +--------------------+
+    |                 col|
+    +--------------------+
+    |[chunk, 0, 2, NNP...|
+    |[chunk, 3, 3, ., ...|
+    |[chunk, 5, 12, JJ...|
+    |[chunk, 14, 18, N...|
+    |[chunk, 20, 24, V...|
+    |[chunk, 26, 28, I...|
+    |[chunk, 30, 36, N...|
+    |[chunk, 37, 37, ....|
+    +--------------------+
+    """
+    return dataframe.withColumn(output_column, map_annotations(f, output_type)(column).alias(output_column, metadata={
+        'annotatorType': annotatyon_type}))
+
+def map_annotations_cols(dataframe: DataFrame, f, columns: list, output_column: str, annotatyon_type: str,
+                        output_type: DataType = Annotation.arrayType()):
+    """Creates a Spark UDF to map over multiple columns of Annotation results.
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Input DataFrame
+    f : function
+        Function to apply to the column
+    columns : list
+        Name of the input column
+    output_column : str
+        Name of the output column
+    annotatyon_type : str
+        Annotator type
+    output_type : DataType, optional
+        Output type, by default Annotation.arrayType()
+
+    Returns
+    -------
+    :class:`pyspark.sql.DataFrame`
+        Transformed DataFrame
+
+    Examples
+    --------
+    >>> from sparknlp.pretrained import PretrainedPipeline
+    >>> from sparknlp.functions import *
+    >>> explain_document_pipeline = PretrainedPipeline("explain_document_dl")
+    >>> data = spark.createDataFrame([["U.N. official Ekeus heads for Baghdad."]]).toDF("text")
+    >>> result = explain_document_pipeline.transform(data)
+    >>> chunks_df = map_annotations_cols(
+    ...     result,
+    ...     lambda x: [
+    ...         Annotation("tag", a.begin, a.end, a.result, a.metadata, a.embeddings)
+    ...         for a in x
+    ...     ],
+    ...     ["pos", "ner"],
+    ...     "tags",
+    ...     "chunk"
+    ... )
+    >>> chunks_df.selectExpr("explode(tags)").show(truncate=False)
+    +-------------------------------------------+
+    |col                                        |
+    +-------------------------------------------+
+    |[tag, 0, 2, NNP, [word -> U.N], []]        |
+    |[tag, 3, 3, ., [word -> .], []]            |
+    |[tag, 5, 12, JJ, [word -> official], []]   |
+    |[tag, 14, 18, NNP, [word -> Epeus], []]    |
+    |[tag, 20, 24, VBZ, [word -> heads], []]    |
+    |[tag, 26, 28, IN, [word -> for], []]       |
+    |[tag, 30, 36, NNP, [word -> Baghdad], []]  |
+    |[tag, 37, 37, ., [word -> .], []]          |
+    |[tag, 0, 2, B-ORG, [word -> U.N], []]      |
+    |[tag, 3, 3, O, [word -> .], []]            |
+    |[tag, 5, 12, O, [word -> official], []]    |
+    |[tag, 14, 18, B-PER, [word -> Ekeus], []]  |
+    |[tag, 20, 24, O, [word -> heads], []]      |
+    |[tag, 26, 28, O, [word -> for], []]        |
+    |[tag, 30, 36, B-LOC, [word -> Baghdad], []]|
+    |[tag, 37, 37, O, [word -> .], []]          |
+    +-------------------------------------------+
+    """
+    return dataframe.withColumn(output_column, map_annotations_array(f, output_type)(array(*columns)).alias(output_column, metadata={
+        'annotatorType': annotatyon_type}))
 
 
 def filter_by_annotations_col(dataframe, f, column):
+    """Applies a filter over a column of Annotations.
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Input DataFrame
+    f : function
+        Filter function
+    column : str
+        Name of the column
+
+    Returns
+    -------
+    :class:`pyspark.sql.DataFrame`
+        Filtered DataFrame
+
+    Examples
+    --------
+    >>> from sparknlp.pretrained import PretrainedPipeline
+    >>> from sparknlp.functions import *
+    >>> explain_document_pipeline = PretrainedPipeline("explain_document_dl")
+    >>> data = spark.createDataFrame([["U.N. official Ekeus heads for Baghdad."]]).toDF("text")
+    >>> result = explain_document_pipeline.transform(data)
+    >>> def filter_pos(annotation: Annotation):
+    ...     return annotation.result == "NNP"
+    >>> filter_by_annotations_col(
+    ...     explode_annotations_col(result, "pos", "pos"), filter_pos, "pos"
+    ... ).select("pos").show(truncate=False)
+    +-----------------------------------------+
+    |pos                                      |
+    +-----------------------------------------+
+    |[pos, 0, 2, NNP, [word -> U.N], []]      |
+    |[pos, 14, 18, NNP, [word -> Epeus], []]  |
+    |[pos, 30, 36, NNP, [word -> Baghdad], []]|
+    +-----------------------------------------+
+    """
     this_udf = udf(
         lambda content: f(content),
         BooleanType()
@@ -31,5 +298,48 @@ def filter_by_annotations_col(dataframe, f, column):
 
 
 def explode_annotations_col(dataframe: DataFrame, column, output_column):
+    """Explodes an Annotation column, putting each result onto a separate row.
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        The Spark DataFrame containing output Annotations
+    column : str
+        Name of the column
+    output_column : str
+        Name of the output column
+
+    Returns
+    -------
+    :class:`pyspark.sql.DataFrame`
+        Transformed DataFrame
+
+    Examples
+    --------
+    >>> from sparknlp.pretrained import PretrainedPipeline
+    >>> from sparknlp.functions import *
+    >>> explain_document_pipeline = PretrainedPipeline("explain_document_dl")
+    >>> data = spark.createDataFrame([["U.N. official Ekeus heads for Baghdad."]]).toDF("text")
+    >>> result = explain_document_pipeline.transform(data)
+    >>> result.select("pos.result").show(truncate=False)
+    +----------------------------------+
+    |result                            |
+    +----------------------------------+
+    |[NNP, ., JJ, NNP, VBZ, IN, NNP, .]|
+    +----------------------------------+
+    >>> explode_annotations_col(result, "pos", "pos").select("pos.result").show()
+    +------+
+    |result|
+    +------+
+    |   NNP|
+    |     .|
+    |    JJ|
+    |   NNP|
+    |   VBZ|
+    |    IN|
+    |   NNP|
+    |     .|
+    +------+
+    """
     from pyspark.sql.functions import explode
     return dataframe.withColumn(output_column, explode(column))