spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/internal/annotator_java_ml.py

@@ -0,0 +1,32 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains utility classes for annotators."""
+
+from pyspark.ml.util import JavaMLReader, JavaMLReadable
+
+
+class AnnotatorJavaMLReadable(JavaMLReadable):
+    @classmethod
+    def read(cls):
+        """Returns an MLReader instance for this class."""
+        return AnnotatorJavaMLReader(cls())
+
+
+class AnnotatorJavaMLReader(JavaMLReader):
+    @classmethod
+    def _java_loader_class(cls, clazz):
+        if hasattr(clazz, '_java_class_name') and clazz._java_class_name is not None:
+            return clazz._java_class_name
+        else:
+            return JavaMLReader._java_loader_class(clazz)

sparknlp/internal/annotator_transformer.py

@@ -0,0 +1,37 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains the base classes for Annotators based on the Spark Transformer."""
+
+from pyspark import keyword_only
+from pyspark.ml.util import JavaMLWritable
+from pyspark.ml.wrapper import JavaTransformer
+
+from sparknlp.internal.params_getters_setters import ParamsGettersSetters
+from sparknlp.internal.annotator_java_ml import AnnotatorJavaMLReadable
+
+
+class AnnotatorTransformer(JavaTransformer, AnnotatorJavaMLReadable, JavaMLWritable, ParamsGettersSetters):
+
+    outputAnnotatorType = None
+
+    @keyword_only
+    def __init__(self, classname):
+        super(AnnotatorTransformer, self).__init__()
+        kwargs = self._input_kwargs
+        if 'classname' in kwargs:
+            kwargs.pop('classname')
+        self.setParams(**kwargs)
+        self.__class__._java_class_name = classname
+        self._java_obj = self._new_java_obj(classname, self.uid)
+

sparknlp/internal/extended_java_wrapper.py

@@ -0,0 +1,63 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for extensions of the PySpark JavaWrapper."""
+
+from pyspark import SparkContext
+from pyspark.ml.wrapper import JavaWrapper
+from pyspark.sql import DataFrame
+from distutils.version import LooseVersion
+
+
+class ExtendedJavaWrapper(JavaWrapper):
+
+    def __init__(self, java_obj, *args):
+        super(ExtendedJavaWrapper, self).__init__(java_obj)
+        self.sc = SparkContext._active_spark_context
+        self._java_obj = self.new_java_obj(java_obj, *args)
+        self.java_obj = self._java_obj
+
+    def __del__(self):
+        pass
+
+    def apply(self):
+        return self._java_obj
+
+    def new_java_obj(self, java_class, *args):
+        return self._new_java_obj(java_class, *args)
+
+    def new_java_array(self, pylist, java_class):
+        """
+        ToDo: Inspired from spark 2.0. Review if spark changes
+        """
+        java_array = self.sc._gateway.new_array(java_class, len(pylist))
+        for i in range(len(pylist)):
+            java_array[i] = pylist[i]
+        return java_array
+
+    def new_java_array_string(self, pylist):
+        java_array = self._new_java_array(pylist, self.sc._gateway.jvm.java.lang.String)
+        return java_array
+
+    def new_java_array_integer(self, pylist):
+        java_array = self._new_java_array(pylist, self.sc._gateway.jvm.java.lang.Integer)
+        return java_array
+
+    def spark_version(self):
+        return self.sc.version
+
+    def getDataFrame(self, spark, jdf):
+        if LooseVersion(self.spark_version()) >= LooseVersion("3.3.0"):
+            return DataFrame(jdf, spark._getActiveSessionOrCreate())
+        else:
+            return DataFrame(jdf, spark._wrapped)

sparknlp/internal/params_getters_setters.py

@@ -0,0 +1,71 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains utility classes for ParamsGettersSetters."""
+
+import re
+
+from pyspark.ml.param import Params
+
+
+class ParamsGettersSetters(Params):
+    getter_attrs = []
+
+    def __init__(self):
+        super(ParamsGettersSetters, self).__init__()
+        for param in self.params:
+            param_name = param.name
+            fg_attr = "get" + re.sub(r"(?:^|_)(.)", lambda m: m.group(1).upper(), param_name)
+            fs_attr = "set" + re.sub(r"(?:^|_)(.)", lambda m: m.group(1).upper(), param_name)
+            # Generates getter and setter only if not exists
+            try:
+                getattr(self, fg_attr)
+            except AttributeError:
+                setattr(self, fg_attr, self.getParamValue(param_name))
+            try:
+                getattr(self, fs_attr)
+            except AttributeError:
+                setattr(self, fs_attr, self.setParamValue(param_name))
+
+    def getParamValue(self, paramName):
+        """Gets the value of a parameter.
+
+        Parameters
+        ----------
+        paramName : str
+            Name of the parameter
+        """
+
+        def r():
+            try:
+                return self.getOrDefault(paramName)
+            except KeyError:
+                return None
+
+        return r
+
+    def setParamValue(self, paramName):
+        """Sets the value of a parameter.
+
+        Parameters
+        ----------
+        paramName : str
+            Name of the parameter
+        """
+
+        def r(v):
+            self.set(self.getParam(paramName), v)
+            return self
+
+        return r
+

sparknlp/internal/recursive.py

@@ -0,0 +1,70 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains base classes for Recursive Annotators and Estimators."""
+
+from abc import ABC
+
+from pyspark.ml.wrapper import JavaEstimator, JavaModel
+from pyspark.sql import DataFrame
+
+
+class RecursiveEstimator(JavaEstimator, ABC):
+
+    def _fit_java(self, dataset, pipeline=None):
+        self._transfer_params_to_java()
+        if pipeline:
+            return self._java_obj.recursiveFit(dataset._jdf, pipeline._to_java())
+        else:
+            return self._java_obj.fit(dataset._jdf)
+
+    def _fit(self, dataset, pipeline=None):
+        java_model = self._fit_java(dataset, pipeline)
+        model = self._create_model(java_model)
+        return self._copyValues(model)
+
+    def fit(self, dataset, params=None, pipeline=None):
+        if params is None:
+            params = dict()
+        if isinstance(params, (list, tuple)):
+            models = [None] * len(params)
+            for index, model in self.fitMultiple(dataset, params):
+                models[index] = model
+            return models
+        elif isinstance(params, dict):
+            if params:
+                return self.copy(params)._fit(dataset, pipeline=pipeline)
+            else:
+                return self._fit(dataset, pipeline=pipeline)
+        else:
+            raise ValueError("Params must be either a param map or a list/tuple of param maps, "
+                             "but got %s." % type(params))
+
+class RecursiveTransformer(JavaModel):
+
+    def _transform_recursive(self, dataset, recursive_pipeline):
+        self._transfer_params_to_java()
+        return DataFrame(self._java_obj.recursiveTransform(dataset._jdf, recursive_pipeline._to_java()),
+                         dataset.sql_ctx)
+
+    def transform_recursive(self, dataset, recursive_pipeline, params=None):
+        if params is None:
+            params = dict()
+        if isinstance(params, dict):
+            if params:
+                return self.copy(params)._transform_recursive(dataset, recursive_pipeline)
+            else:
+                return self._transform_recursive(dataset, recursive_pipeline)
+        else:
+            raise ValueError("Params must be a param map but got %s." % type(params))
+

sparknlp/logging/__init__.py

@@ -0,0 +1,15 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Contains third party logging applications."""