spark-nlp 4.2.6__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py

@@ -19,7 +19,8 @@ from sparknlp.common import *
 class XlmRoBertaForTokenClassification(AnnotatorModel,
                                        HasCaseSensitiveProperties,
                                        HasBatchedAnnotate,
-                                       HasEngine):
+                                       HasEngine,
+                                       HasMaxSentenceLengthLimit):
     """XlmRoBertaForTokenClassification can load XLM-RoBERTa Models with a token
     classification head on top (a linear layer on top of the hidden-states
     output) e.g. for Named-Entity-Recognition (NER) tasks.
@@ -30,11 +31,11 @@ class XlmRoBertaForTokenClassification(AnnotatorModel,
     >>> token_classifier = XlmRoBertaForTokenClassification.pretrained() \\
     ...     .setInputCols(["token", "document"]) \\
     ...     .setOutputCol("label")
-    The default model is ``"xlm_roberta_base_token_classifier_conll03"``, if no
+    The default model is ``"mpnet_base_token_classifier"``, if no
     name is provided.
 
     For available pretrained models please see the `Models Hub
-    <https://nlp.johnsnowlabs.com/models?task=Named+Entity+Recognition>`__.
+    <https://sparknlp.org/models?task=Named+Entity+Recognition>`__.
     To see which models are compatible and how to import them see
     `Import Transformers into Spark NLP 🚀
     <https://github.com/JohnSnowLabs/spark-nlp/discussions/5669>`_.
@@ -94,11 +95,6 @@ class XlmRoBertaForTokenClassification(AnnotatorModel,
 
     outputAnnotatorType = AnnotatorType.NAMED_ENTITY
 
-    maxSentenceLength = Param(Params._dummy(),
-                              "maxSentenceLength",
-                              "Max sentence length to process",
-                              typeConverter=TypeConverters.toInt)
-
     configProtoBytes = Param(Params._dummy(),
                              "configProtoBytes",
                              "ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()",
@@ -120,16 +116,6 @@ class XlmRoBertaForTokenClassification(AnnotatorModel,
         """
         return self._set(configProtoBytes=b)
 
-    def setMaxSentenceLength(self, value):
-        """Sets max sentence length to process, by default 128.
-
-        Parameters
-        ----------
-        value : int
-            Max sentence length to process
-        """
-        return self._set(maxSentenceLength=value)
-
     @keyword_only
     def __init__(self, classname="com.johnsnowlabs.nlp.annotators.classifier.dl.XlmRoBertaForTokenClassification",
                  java_model=None):
@@ -164,14 +150,14 @@ class XlmRoBertaForTokenClassification(AnnotatorModel,
         return XlmRoBertaForTokenClassification(java_model=jModel)
 
     @staticmethod
-    def pretrained(name="xlm_roberta_base_token_classifier_conll03", lang="en", remote_loc=None):
+    def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None):
         """Downloads and loads a pretrained model.
 
         Parameters
         ----------
         name : str, optional
             Name of the pretrained model, by default
-            "xlm_roberta_base_token_classifier_conll03"
+            "mpnet_base_token_classifier"
         lang : str, optional
             Language of the pretrained model, by default "en"
         remote_loc : str, optional

sparknlp/annotator/classifier_dl/xlm_roberta_for_zero_shot_classification.py

@@ -0,0 +1,225 @@
+#  Copyright 2017-2023 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for XlmRoBertaForZeroShotClassification."""
+
+from sparknlp.common import *
+
+
+class XlmRoBertaForZeroShotClassification(AnnotatorModel,
+                                       HasCaseSensitiveProperties,
+                                       HasBatchedAnnotate,
+                                       HasClassifierActivationProperties,
+                                       HasCandidateLabelsProperties,
+                                       HasEngine):
+    """XlmRoBertaForZeroShotClassification using a `ModelForSequenceClassification` trained on NLI (natural language
+    inference) tasks. Equivalent of `XlmRoBertaForSequenceClassification` models, but these models don't require a hardcoded
+    number of potential classes, they can be chosen at runtime. It usually means it's slower but it is much more
+    flexible.
+
+    Note that the model will loop through all provided labels. So the more labels you have, the
+    longer this process will take.
+
+    Any combination of sequences and labels can be passed and each combination will be posed as a premise/hypothesis
+    pair and passed to the pretrained model.
+
+    Pretrained models can be loaded with :meth:`.pretrained` of the companion
+    object:
+
+    >>> sequenceClassifier = XlmRoBertaForZeroShotClassification.pretrained() \\
+    ...     .setInputCols(["token", "document"]) \\
+    ...     .setOutputCol("label")
+
+    The default model is ``"xlm_roberta_large_zero_shot_classifier_xnli_anli"``, if no name is
+    provided.
+
+    For available pretrained models please see the `Models Hub
+    <https://sparknlp.orgtask=Text+Classification>`__.
+
+    To see which models are compatible and how to import them see
+    `Import Transformers into Spark NLP 🚀
+    <https://github.com/JohnSnowLabs/spark-nlp/discussions/5669>`_.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT, TOKEN``    ``CATEGORY``
+    ====================== ======================
+
+    Parameters
+    ----------
+    batchSize
+        Batch size. Large values allows faster processing but requires more
+        memory, by default 8
+    caseSensitive
+        Whether to ignore case in tokens for embeddings matching, by default
+        True
+    configProtoBytes
+        ConfigProto from tensorflow, serialized into byte array.
+    maxSentenceLength
+        Max sentence length to process, by default 128
+    coalesceSentences
+        Instead of 1 class per sentence (if inputCols is `sentence`) output 1
+        class per document by averaging probabilities in all sentences, by
+        default False
+    activation
+        Whether to calculate logits via Softmax or Sigmoid, by default
+        `"softmax"`.
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> tokenizer = Tokenizer() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("token")
+    >>> sequenceClassifier = XlmRoBertaForZeroShotClassification.pretrained() \\
+    ...     .setInputCols(["token", "document"]) \\
+    ...     .setOutputCol("label") \\
+    ...     .setCaseSensitive(True)
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     tokenizer,
+    ...     sequenceClassifier
+    ... ])
+    >>> data = spark.createDataFrame([["I loved this movie when I was a child.", "It was pretty boring."]]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.select("label.result").show(truncate=False)
+    +------+
+    |result|
+    +------+
+    |[pos] |
+    |[neg] |
+    +------+
+    """
+    name = "XlmRoBertaForZeroShotClassification"
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT, AnnotatorType.TOKEN]
+
+    outputAnnotatorType = AnnotatorType.CATEGORY
+
+    maxSentenceLength = Param(Params._dummy(),
+                              "maxSentenceLength",
+                              "Max sentence length to process",
+                              typeConverter=TypeConverters.toInt)
+
+    configProtoBytes = Param(Params._dummy(),
+                             "configProtoBytes",
+                             "ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()",
+                             TypeConverters.toListInt)
+
+    coalesceSentences = Param(Params._dummy(), "coalesceSentences",
+                              "Instead of 1 class per sentence (if inputCols is '''sentence''') output 1 class per document by averaging probabilities in all sentences.",
+                              TypeConverters.toBoolean)
+
+    def getClasses(self):
+        """
+        Returns labels used to train this model
+        """
+        return self._call_java("getClasses")
+
+    def setConfigProtoBytes(self, b):
+        """Sets configProto from tensorflow, serialized into byte array.
+
+        Parameters
+        ----------
+        b : List[int]
+            ConfigProto from tensorflow, serialized into byte array
+        """
+        return self._set(configProtoBytes=b)
+
+    def setMaxSentenceLength(self, value):
+        """Sets max sentence length to process, by default 128.
+
+        Parameters
+        ----------
+        value : int
+            Max sentence length to process
+        """
+        return self._set(maxSentenceLength=value)
+
+    def setCoalesceSentences(self, value):
+        """Instead of 1 class per sentence (if inputCols is '''sentence''') output 1 class per document by averaging
+        probabilities in all sentences. Due to max sequence length limit in almost all transformer models such as XlmRoBerta
+        (512 tokens), this parameter helps to feed all the sentences into the model and averaging all the probabilities
+        for the entire document instead of probabilities per sentence. (Default: true)
+
+        Parameters
+        ----------
+        value : bool
+            If the output of all sentences will be averaged to one output
+        """
+        return self._set(coalesceSentences=value)
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.classifier.dl.XlmRoBertaForZeroShotClassification",
+                 java_model=None):
+        super(XlmRoBertaForZeroShotClassification, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+        self._setDefault(
+            batchSize=8,
+            maxSentenceLength=128,
+            caseSensitive=True,
+            coalesceSentences=False,
+            activation="softmax"
+        )
+
+    @staticmethod
+    def loadSavedModel(folder, spark_session):
+        """Loads a locally saved model.
+
+        Parameters
+        ----------
+        folder : str
+            Folder of the saved model
+            spark_session : pyspark.sql.SparkSession
+            The current SparkSession
+
+        Returns
+        -------
+        XlmRoBertaForZeroShotClassification
+            The restored model
+        """
+        from sparknlp.internal import _XlmRoBertaForZeroShotClassification
+        jModel = _XlmRoBertaForZeroShotClassification(folder, spark_session._jsparkSession)._java_obj
+        return XlmRoBertaForZeroShotClassification(java_model=jModel)
+
+    @staticmethod
+    def pretrained(name="xlm_roberta_large_zero_shot_classifier_xnli_anli", lang="xx", remote_loc=None):
+        """Downloads and loads a pretrained model.
+
+        Parameters
+        ----------
+        name : str, optional
+            Name of the pretrained model, by default
+            "xlm_roberta_large_zero_shot_classifier_xnli_anli"
+            lang : str, optional
+            Language of the pretrained model, by default "en"
+            remote_loc : str, optional
+            Optional remote address of the resource, by default None. Will use
+            Spark NLPs repositories otherwise.
+
+        Returns
+        -------
+        XlmRoBertaForZeroShotClassification
+            The restored model
+        """
+        from sparknlp.pretrained import ResourceDownloader
+        return ResourceDownloader.downloadModel(XlmRoBertaForZeroShotClassification, name, lang, remote_loc)

sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py

@@ -20,7 +20,8 @@ class XlnetForSequenceClassification(AnnotatorModel,
                                      HasCaseSensitiveProperties,
                                      HasBatchedAnnotate,
                                      HasClassifierActivationProperties,
-                                     HasEngine):
+                                     HasEngine,
+                                     HasMaxSentenceLengthLimit):
     """XlnetForSequenceClassification can load XLNet Models with sequence classification/regression head on
     top (a linear layer on top of the pooled output) e.g. for multi-class document classification tasks.
 
@@ -35,7 +36,7 @@ class XlnetForSequenceClassification(AnnotatorModel,
     provided.
 
     For available pretrained models please see the `Models Hub
-    <https://nlp.johnsnowlabs.com/models?task=Text+Classification>`__.
+    <https://sparknlp.org/models?task=Text+Classification>`__.
 
     To see which models are compatible and how to import them see
     `Import Transformers into Spark NLP 🚀
@@ -61,7 +62,7 @@ class XlnetForSequenceClassification(AnnotatorModel,
         Max sentence length to process, by default 128
     coalesceSentences
         Instead of 1 class per sentence (if inputCols is `sentence`) output
-        1 class per document by averaging probabilities in all sentences, by 
+        1 class per document by averaging probabilities in all sentences, by
         default False.
     activation
         Whether to calculate logits via Softmax or Sigmoid, by default
@@ -104,11 +105,6 @@ class XlnetForSequenceClassification(AnnotatorModel,
 
     outputAnnotatorType = AnnotatorType.CATEGORY
 
-    maxSentenceLength = Param(Params._dummy(),
-                              "maxSentenceLength",
-                              "Max sentence length to process",
-                              typeConverter=TypeConverters.toInt)
-
     configProtoBytes = Param(Params._dummy(),
                              "configProtoBytes",
                              "ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()",
@@ -134,16 +130,6 @@ class XlnetForSequenceClassification(AnnotatorModel,
         """
         return self._set(configProtoBytes=b)
 
-    def setMaxSentenceLength(self, value):
-        """Sets max sentence length to process, by default 128.
-
-        Parameters
-        ----------
-        value : int
-            Max sentence length to process
-        """
-        return self._set(maxSentenceLength=value)
-
     def setCoalesceSentences(self, value):
         """Instead of 1 class per sentence (if inputCols is '''sentence''') output 1 class per document by averaging probabilities in all sentences.
         Due to max sequence length limit in almost all transformer models such as BERT (512 tokens), this parameter helps feeding all the sentences

sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py

@@ -19,7 +19,8 @@ from sparknlp.common import *
 class XlnetForTokenClassification(AnnotatorModel,
                                   HasCaseSensitiveProperties,
                                   HasBatchedAnnotate,
-                                  HasEngine):
+                                  HasEngine,
+                                  HasMaxSentenceLengthLimit):
     """XlnetForTokenClassification can load XLNet Models with a token
     classification head on top (a linear layer on top of the hidden-states
     output) e.g. for Named-Entity-Recognition (NER) tasks.
@@ -35,7 +36,7 @@ class XlnetForTokenClassification(AnnotatorModel,
     provided.
 
     For available pretrained models please see the `Models Hub
-    <https://nlp.johnsnowlabs.com/models?task=Named+Entity+Recognition>`__.
+    <https://sparknlp.org/models?task=Named+Entity+Recognition>`__.
 
     To see which models are compatible and how to import them see
     `Import Transformers into Spark NLP 🚀
@@ -97,11 +98,6 @@ class XlnetForTokenClassification(AnnotatorModel,
 
     outputAnnotatorType = AnnotatorType.NAMED_ENTITY
 
-    maxSentenceLength = Param(Params._dummy(),
-                              "maxSentenceLength",
-                              "Max sentence length to process",
-                              typeConverter=TypeConverters.toInt)
-
     configProtoBytes = Param(Params._dummy(),
                              "configProtoBytes",
                              "ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()",
@@ -123,16 +119,6 @@ class XlnetForTokenClassification(AnnotatorModel,
         """
         return self._set(configProtoBytes=b)
 
-    def setMaxSentenceLength(self, value):
-        """Sets max sentence length to process, by default 128.
-
-        Parameters
-        ----------
-        value : int
-            Max sentence length to process
-        """
-        return self._set(maxSentenceLength=value)
-
     @keyword_only
     def __init__(self, classname="com.johnsnowlabs.nlp.annotators.classifier.dl.XlnetForTokenClassification",
                  java_model=None):

sparknlp/annotator/cleaners/__init__.py

@@ -0,0 +1,15 @@
+#  Copyright 2017-2025 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from sparknlp.annotator.cleaners.extractor import *
+from sparknlp.annotator.cleaners.cleaner import *

sparknlp/annotator/cleaners/cleaner.py

@@ -0,0 +1,202 @@
+#  Copyright 2017-2025 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for Cleaner."""
+from sparknlp.annotator import MarianTransformer
+from sparknlp.common import *
+
+class Cleaner(MarianTransformer):
+    name = "Cleaner"
+
+    inputAnnotatorTypes = [AnnotatorType.TOKEN]
+
+    outputAnnotatorType = AnnotatorType.CHUNK
+
+    encoding = Param(Params._dummy(),
+                   "encoding",
+                   "The encoding to be used for decoding the byte string (default is utf-8)",
+                   typeConverter=TypeConverters.toString)
+
+    cleanPrefixPattern = Param(Params._dummy(),
+                     "cleanPrefixPattern",
+                     "The pattern for the prefix. Can be a simple string or a regex pattern.",
+                     typeConverter=TypeConverters.toString)
+
+    cleanPostfixPattern = Param(Params._dummy(),
+                               "cleanPostfixPattern",
+                               "The pattern for the postfix. Can be a simple string or a regex pattern.",
+                               typeConverter=TypeConverters.toString)
+
+    cleanerMode = Param(
+        Params._dummy(),
+        "cleanerMode",
+        "possible values: " +
+        "clean, bytes_string_to_string, clean_non_ascii_chars, clean_ordered_bullets, clean_postfix, clean_prefix, remove_punctuation, replace_unicode_quotes",
+        typeConverter=TypeConverters.toString
+    )
+
+    extraWhitespace = Param(Params._dummy(),
+                    "extraWhitespace",
+                    "Whether to remove extra whitespace.",
+                    typeConverter=TypeConverters.toBoolean)
+
+    dashes = Param(Params._dummy(),
+                "dashes",
+                "Whether to handle dashes in text.",
+                typeConverter=TypeConverters.toBoolean)
+
+    bullets = Param(Params._dummy(),
+                   "bullets",
+                   "Whether to handle bullets in text.",
+                   typeConverter=TypeConverters.toBoolean)
+
+    trailingPunctuation = Param(Params._dummy(),
+                    "trailingPunctuation",
+                    "Whether to remove trailing punctuation from text.",
+                    typeConverter=TypeConverters.toBoolean)
+
+    lowercase = Param(Params._dummy(),
+                "lowercase",
+                "Whether to convert text to lowercase.",
+                typeConverter=TypeConverters.toBoolean)
+
+    ignoreCase = Param(Params._dummy(),
+                      "ignoreCase",
+                      "If true, ignores case in the pattern.",
+                      typeConverter=TypeConverters.toBoolean)
+
+    strip = Param(Params._dummy(),
+               "strip",
+               "If true, removes leading or trailing whitespace from the cleaned string.",
+               typeConverter=TypeConverters.toBoolean)
+
+    def setEncoding(self, value):
+        """Sets the encoding to be used for decoding the byte string (default is utf-8).
+
+        Parameters
+        ----------
+        value : str
+            The encoding to be used for decoding the byte string (default is utf-8)
+        """
+        return self._set(encoding=value)
+
+    def setCleanPrefixPattern(self, value):
+        """Sets the pattern for the prefix. Can be a simple string or a regex pattern.
+
+        Parameters
+        ----------
+        value : str
+            The pattern for the prefix. Can be a simple string or a regex pattern.
+        """
+        return self._set(cleanPrefixPattern=value)
+
+    def setCleanPostfixPattern(self, value):
+        """Sets the pattern for the postfix. Can be a simple string or a regex pattern.
+
+        Parameters
+        ----------
+        value : str
+            The pattern for the postfix. Can be a simple string or a regex pattern.
+        """
+        return self._set(cleanPostfixPattern=value)
+
+    def setCleanerMode(self, value):
+        """Sets the cleaner mode.
+
+        Possible values:
+            clean, bytes_string_to_string, clean_non_ascii_chars, clean_ordered_bullets,
+            clean_postfix, clean_prefix, remove_punctuation, replace_unicode_quotes
+
+        Parameters
+        ----------
+        value : str
+            The mode for cleaning operations.
+        """
+        return self._set(cleanerMode=value)
+
+    def setExtraWhitespace(self, value):
+        """Sets whether to remove extra whitespace.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to remove extra whitespace.
+        """
+        return self._set(extraWhitespace=value)
+
+    def setDashes(self, value):
+        """Sets whether to handle dashes in text.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to handle dashes in text.
+        """
+        return self._set(dashes=value)
+
+    def setBullets(self, value):
+        """Sets whether to handle bullets in text.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to handle bullets in text.
+        """
+        return self._set(bullets=value)
+
+    def setTrailingPunctuation(self, value):
+        """Sets whether to remove trailing punctuation from text.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to remove trailing punctuation from text.
+        """
+        return self._set(trailingPunctuation=value)
+
+    def setLowercase(self, value):
+        """Sets whether to convert text to lowercase.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to convert text to lowercase.
+        """
+        return self._set(lowercase=value)
+
+    def setIgnoreCase(self, value):
+        """Sets whether to ignore case in the pattern.
+
+        Parameters
+        ----------
+        value : bool
+            If true, ignores case in the pattern.
+        """
+        return self._set(ignoreCase=value)
+
+    def setStrip(self, value):
+        """Sets whether to remove leading or trailing whitespace from the cleaned string.
+
+        Parameters
+        ----------
+        value : bool
+            If true, removes leading or trailing whitespace from the cleaned string.
+        """
+        return self._set(strip=value)
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.cleaners.Cleaner", java_model=None):
+        super(Cleaner, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )