spark-nlp 6.1.3rc1__py2.py3-none-any.whl → 6.1.4__py2.py3-none-any.whl

sparknlp/reader/reader2doc.py

@@ -21,9 +21,10 @@ from sparknlp.partition.partition_properties import *
 
 class Reader2Doc(
     AnnotatorTransformer,
+    HasReaderProperties,
+    HasHTMLReaderProperties,
     HasEmailReaderProperties,
     HasExcelReaderProperties,
-    HasHTMLReaderProperties,
     HasPowerPointProperties,
     HasTextReaderProperties
 ):
@@ -73,33 +74,6 @@ class Reader2Doc(
     name = "Reader2Doc"
     outputAnnotatorType = AnnotatorType.DOCUMENT
 
-    contentPath = Param(
-        Params._dummy(),
-        "contentPath",
-        "contentPath path to files to read",
-        typeConverter=TypeConverters.toString
-    )
-
-    outputCol = Param(
-        Params._dummy(),
-        "outputCol",
-        "output column name",
-        typeConverter=TypeConverters.toString
-    )
-
-    contentType = Param(
-        Params._dummy(),
-        "contentType",
-        "Set the content type to load following MIME specification",
-        typeConverter=TypeConverters.toString
-    )
-
-    explodeDocs = Param(
-        Params._dummy(),
-        "explodeDocs",
-        "whether to explode the documents into separate rows",
-        typeConverter=TypeConverters.toBoolean
-    )
 
     flattenOutput = Param(
         Params._dummy(),
@@ -115,11 +89,18 @@ class Reader2Doc(
         typeConverter=TypeConverters.toFloat
     )
 
-    outputFormat = Param(
+    outputAsDocument = Param(
+        Params._dummy(),
+        "outputAsDocument",
+        "Whether to return all sentences joined into a single document",
+        typeConverter=TypeConverters.toBoolean
+    )
+
+    excludeNonText = Param(
         Params._dummy(),
-        "outputFormat",
-        "Output format for the table content. Options are 'plain-text' or 'html-table'. Default is 'json-table'.",
-        typeConverter=TypeConverters.toString
+        "excludeNonText",
+        "Whether to exclude non-text content from the output. Default is False.",
+        typeConverter=TypeConverters.toBoolean
     )
 
     @keyword_only
@@ -137,52 +118,11 @@ class Reader2Doc(
         kwargs = self._input_kwargs
         return self._set(**kwargs)
 
-    def setContentPath(self, value):
-        """Sets content path.
-
-        Parameters
-        ----------
-        value : str
-            contentPath path to files to read
-        """
-        return self._set(contentPath=value)
-
-    def setContentType(self, value):
-        """
-        Set the content type to load following MIME specification
-
-        Parameters
-        ----------
-        value : str
-            content type to load following MIME specification
-        """
-        return self._set(contentType=value)
-
-    def setExplodeDocs(self, value):
-        """Sets whether to explode the documents into separate rows.
-
-
-        Parameters
-        ----------
-        value : boolean
-        Whether to explode the documents into separate rows
-        """
-        return self._set(explodeDocs=value)
-
-    def setOutputCol(self, value):
-        """Sets output column name.
-
-        Parameters
-        ----------
-        value : str
-            Name of the Output Column
-        """
-        return self._set(outputCol=value)
 
     def setFlattenOutput(self, value):
         """Sets whether to flatten the output to plain text with minimal metadata.
 
-        Parameters
+        ParametersF
         ----------
         value : bool
             If true, output is flattened to plain text with minimal metadata
@@ -199,12 +139,22 @@ class Reader2Doc(
         """
         return self._set(titleThreshold=value)
 
-    def setOutputFormat(self, value):
-        """Sets the output format for the table content.
+    def setOutputAsDocument(self, value):
+        """Sets whether to return all sentences joined into a single document.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to return all sentences joined into a single document
+        """
+        return self._set(outputAsDocument=value)
+
+    def setExcludeNonText(self, value):
+        """Sets whether to exclude non-text content from the output.
 
         Parameters
         ----------
-        value : str
-            Output format for the table content. Options are 'plain-text' or 'html-table'. Default is 'json-table'.
+        value : bool
+            Whether to exclude non-text content from the output. Default is False.
         """
-        return self._set(outputFormat=value)
+        return self._set(excludeNonText=value)

sparknlp/reader/reader2image.py

@@ -20,46 +20,84 @@ from sparknlp.partition.partition_properties import *
 
 class Reader2Image(
     AnnotatorTransformer,
-    HasHTMLReaderProperties
+    HasReaderProperties,
+    HasHTMLReaderProperties,
+    HasPdfProperties
 ):
+    """
+    The Reader2Image annotator allows you to use the reading files with images more smoothly within existing
+    Spark NLP workflows, enabling seamless reuse of your pipelines. Reader2Image can be used for
+    extracting structured image content from various document types using Spark NLP readers. It supports
+    reading from many file types and returns parsed output as a structured Spark DataFrame.
+
+    Supported formats include HTML and Markdown.
+
+    == Example ==
+    This example demonstrates how to load HTML files with images and process them into a structured
+    Spark DataFrame using Reader2Image.
+
+    Expected output:
+    +-------------------+--------------------+
+    |           fileName|               image|
+    +-------------------+--------------------+
+    |example-images.html|[{image, example-...|
+    |example-images.html|[{image, example-...|
+    +-------------------+--------------------+
+
+    Schema:
+    root
+     |-- fileName: string (nullable = true)
+     |-- image: array (nullable = false)
+     |    |-- element: struct (containsNull = true)
+     |    |    |-- annotatorType: string (nullable = true)
+     |    |    |-- origin: string (nullable = true)
+     |    |    |-- height: integer (nullable = false)
+     |    |    |-- width: integer (nullable = false)
+     |    |    |-- nChannels: integer (nullable = false)
+     |    |    |-- mode: integer (nullable = false)
+     |    |    |-- result: binary (nullable = true)
+     |    |    |-- metadata: map (nullable = true)
+     |    |    |    |-- key: string
+     |    |    |    |-- value: string (valueContainsNull = true)
+     |    |    |-- text: string (nullable = true)
+    """
+
     name = "Reader2Image"
     outputAnnotatorType = AnnotatorType.IMAGE
 
-    contentPath = Param(
+    userMessage = Param(
         Params._dummy(),
-        "contentPath",
-        "contentPath path to files to read",
+        "userMessage",
+        "Custom user message.",
         typeConverter=TypeConverters.toString
     )
 
-    outputCol = Param(
+    promptTemplate = Param(
         Params._dummy(),
-        "outputCol",
-        "output column name",
+        "promptTemplate",
+        "Format of the output prompt.",
         typeConverter=TypeConverters.toString
     )
 
-    contentType = Param(
+    customPromptTemplate = Param(
         Params._dummy(),
-        "contentType",
-        "Set the content type to load following MIME specification",
+        "customPromptTemplate",
+        "Custom prompt template for image models.",
         typeConverter=TypeConverters.toString
     )
 
-    explodeDocs = Param(
-        Params._dummy(),
-        "explodeDocs",
-        "whether to explode the documents into separate rows",
-        typeConverter=TypeConverters.toBoolean
-    )
-
     @keyword_only
     def __init__(self):
         super(Reader2Image, self).__init__(classname="com.johnsnowlabs.reader.Reader2Image")
         self._setDefault(
-            outputCol="document",
+            contentType="",
+            outputFormat="image",
             explodeDocs=True,
-            contentType=""
+            userMessage="Describe this image",
+            promptTemplate="qwen2vl-chat",
+            readAsImage=True,
+            customPromptTemplate="",
+            ignoreExceptions=True
         )
 
     @keyword_only
@@ -67,44 +105,32 @@ class Reader2Image(
         kwargs = self._input_kwargs
         return self._set(**kwargs)
 
-    def setContentPath(self, value):
-        """Sets content path.
+    def setUserMessage(self, value: str):
+        """Sets custom user message.
 
         Parameters
         ----------
         value : str
-            contentPath path to files to read
+            Custom user message to include.
         """
-        return self._set(contentPath=value)
+        return self._set(userMessage=value)
 
-    def setContentType(self, value):
-        """
-        Set the content type to load following MIME specification
+    def setPromptTemplate(self, value: str):
+        """Sets format of the output prompt.
 
         Parameters
         ----------
         value : str
-            content type to load following MIME specification
-        """
-        return self._set(contentType=value)
-
-    def setExplodeDocs(self, value):
-        """Sets whether to explode the documents into separate rows.
-
-
-        Parameters
-        ----------
-        value : boolean
-        Whether to explode the documents into separate rows
+            Prompt template format.
         """
-        return self._set(explodeDocs=value)
+        return self._set(promptTemplate=value)
 
-    def setOutputCol(self, value):
-        """Sets output column name.
+    def setCustomPromptTemplate(self, value: str):
+        """Sets custom prompt template for image models.
 
         Parameters
         ----------
         value : str
-            Name of the Output Column
+            Custom prompt template string.
         """
-        return self._set(outputCol=value)
+        return self._set(customPromptTemplate=value)

sparknlp/reader/reader2table.py

@@ -13,14 +13,15 @@
 #  limitations under the License.
 
 from pyspark import keyword_only
-from pyspark.ml.param import TypeConverters, Params, Param
 
 from sparknlp.common import AnnotatorType
 from sparknlp.internal import AnnotatorTransformer
 from sparknlp.partition.partition_properties import *
 
+
 class Reader2Table(
     AnnotatorTransformer,
+    HasReaderProperties,
     HasEmailReaderProperties,
     HasExcelReaderProperties,
     HasHTMLReaderProperties,
@@ -31,34 +32,6 @@ class Reader2Table(
 
     outputAnnotatorType = AnnotatorType.DOCUMENT
 
-    contentPath = Param(
-        Params._dummy(),
-        "contentPath",
-        "contentPath path to files to read",
-        typeConverter=TypeConverters.toString
-    )
-
-    outputCol = Param(
-        Params._dummy(),
-        "outputCol",
-        "output column name",
-        typeConverter=TypeConverters.toString
-    )
-
-    contentType = Param(
-        Params._dummy(),
-        "contentType",
-        "Set the content type to load following MIME specification",
-        typeConverter=TypeConverters.toString
-    )
-
-    explodeDocs = Param(
-        Params._dummy(),
-        "explodeDocs",
-        "whether to explode the documents into separate rows",
-        typeConverter=TypeConverters.toBoolean
-    )
-
     flattenOutput = Param(
         Params._dummy(),
         "flattenOutput",
@@ -73,13 +46,6 @@ class Reader2Table(
         typeConverter=TypeConverters.toFloat
     )
 
-    outputFormat = Param(
-        Params._dummy(),
-        "outputFormat",
-        "Output format for the table content. Options are 'plain-text' or 'html-table'. Default is 'json-table'.",
-        typeConverter=TypeConverters.toString
-    )
-
     @keyword_only
     def __init__(self):
         super(Reader2Table, self).__init__(classname="com.johnsnowlabs.reader.Reader2Table")
@@ -90,48 +56,6 @@ class Reader2Table(
         kwargs = self._input_kwargs
         return self._set(**kwargs)
 
-    def setContentPath(self, value):
-        """Sets content path.
-
-        Parameters
-        ----------
-        value : str
-            contentPath path to files to read
-        """
-        return self._set(contentPath=value)
-
-    def setContentType(self, value):
-        """
-        Set the content type to load following MIME specification
-
-        Parameters
-        ----------
-        value : str
-            content type to load following MIME specification
-        """
-        return self._set(contentType=value)
-
-    def setExplodeDocs(self, value):
-        """Sets whether to explode the documents into separate rows.
-
-
-        Parameters
-        ----------
-        value : boolean
-        Whether to explode the documents into separate rows
-        """
-        return self._set(explodeDocs=value)
-
-    def setOutputCol(self, value):
-        """Sets output column name.
-
-        Parameters
-        ----------
-        value : str
-            Name of the Output Column
-        """
-        return self._set(outputCol=value)
-
     def setFlattenOutput(self, value):
         """Sets whether to flatten the output to plain text with minimal metadata.
 
@@ -151,13 +75,3 @@ class Reader2Table(
             Minimum font size threshold for title detection in PDF docs
         """
         return self._set(titleThreshold=value)
-
-    def setOutputFormat(self, value):
-        """Sets the output format for the table content.
-
-        Parameters
-        ----------
-        value : str
-            Output format for the table content. Options are 'plain-text' or 'html-table'. Default is 'json-table'.
-        """
-        return self._set(outputFormat=value)

sparknlp/annotator/classifier_dl/roberta_bert_for_zero_shot_classification.py

@@ -1,225 +0,0 @@
-#  Copyright 2017-2023 John Snow Labs
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""Contains classes for RoBertaForZeroShotClassification."""
-
-from sparknlp.common import *
-
-
-class RoBertaForZeroShotClassification(AnnotatorModel,
-                                       HasCaseSensitiveProperties,
-                                       HasBatchedAnnotate,
-                                       HasClassifierActivationProperties,
-                                       HasCandidateLabelsProperties,
-                                       HasEngine):
-    """RoBertaForZeroShotClassification using a `ModelForSequenceClassification` trained on NLI (natural language
-    inference) tasks. Equivalent of `RoBertaForSequenceClassification` models, but these models don't require a hardcoded
-    number of potential classes, they can be chosen at runtime. It usually means it's slower but it is much more
-    flexible.
-
-    Note that the model will loop through all provided labels. So the more labels you have, the
-    longer this process will take.
-
-    Any combination of sequences and labels can be passed and each combination will be posed as a premise/hypothesis
-    pair and passed to the pretrained model.
-
-    Pretrained models can be loaded with :meth:`.pretrained` of the companion
-    object:
-
-    >>> sequenceClassifier = RoBertaForZeroShotClassification.pretrained() \\
-    ...     .setInputCols(["token", "document"]) \\
-    ...     .setOutputCol("label")
-
-    The default model is ``"roberta_base_zero_shot_classifier_nli"``, if no name is
-    provided.
-
-    For available pretrained models please see the `Models Hub
-    <https://sparknlp.orgtask=Text+Classification>`__.
-
-    To see which models are compatible and how to import them see
-    `Import Transformers into Spark NLP 🚀
-    <https://github.com/JohnSnowLabs/spark-nlp/discussions/5669>`_.
-
-    ====================== ======================
-    Input Annotation types Output Annotation type
-    ====================== ======================
-    ``DOCUMENT, TOKEN``    ``CATEGORY``
-    ====================== ======================
-
-    Parameters
-    ----------
-    batchSize
-        Batch size. Large values allows faster processing but requires more
-        memory, by default 8
-    caseSensitive
-        Whether to ignore case in tokens for embeddings matching, by default
-        True
-    configProtoBytes
-        ConfigProto from tensorflow, serialized into byte array.
-    maxSentenceLength
-        Max sentence length to process, by default 128
-    coalesceSentences
-        Instead of 1 class per sentence (if inputCols is `sentence`) output 1
-        class per document by averaging probabilities in all sentences, by
-        default False
-    activation
-        Whether to calculate logits via Softmax or Sigmoid, by default
-        `"softmax"`.
-
-    Examples
-    --------
-    >>> import sparknlp
-    >>> from sparknlp.base import *
-    >>> from sparknlp.annotator import *
-    >>> from pyspark.ml import Pipeline
-    >>> documentAssembler = DocumentAssembler() \\
-    ...     .setInputCol("text") \\
-    ...     .setOutputCol("document")
-    >>> tokenizer = Tokenizer() \\
-    ...     .setInputCols(["document"]) \\
-    ...     .setOutputCol("token")
-    >>> sequenceClassifier = RoBertaForZeroShotClassification.pretrained() \\
-    ...     .setInputCols(["token", "document"]) \\
-    ...     .setOutputCol("label") \\
-    ...     .setCaseSensitive(True)
-    >>> pipeline = Pipeline().setStages([
-    ...     documentAssembler,
-    ...     tokenizer,
-    ...     sequenceClassifier
-    ... ])
-    >>> data = spark.createDataFrame([["I loved this movie when I was a child.", "It was pretty boring."]]).toDF("text")
-    >>> result = pipeline.fit(data).transform(data)
-    >>> result.select("label.result").show(truncate=False)
-    +------+
-    |result|
-    +------+
-    |[pos] |
-    |[neg] |
-    +------+
-    """
-    name = "RoBertaForZeroShotClassification"
-
-    inputAnnotatorTypes = [AnnotatorType.DOCUMENT, AnnotatorType.TOKEN]
-
-    outputAnnotatorType = AnnotatorType.CATEGORY
-
-    maxSentenceLength = Param(Params._dummy(),
-                              "maxSentenceLength",
-                              "Max sentence length to process",
-                              typeConverter=TypeConverters.toInt)
-
-    configProtoBytes = Param(Params._dummy(),
-                             "configProtoBytes",
-                             "ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()",
-                             TypeConverters.toListInt)
-
-    coalesceSentences = Param(Params._dummy(), "coalesceSentences",
-                              "Instead of 1 class per sentence (if inputCols is '''sentence''') output 1 class per document by averaging probabilities in all sentences.",
-                              TypeConverters.toBoolean)
-
-    def getClasses(self):
-        """
-        Returns labels used to train this model
-        """
-        return self._call_java("getClasses")
-
-    def setConfigProtoBytes(self, b):
-        """Sets configProto from tensorflow, serialized into byte array.
-
-        Parameters
-        ----------
-        b : List[int]
-            ConfigProto from tensorflow, serialized into byte array
-        """
-        return self._set(configProtoBytes=b)
-
-    def setMaxSentenceLength(self, value):
-        """Sets max sentence length to process, by default 128.
-
-        Parameters
-        ----------
-        value : int
-            Max sentence length to process
-        """
-        return self._set(maxSentenceLength=value)
-
-    def setCoalesceSentences(self, value):
-        """Instead of 1 class per sentence (if inputCols is '''sentence''') output 1 class per document by averaging
-        probabilities in all sentences. Due to max sequence length limit in almost all transformer models such as RoBerta
-        (512 tokens), this parameter helps to feed all the sentences into the model and averaging all the probabilities
-        for the entire document instead of probabilities per sentence. (Default: true)
-
-        Parameters
-        ----------
-        value : bool
-            If the output of all sentences will be averaged to one output
-        """
-        return self._set(coalesceSentences=value)
-
-    @keyword_only
-    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.classifier.dl.RoBertaForZeroShotClassification",
-                 java_model=None):
-        super(RoBertaForZeroShotClassification, self).__init__(
-            classname=classname,
-            java_model=java_model
-        )
-        self._setDefault(
-            batchSize=8,
-            maxSentenceLength=128,
-            caseSensitive=True,
-            coalesceSentences=False,
-            activation="softmax"
-        )
-
-    @staticmethod
-    def loadSavedModel(folder, spark_session):
-        """Loads a locally saved model.
-
-        Parameters
-        ----------
-        folder : str
-            Folder of the saved model
-            spark_session : pyspark.sql.SparkSession
-            The current SparkSession
-
-        Returns
-        -------
-        RoBertaForZeroShotClassification
-            The restored model
-        """
-        from sparknlp.internal import _RoBertaForZeroShotClassification
-        jModel = _RoBertaForZeroShotClassification(folder, spark_session._jsparkSession)._java_obj
-        return RoBertaForZeroShotClassification(java_model=jModel)
-
-    @staticmethod
-    def pretrained(name="roberta_base_zero_shot_classifier_nli", lang="en", remote_loc=None):
-        """Downloads and loads a pretrained model.
-
-        Parameters
-        ----------
-        name : str, optional
-            Name of the pretrained model, by default
-            "roberta_base_zero_shot_classifier_nli"
-            lang : str, optional
-            Language of the pretrained model, by default "en"
-            remote_loc : str, optional
-            Optional remote address of the resource, by default None. Will use
-            Spark NLPs repositories otherwise.
-
-        Returns
-        -------
-        RoBertaForZeroShotClassification
-            The restored model
-        """
-        from sparknlp.pretrained import ResourceDownloader
-        return ResourceDownloader.downloadModel(RoBertaForZeroShotClassification, name, lang, remote_loc)