spark-nlp 6.0.1rc1__py2.py3-none-any.whl → 6.0.2__py2.py3-none-any.whl

sparknlp/annotator/seq2seq/mistral_transformer.py

@@ -91,8 +91,7 @@ class MistralTransformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
 
     References
     ----------
-    - `Mistral 7B
-      <https://mistral.ai/news/announcing-mistral_7b/>`__
+    - `Mistral 7B <https://mistral.ai/news/announcing-mistral_7b/>`__
     - https://github.com/mistralai/mistral-src
 
     **Paper Abstract:**
@@ -126,7 +125,7 @@ class MistralTransformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
     +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     |result                                                                                                                                                                                              |
     +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |[Leonardo Da Vinci invented the microscope?\n Question: Leonardo Da Vinci invented the microscope?\n Answer: No, Leonardo Da Vinci did not invent the microscope. The first microscope was invented |
+    |[Leonardo Da Vinci invented the microscope?\\n Question: Leonardo Da Vinci invented the microscope?\\n Answer: No, Leonardo Da Vinci did not invent the microscope. The first microscope was invented |
     | in the late 16th century, long after Leonardo']                                                                                                                                                    |
     -----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     """

sparknlp/annotator/seq2seq/nllb_transformer.py

@@ -77,7 +77,7 @@ class NLLBTransformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
         Target Language (Default: `fr`)
 
     Languages Covered
-    -----
+    -----------------
     Acehnese (Arabic script) (ace_Arab), Acehnese (Latin script) (ace_Latn), Mesopotamian Arabic
     (acm_Arab), Ta’izzi-Adeni Arabic (acq_Arab), Tunisian Arabic (aeb_Arab), Afrikaans (afr_Latn),
     South Levantine Arabic (ajp_Arab), Akan (aka_Latn), Amharic (amh_Ethi), North Levantine Arabic

sparknlp/annotator/seq2seq/qwen_transformer.py

@@ -52,6 +52,32 @@ class QwenTransformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
     ``DOCUMENT``           ``DOCUMENT``
     ====================== ======================
 
+    **References**
+    
+    - `Qwen Technical Report
+      <https://arxiv.org/pdf/2309.16609.pdf>`__
+    - https://qwenlm.github.io/blog/qwen1.5/
+    - https://github.com/QwenLM/Qwen1.5
+
+    **Paper Abstract:**
+
+    *Large language models (LLMs) have revolutionized the field of artificial intelligence,
+    enabling natural language processing tasks that were previously thought to be exclusive to
+    humans. In this work, we introduce Qwen, the first installment of our large language model
+    series. Qwen is a comprehensive language model series that encompasses distinct models with
+    varying parameter counts. It includes Qwen, the base pretrained language models, and
+    Qwen-Chat, the chat models finetuned with human alignment techniques. The base language models
+    consistently demonstrate superior performance across a multitude of downstream tasks, and the
+    chat models, particularly those trained using Reinforcement Learning from Human Feedback
+    (RLHF), are highly competitive. The chat models possess advanced tool-use and planning
+    capabilities for creating agent applications, showcasing impressive performance even when
+    compared to bigger models on complex tasks like utilizing a code interpreter. Furthermore, we
+    have developed coding-specialized models, Code-Qwen and Code-Qwen-Chat, as well as
+    mathematics-focused models, Math-Qwen-Chat, which are built upon base language models. These
+    models demonstrate significantly improved performance in comparison with open-source models,
+    and slightly fall behind the proprietary models.*
+
+
     Parameters
     ----------
     configProtoBytes
@@ -87,31 +113,6 @@ class QwenTransformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
     This is a very computationally expensive module especially on larger
     sequence. The use of an accelerator such as GPU is recommended.
 
-    References
-    ----------
-    - `Qwen Technical Report
-      <https://arxiv.org/pdf/2309.16609.pdf>`__
-    - https://qwenlm.github.io/blog/qwen1.5/
-    - https://github.com/QwenLM/Qwen1.5
-
-    **Paper Abstract:**
-
-    *Large language models (LLMs) have revolutionized the field of artificial intelligence,
-    enabling natural language processing tasks that were previously thought to be exclusive to
-    humans. In this work, we introduce Qwen, the first installment of our large language model
-    series. Qwen is a comprehensive language model series that encompasses distinct models with
-    varying parameter counts. It includes Qwen, the base pretrained language models, and
-    Qwen-Chat, the chat models finetuned with human alignment techniques. The base language models
-    consistently demonstrate superior performance across a multitude of downstream tasks, and the
-    chat models, particularly those trained using Reinforcement Learning from Human Feedback
-    (RLHF), are highly competitive. The chat models possess advanced tool-use and planning
-    capabilities for creating agent applications, showcasing impressive performance even when
-    compared to bigger models on complex tasks like utilizing a code interpreter. Furthermore, we
-    have developed coding-specialized models, Code-Qwen and Code-Qwen-Chat, as well as
-    mathematics-focused models, Math-Qwen-Chat, which are built upon base language models. These
-    models demonstrate significantly improved performance in comparison with open-source models,
-    and slightly fall behind the proprietary models.*
-
     Examples
     --------
     >>> import sparknlp

sparknlp/annotator/spell_check/context_spell_checker.py

@@ -565,7 +565,7 @@ class ContextSpellCheckerModel(AnnotatorModel, HasEngine):
 
 
     References
-    -------------
+    ----------
     For an in-depth explanation of the module see the article `Applying Context
     Aware Spell Checking in Spark NLP
     <https://medium.com/spark-nlp/applying-context-aware-spell-checking-in-spark-nlp-3c29c46963bc>`__.

sparknlp/base/prompt_assembler.py

@@ -122,7 +122,7 @@ class PromptAssembler(AnnotatorTransformer):
     +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     |result                                                                                                                                                                                                                                                                                                                      |
     +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |[<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nHello there, how can I help you?<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nI need help with organizing my room.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n]|
+    |[<|start_header_id|>system<|end_header_id|>\\n\\nYou are a helpful assistant.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\nHello there, how can I help you?<|eot_id|><|start_header_id|>user<|end_header_id|>\\n\\nI need help with organizing my room.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\n]|
     +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     """
 

sparknlp/common/properties.py

@@ -38,7 +38,7 @@ class HasBatchedAnnotate:
         int
             Current batch size
         """
-        return self.getOrDefault("batchSize")
+        return self.getOrDefault(self.batchSize)
 
 
 class HasCaseSensitiveProperties:
@@ -245,7 +245,7 @@ class HasBatchedAnnotateImage:
         int
             Current batch size
         """
-        return self.getOrDefault("batchSize")
+        return self.getOrDefault(self.batchSize)
 
 
 class HasImageFeatureProperties:
@@ -402,7 +402,7 @@ class HasBatchedAnnotateAudio:
         int
             Current batch size
         """
-        return self.getOrDefault("batchSize")
+        return self.getOrDefault(self.batchSize)
 
 
 class HasAudioFeatureProperties:
@@ -1099,7 +1099,7 @@ class HasLlamaCppProperties:
         return self._set(flashAttention=flashAttention)
 
     def setInputPrefixBos(self, inputPrefixBos: bool):
-        """Whether to add prefix BOS to user inputs, preceding the `--in-prefix` bool"""
+        """Whether to add prefix BOS to user inputs, preceding the `--in-prefix` string"""
         return self._set(inputPrefixBos=inputPrefixBos)
 
     def setUseMmap(self, useMmap: bool):
@@ -1114,7 +1114,7 @@ class HasLlamaCppProperties:
         """Whether to disable KV offload"""
         return self._set(noKvOffload=noKvOffload)
 
-    def setSystemPrompt(self, systemPrompt: bool):
+    def setSystemPrompt(self, systemPrompt: str):
         """Set a system prompt to use"""
         return self._set(systemPrompt=systemPrompt)
 
@@ -1219,7 +1219,7 @@ class HasLlamaCppProperties:
         """Set the amount of tokens the samplers should return at least (0 = disabled)"""
         return self._set(minKeep=minKeep)
 
-    def setGrammar(self, grammar: bool):
+    def setGrammar(self, grammar: str):
         """Set BNF-like grammar to constrain generations"""
         return self._set(grammar=grammar)
 
@@ -1261,7 +1261,7 @@ class HasLlamaCppProperties:
         return self._call_java("setTokenBias", tokenBias)
 
     def setLoraAdapters(self, loraAdapters: Dict[str, float]):
-        """Set token id bias"""
+        """Set LoRA adapters with their scaling factors"""
         return self._call_java("setLoraAdapters", loraAdapters)
 
     def getMetadata(self):

sparknlp/internal/__init__.py

@@ -281,6 +281,16 @@ class _Gemma3ForMultiModalLoader(ExtendedJavaWrapper):
             use_openvino
         )
 
+class _InternVLForMultiModalLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark, use_openvino=False):
+        super(_InternVLForMultiModalLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.cv.InternVLForMultiModal.loadSavedModel",
+            path,
+            jspark,
+            use_openvino
+        )
+
+
 class _JanusForMultiModalLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark, use_openvino=False):
         super(_JanusForMultiModalLoader, self).__init__(
@@ -1146,3 +1156,12 @@ class _SmolVLMTransformerLoader(ExtendedJavaWrapper):
             jspark,
             use_openvino
         )
+
+class _Florence2TransformerLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark, use_openvino=False):
+        super(_Florence2TransformerLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.cv.Florence2Transformer.loadSavedModel",
+            path,
+            jspark,
+            use_openvino,
+        )

sparknlp/partition/__init__.py

@@ -0,0 +1,16 @@
+#  Copyright 2017-2025 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Module to read various types of documents into chunks"""
+from sparknlp.partition.partition import *
+from sparknlp.partition.partition_transformer import *

sparknlp/partition/partition.py

@@ -0,0 +1,244 @@
+#  Copyright 2017-2025 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains the Partition annotator for reading and processing various document types."""
+import sparknlp
+from sparknlp.internal import ExtendedJavaWrapper
+
+
+class Partition(ExtendedJavaWrapper):
+    """
+    A unified interface for extracting structured content from various document types
+    using Spark NLP readers.
+
+    This class supports reading from files, URLs, in-memory strings, or byte arrays,
+    and returns parsed output as a structured Spark DataFrame.
+
+    Supported formats include:
+    - Plain text
+    - HTML
+    - Word (.doc/.docx)
+    - Excel (.xls/.xlsx)
+    - PowerPoint (.ppt/.pptx)
+    - Email files (.eml, .msg)
+    - PDFs
+
+    Parameters
+    ----------
+    params : dict, optional
+        Configuration parameters, including:
+
+        - content_type : str
+            Override automatic file type detection.
+        - store_content : bool
+            Include raw file content in the output DataFrame.
+        - timeout : int
+            Timeout for fetching HTML content.
+        - title_font_size : int
+            Font size used to identify titles.
+        - include_page_breaks : bool
+            Tag content with page break metadata.
+        - group_broken_paragraphs : bool
+            Merge broken lines into full paragraphs.
+        - title_length_size : int
+            Max character length to qualify as title.
+        - paragraph_split : str
+            Regex to detect paragraph boundaries.
+        - short_line_word_threshold : int
+            Max words in a line to be considered short.
+        - threshold : float
+            Ratio of empty lines for switching grouping.
+        - max_line_count : int
+            Max lines evaluated in paragraph analysis.
+        - include_slide_notes : bool
+            Include speaker notes in output.
+        - infer_table_structure : bool
+            Generate HTML table structure.
+        - append_cells : bool
+            Merge Excel rows into one block.
+        - cell_separator : str
+            Join cell values in a row.
+        - add_attachment_content : bool
+            Include text of plain-text attachments.
+        - headers : dict
+            Request headers when using URLs.
+
+    Examples
+    --------
+
+    Reading Text Files
+
+    >>> txt_directory = "/content/txtfiles/reader/txt"
+    >>> partition_df = Partition(content_type="text/plain").partition(txt_directory)
+    >>> partition_df.show()
+    >>> partition_df = Partition().partition("./email-files/test-several-attachments.eml")
+    >>> partition_df.show()
+    >>> partition_df = Partition().partition(
+    ...     "https://www.wikipedia.com",
+    ...     headers={"Accept-Language": "es-ES"}
+    ... )
+    >>> partition_df.show()
+    +--------------------+--------------------+
+    |                path|                 txt|
+    +--------------------+--------------------+
+    |file:/content/txt...|[{Title, BIG DATA...|
+    +--------------------+--------------------+
+
+    Reading Email Files
+
+    >>> partition_df = Partition().partition("./email-files/test-several-attachments.eml")
+    >>> partition_df.show()
+    +--------------------+--------------------+
+    |                path|               email|
+    +--------------------+--------------------+
+    |file:/content/ema...|[{Title, Test Sev...|
+    +--------------------+--------------------+
+
+    Reading Webpages
+
+    >>> partition_df = Partition().partition("https://www.wikipedia.com", headers = {"Accept-Language": "es-ES"})
+    >>> partition_df.show()
+    +--------------------+--------------------+
+    |                 url|                html|
+    +--------------------+--------------------+
+    |https://www.wikip...|[{Title, Wikipedi...|
+    +--------------------+--------------------+
+
+    For more examples, refer to:
+    `examples/python/data-preprocessing/SparkNLP_Partition_Reader_Demo.ipynb`
+    """
+    def  __init__(self, **kwargs):
+        self.spark = sparknlp.start()
+        params = {}
+        for key, value in kwargs.items():
+            try:
+                params[key] = str(value)
+            except Exception as e:
+                raise ValueError(f"Invalid value for key '{key}': Cannot cast {type(value)} to string. Original error: {e}")
+
+        super(Partition, self).__init__("com.johnsnowlabs.partition.Partition", params)
+
+
+    def partition(self, path, headers=None):
+        """
+        Reads and parses content from a URL, file, or directory path.
+
+        Parameters
+        ----------
+        path : str
+            Path to file or directory. URLs and DFS are supported.
+        headers : dict, optional
+            Headers for URL requests.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            DataFrame with parsed content.
+    """
+        if headers is None:
+            headers = {}
+        jdf = self._java_obj.partition(path, headers)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe
+
+
+    def partition_urls(self, path, headers=None):
+        """
+        Reads and parses content from multiple URLs.
+
+        Parameters
+        ----------
+        path : list[str]
+            List of URLs.
+        headers : dict, optional
+            Request headers for URLs.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            DataFrame with parsed URL content.
+
+        Examples
+        --------
+        >>> urls_df = Partition().partition_urls([
+        ...     "https://www.wikipedia.org", "https://example.com/"
+        ... ])
+        >>> urls_df.show()
+        +--------------------+--------------------+
+        |                 url|                html|
+        +--------------------+--------------------+
+        |https://www.wikip...|[{Title, Wikipedi...|
+        |https://example.com/|[{Title, Example ...|
+        +--------------------+--------------------+
+
+        >>> urls_df.printSchema()
+         root
+         |-- url: string (nullable = true)
+         |-- html: array (nullable = true)
+         |    |-- element: struct (containsNull = true)
+         |    |    |-- elementType: string (nullable = true)
+         |    |    |-- content: string (nullable = true)
+         |    |    |-- metadata: map (nullable = true)
+         |    |    |    |-- key: string
+         |    |    |    |-- value: string (valueContainsNull = true)
+        """
+        if headers is None:
+            headers = {}
+        jdf = self._java_obj.partitionUrlsJava(path, headers)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe
+
+
+    def partition_text(self, text):
+        """
+        Parses content from a raw text string.
+
+        Parameters
+        ----------
+        text : str
+            Raw text input.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            DataFrame with parsed text.
+
+        Examples
+        --------
+        >>> raw_text = (
+        ...     "The big brown fox\\n"
+        ...     "was walking down the lane.\\n"
+        ...     "\\n"
+        ...     "At the end of the lane,\\n"
+        ...     "the fox met a bear."
+        ... )
+        >>> text_df = Partition(group_broken_paragraphs=True).partition_text(text=raw_text)
+        >>> text_df.show()
+        +--------------------------------------+
+        |txt                                   |
+        +--------------------------------------+
+        |[{NarrativeText, The big brown fox was|
+        +--------------------------------------+
+        >>> text_df.printSchema()
+        root
+         |-- txt: array (nullable = true)
+         |    |-- element: struct (containsNull = true)
+         |    |    |-- elementType: string (nullable = true)
+         |    |    |-- content: string (nullable = true)
+         |    |    |-- metadata: map (nullable = true)
+         |    |    |    |-- key: string
+         |    |    |    |-- value: string (valueContainsNull = true)
+        """
+        jdf = self._java_obj.partitionText(text)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe