spark-nlp 5.5.0rc1__py2.py3-none-any.whl → 5.5.2__py2.py3-none-any.whl

sparknlp/annotator/seq2seq/nllb_transformer.py

@@ -32,7 +32,7 @@ class NLLBTransformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
     ...     .setOutputCol("generation")
 
 
-    The default model is ``"nllb_418M"``, if no name is provided. For available
+    The default model is ``"nllb_distilled_600M_8int"``, if no name is provided. For available
     pretrained models please see the `Models Hub
     <https://sparknlp.org/models?q=nllb>`__.
 
@@ -164,7 +164,7 @@ class NLLBTransformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
     >>> documentAssembler = DocumentAssembler() \\
     ...     .setInputCol("text") \\
     ...     .setOutputCol("documents")
-    >>> nllb = NLLBTransformer.pretrained("nllb_418M") \\
+    >>> nllb = NLLBTransformer.pretrained("nllb_distilled_600M_8int") \\
     ...     .setInputCols(["documents"]) \\
     ...     .setMaxOutputLength(50) \\
     ...     .setOutputCol("generation") \\
@@ -398,13 +398,13 @@ class NLLBTransformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
         return NLLBTransformer(java_model=jModel)
 
     @staticmethod
-    def pretrained(name="nllb_418M", lang="xx", remote_loc=None):
+    def pretrained(name="nllb_distilled_600M_8int", lang="xx", remote_loc=None):
         """Downloads and loads a pretrained model.
 
         Parameters
         ----------
         name : str, optional
-            Name of the pretrained model, by default "nllb_418M"
+            Name of the pretrained model, by default "nllb_distilled_600M_8int"
         lang : str, optional
             Language of the pretrained model, by default "en"
         remote_loc : str, optional

sparknlp/annotator/seq2seq/phi3_transformer.py

@@ -37,7 +37,7 @@ class Phi3Transformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
     ...     .setOutputCol("generation")
 
 
-    The default model is ``"phi3"``, if no name is provided. For available
+    The default model is ``phi_3_mini_128k_instruct``, if no name is provided. For available
     pretrained models please see the `Models Hub
     <https://sparknlp.org/models?q=phi3>`__.
 
@@ -112,7 +112,7 @@ class Phi3Transformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
     >>> documentAssembler = DocumentAssembler() \\
     ...     .setInputCol("text") \\
     ...     .setOutputCol("documents")
-    >>> phi3 = Phi3Transformer.pretrained("phi3") \\
+    >>> phi3 = Phi3Transformer.pretrained(phi_3_mini_128k_instruct) \\
     ...     .setInputCols(["documents"]) \\
     ...     .setMaxOutputLength(50) \\
     ...     .setOutputCol("generation")
@@ -308,13 +308,13 @@ class Phi3Transformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
         return Phi3Transformer(java_model=jModel)
 
     @staticmethod
-    def pretrained(name="phi3", lang="en", remote_loc=None):
+    def pretrained(name="phi_3_mini_128k_instruct", lang="en", remote_loc=None):
         """Downloads and loads a pretrained model.
 
         Parameters
         ----------
         name : str, optional
-            Name of the pretrained model, by default "phi3"
+            Name of the pretrained model, by default phi_3_mini_128k_instruct
         lang : str, optional
             Language of the pretrained model, by default "en"
         remote_loc : str, optional

sparknlp/annotator/seq2seq/qwen_transformer.py

@@ -121,7 +121,7 @@ class QwenTransformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
     >>> documentAssembler = DocumentAssembler() \\
     ...     .setInputCol("text") \\
     ...     .setOutputCol("documents")
-    >>> qwen = QwenTransformer.pretrained("qwen-7b") \\
+    >>> qwen = QwenTransformer.pretrained("qwen_7.5b_chat") \\
     ...     .setInputCols(["documents"]) \\
     ...     .setMaxOutputLength(50) \\
     ...     .setOutputCol("generation")
@@ -317,13 +317,13 @@ class QwenTransformer(AnnotatorModel, HasBatchedAnnotate, HasEngine):
         return QwenTransformer(java_model=jModel)
 
     @staticmethod
-    def pretrained(name="qwen-7b", lang="en", remote_loc=None):
+    def pretrained(name="qwen_7.5b_chat", lang="en", remote_loc=None):
         """Downloads and loads a pretrained model.
 
         Parameters
         ----------
         name : str, optional
-            Name of the pretrained model, by default "qwen-7b"
+            Name of the pretrained model, by default "qwen_7.5b_chat"
         lang : str, optional
             Language of the pretrained model, by default "en"
         remote_loc : str, optional

sparknlp/base/__init__.py

@@ -26,3 +26,4 @@ from sparknlp.base.token_assembler import *
 from sparknlp.base.image_assembler import *
 from sparknlp.base.audio_assembler import *
 from sparknlp.base.table_assembler import *
+from sparknlp.base.prompt_assembler import *

sparknlp/base/image_assembler.py

@@ -65,6 +65,7 @@ class ImageAssembler(AnnotatorTransformer):
     outputAnnotatorType = AnnotatorType.IMAGE
 
     inputCol = Param(Params._dummy(), "inputCol", "input column name", typeConverter=TypeConverters.toString)
+    textCol = Param(Params._dummy(), "textCol", "text column name", typeConverter=TypeConverters.toString)
     outputCol = Param(Params._dummy(), "outputCol", "output column name", typeConverter=TypeConverters.toString)
     name = 'ImageAssembler'
 
@@ -101,3 +102,13 @@ class ImageAssembler(AnnotatorTransformer):
     def getOutputCol(self):
         """Gets output column name of annotations."""
         return self.getOrDefault(self.outputCol)
+
+    def setTextCol(self, value):
+        """Sets an optional text column name.
+
+        Parameters
+        ----------
+        value : str
+            Name of an optional input text column
+        """
+        return self._set(inputCol=value)

sparknlp/base/light_pipeline.py

@@ -277,7 +277,7 @@ class LightPipeline:
 
         return result
 
-    def fullAnnotateImage(self, path_to_image):
+    def fullAnnotateImage(self, path_to_image, text=None):
         """Annotates the data provided into `Annotation` type results.
 
         The data should be either a list or a str.
@@ -287,27 +287,38 @@ class LightPipeline:
         path_to_image : list or str
             Source path of image, list of paths to images
 
+        text: list or str, optional
+           Optional list or str of texts. If None, defaults to empty list if path_to_image is a list, or empty string if path_to_image is a string.
+
         Returns
         -------
         List[AnnotationImage]
             The result of the annotation
         """
+        if not isinstance(path_to_image, (str, list)):
+            raise TypeError("argument for path_to_image must be 'str' or 'list[str]'")
+
+        if text is None:
+            text = "" if isinstance(path_to_image, str) else []
+
+        if type(path_to_image) != type(text):
+            raise ValueError("`path_to_image` and `text` must be of the same type")
+
         stages = self.pipeline_model.stages
         if not self._skipPipelineValidation(stages):
             self._validateStagesInputCols(stages)
 
-        if type(path_to_image) is str:
+        if isinstance(path_to_image, str):
             path_to_image = [path_to_image]
+            text = [text]
 
-        if type(path_to_image) is list:
-            result = []
+        result = []
 
-            for image_result in self._lightPipeline.fullAnnotateImageJava(path_to_image):
-                result.append(self.__buildStages(image_result))
+        for image_result in self._lightPipeline.fullAnnotateImageJava(path_to_image, text):
+            result.append(self.__buildStages(image_result))
+
+        return result
 
-            return result
-        else:
-            raise TypeError("argument for annotation may be 'str' or list[str]")
 
     def __buildStages(self, annotations_result):
         stages = {}

sparknlp/base/prompt_assembler.py

@@ -0,0 +1,207 @@
+#  Copyright 2017-2024 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the PromptAssembler."""
+
+from pyspark import keyword_only
+from pyspark.ml.param import TypeConverters, Params, Param
+
+from sparknlp.common import AnnotatorType
+from sparknlp.internal import AnnotatorTransformer
+
+
+class PromptAssembler(AnnotatorTransformer):
+    """Assembles a sequence of messages into a single string using a template. These strings can then
+    be used as prompts for large language models.
+
+    This annotator expects an array of two-tuples as the type of the input column (one array of
+    tuples per row). The first element of the tuples should be the role and the second element is
+    the text of the message. Possible roles are "system", "user" and "assistant".
+
+    An assistant header can be added to the end of the generated string by using
+    ``setAddAssistant(True)``.
+
+    At the moment, this annotator uses llama.cpp as a backend to parse and apply the templates.
+    llama.cpp uses basic pattern matching to determine the type of the template, then applies a
+    basic version of the template to the messages. This means that more advanced templates are not
+    supported.
+
+    For an extended example see the
+    `example notebook <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/llama.cpp/PromptAssember_with_AutoGGUFModel.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``NONE``               ``DOCUMENT``
+    ====================== ======================
+
+    Parameters
+    ----------
+    inputCol
+        Input column name
+    outputCol
+        Output column name
+    chatTemplate
+        Template used for the chat
+    addAssistant
+        Whether to add an assistant header to the end of the generated string
+
+    Examples
+    --------
+    >>> from sparknlp.base import *
+    >>> messages = [
+    ...     [
+    ...         ("system", "You are a helpful assistant."),
+    ...         ("assistant", "Hello there, how can I help you?"),
+    ...         ("user", "I need help with organizing my room."),
+    ...     ]
+    ... ]
+    >>> df = spark.createDataFrame([messages]).toDF("messages")
+    >>> template = (
+    ...     "{{- bos_token }} {%- if custom_tools is defined %} {%- set tools = custom_tools %} {%- "
+    ...     "endif %} {%- if not tools_in_user_message is defined %} {%- set tools_in_user_message = true %} {%- "
+    ...     'endif %} {%- if not date_string is defined %} {%- set date_string = "26 Jul 2024" %} {%- endif %} '
+    ...     "{%- if not tools is defined %} {%- set tools = none %} {%- endif %} {#- This block extracts the "
+    ...     "system message, so we can slot it into the right place. #} {%- if messages[0]['role'] == 'system' %}"
+    ...     " {%- set system_message = messages[0]['content']|trim %} {%- set messages = messages[1:] %} {%- else"
+    ...     ' %} {%- set system_message = "" %} {%- endif %} {#- System message + builtin tools #} {{- '
+    ...     '"<|start_header_id|>system<|end_header_id|>\\n\\n" }} {%- if builtin_tools is defined or tools is '
+    ...     'not none %} {{- "Environment: ipython\\n" }} {%- endif %} {%- if builtin_tools is defined %} {{- '
+    ...     '"Tools: " + builtin_tools | reject(\\'equalto\', \\'code_interpreter\\') | join(", ") + "\\n\\n"}} '
+    ...     '{%- endif %} {{- "Cutting Knowledge Date: December 2023\\n" }} {{- "Today Date: " + date_string '
+    ...     '+ "\\n\\n" }} {%- if tools is not none and not tools_in_user_message %} {{- "You have access to '
+    ...     'the following functions. To call a function, please respond with JSON for a function call." }} {{- '
+    ...     '\\'Respond in the format {"name": function name, "parameters": dictionary of argument name and its'
+    ...     ' value}.\\' }} {{- "Do not use variables.\\n\\n" }} {%- for t in tools %} {{- t | tojson(indent=4) '
+    ...     '}} {{- "\\n\\n" }} {%- endfor %} {%- endif %} {{- system_message }} {{- "<|eot_id|>" }} {#- '
+    ...     "Custom tools are passed in a user message with some extra guidance #} {%- if tools_in_user_message "
+    ...     "and not tools is none %} {#- Extract the first user message so we can plug it in here #} {%- if "
+    ...     "messages | length != 0 %} {%- set first_user_message = messages[0]['content']|trim %} {%- set "
+    ...     'messages = messages[1:] %} {%- else %} {{- raise_exception("Cannot put tools in the first user '
+    ...     "message when there's no first user message!\\") }} {%- endif %} {{- "
+    ...     "'<|start_header_id|>user<|end_header_id|>\\n\\n' -}} {{- \\"Given the following functions, please "
+    ...     'respond with a JSON for a function call " }} {{- "with its proper arguments that best answers the '
+    ...     'given prompt.\\n\\n" }} {{- \\'Respond in the format {"name": function name, "parameters": '
+    ...     'dictionary of argument name and its value}.\\' }} {{- "Do not use variables.\\n\\n" }} {%- for t in '
+    ...     'tools %} {{- t | tojson(indent=4) }} {{- "\\n\\n" }} {%- endfor %} {{- first_user_message + '
+    ...     "\\"<|eot_id|>\\"}} {%- endif %} {%- for message in messages %} {%- if not (message.role == 'ipython' "
+    ...     "or message.role == 'tool' or 'tool_calls' in message) %} {{- '<|start_header_id|>' + message['role']"
+    ...     " + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }} {%- elif 'tool_calls' in "
+    ...     'message %} {%- if not message.tool_calls|length == 1 %} {{- raise_exception("This model only '
+    ...     'supports single tool-calls at once!") }} {%- endif %} {%- set tool_call = message.tool_calls[0]'
+    ...     ".function %} {%- if builtin_tools is defined and tool_call.name in builtin_tools %} {{- "
+    ...     "'<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}} {{- \\"<|python_tag|>\\" + tool_call.name + "
+    ...     '".call(" }} {%- for arg_name, arg_val in tool_call.arguments | items %} {{- arg_name + \\'="\\' + '
+    ...     'arg_val + \\'"\\' }} {%- if not loop.last %} {{- ", " }} {%- endif %} {%- endfor %} {{- ")" }} {%- '
+    ...     "else %} {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}} {{- '{\\"name\": \\"' + "
+    ...     'tool_call.name + \\'", \\' }} {{- \\'"parameters": \\' }} {{- tool_call.arguments | tojson }} {{- "}" '
+    ...     "}} {%- endif %} {%- if builtin_tools is defined %} {#- This means we're in ipython mode #} {{- "
+    ...     '"<|eom_id|>" }} {%- else %} {{- "<|eot_id|>" }} {%- endif %} {%- elif message.role == "tool" '
+    ...     'or message.role == "ipython" %} {{- "<|start_header_id|>ipython<|end_header_id|>\\n\\n" }} {%- '
+    ...     "if message.content is mapping or message.content is iterable %} {{- message.content | tojson }} {%- "
+    ...     'else %} {{- message.content }} {%- endif %} {{- "<|eot_id|>" }} {%- endif %} {%- endfor %} {%- if '
+    ...     "add_generation_prompt %} {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }} {%- endif %} "
+    ... )
+    >>> prompt_assembler = (
+    ...     PromptAssembler()
+    ...     .setInputCol("messages")
+    ...     .setOutputCol("prompt")
+    ...     .setChatTemplate(template)
+    ... )
+    >>> prompt_assembler.transform(df).select("prompt.result").show(truncate=False)
+    +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |result                                                                                                                                                                                                                                                                                                                      |
+    +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |[<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nHello there, how can I help you?<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nI need help with organizing my room.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n]|
+    +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    """
+
+    outputAnnotatorType = AnnotatorType.DOCUMENT
+
+    inputCol = Param(
+        Params._dummy(),
+        "inputCol",
+        "input column name",
+        typeConverter=TypeConverters.toString,
+    )
+    outputCol = Param(
+        Params._dummy(),
+        "outputCol",
+        "output column name",
+        typeConverter=TypeConverters.toString,
+    )
+    chatTemplate = Param(
+        Params._dummy(),
+        "chatTemplate",
+        "Template used for the chat",
+        typeConverter=TypeConverters.toString,
+    )
+    addAssistant = Param(
+        Params._dummy(),
+        "addAssistant",
+        "Whether to add an assistant header to the end of the generated string",
+        typeConverter=TypeConverters.toBoolean,
+    )
+    name = "PromptAssembler"
+
+    @keyword_only
+    def __init__(self):
+        super(PromptAssembler, self).__init__(
+            classname="com.johnsnowlabs.nlp.PromptAssembler"
+        )
+        self._setDefault(outputCol="prompt", addAssistant=True)
+
+    @keyword_only
+    def setParams(self):
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    def setInputCol(self, value):
+        """Sets input column name.
+
+        Parameters
+        ----------
+        value : str
+            Name of the input column
+        """
+        return self._set(inputCol=value)
+
+    def setOutputCol(self, value):
+        """Sets output column name.
+
+        Parameters
+        ----------
+        value : str
+            Name of the Output Column
+        """
+        return self._set(outputCol=value)
+
+    def setChatTemplate(self, value):
+        """Sets the chat template.
+
+        Parameters
+        ----------
+        value : str
+            Template used for the chat
+        """
+        return self._set(chatTemplate=value)
+
+    def setAddAssistant(self, value):
+        """Sets whether to add an assistant header to the end of the generated string.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to add an assistant header to the end of the generated string
+        """
+        return self._set(addAssistant=value)

sparknlp/internal/__init__.py

@@ -113,6 +113,13 @@ class _BertQuestionAnsweringLoader(ExtendedJavaWrapper):
             jspark,
         )
 
+class _BertMultipleChoiceLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark):
+        super(_BertMultipleChoiceLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.classifier.dl.BertForMultipleChoice.loadSavedModel",
+            path,
+            jspark,
+        )
 
 class _DeBERTaLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark):
@@ -999,3 +1006,18 @@ class _SnowFlakeEmbeddingsLoader(ExtendedJavaWrapper):
         super(_SnowFlakeEmbeddingsLoader, self).__init__(
             "com.johnsnowlabs.nlp.embeddings.SnowFlakeEmbeddings.loadSavedModel", path, jspark
         )
+
+
+class _AutoGGUFEmbeddingsLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark):
+        super(_AutoGGUFEmbeddingsLoader, self).__init__(
+            "com.johnsnowlabs.nlp.embeddings.AutoGGUFEmbeddings.loadSavedModel", path, jspark)
+
+
+class _BLIPForQuestionAnswering(ExtendedJavaWrapper):
+    def __init__(self, path, jspark):
+        super(_BLIPForQuestionAnswering, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.cv.BLIPForQuestionAnswering.loadSavedModel",
+            path,
+            jspark,
+        )

sparknlp/reader/__init__.py

@@ -0,0 +1,15 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Module for reading different files types."""
+from sparknlp.reader.sparknlp_reader import *

sparknlp/reader/sparknlp_reader.py

@@ -0,0 +1,121 @@
+#  Copyright 2017-2024 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from sparknlp.internal import ExtendedJavaWrapper
+
+
+class SparkNLPReader(ExtendedJavaWrapper):
+    """Instantiates class to read HTML files.
+
+    Two types of input paths are supported,
+
+    htmlPath: this is a path to a directory of HTML files or a path to an HTML file
+    E.g. "path/html/files"
+
+    url: this is the URL or set of URLs of a website  . E.g., "https://www.wikipedia.org"
+
+    Parameters
+    ----------
+    params : spark
+        Spark session
+    params : dict, optional
+        Parameter with custom configuration
+
+    Examples
+    --------
+    >>> from sparknlp.reader import SparkNLPReader
+    >>> html_df = SparkNLPReader().html(spark, "https://www.wikipedia.org")
+
+    You can use SparkNLP for one line of code
+    >>> import sparknlp
+    >>> html_df = sparknlp.read().html("https://www.wikipedia.org")
+    >>> html_df.show(truncate=False)
+
+    +--------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |url                 |html                                                                                                                                                                                                                                                                                                                            |
+    +--------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |https://example.com/|[{Title, Example Domain, {pageNumber -> 1}}, {NarrativeText, 0, This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission., {pageNumber -> 1}}, {NarrativeText, 0, More information... More information..., {pageNumber -> 1}}]   |
+    +--------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    >>> html_df.printSchema()
+
+    root
+     |-- url: string (nullable = true)
+     |-- html: array (nullable = true)
+     |    |-- element: struct (containsNull = true)
+     |    |    |-- elementType: string (nullable = true)
+     |    |    |-- content: string (nullable = true)
+     |    |    |-- metadata: map (nullable = true)
+     |    |    |    |-- key: string
+     |    |    |    |-- value: string (valueContainsNull = true)
+
+
+
+    Instantiates class to read email files.
+
+    emailPath: this is a path to a directory of HTML files or a path to an HTML file E.g.
+    "path/html/emails"
+
+    Examples
+    --------
+    >>> from sparknlp.reader import SparkNLPReader
+    >>> email_df = SparkNLPReader().email(spark, "home/user/emails-directory")
+
+    You can use SparkNLP for one line of code
+    >>> import sparknlp
+    >>> email_df = sparknlp.read().email("home/user/emails-directory")
+    >>> email_df.show(truncate=False)
+    +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |email                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+    +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |[{Title, Email Text Attachments, {sent_to -> Danilo Burbano <danilo@johnsnowlabs.com>, sent_from -> Danilo Burbano <danilo@johnsnowlabs.com>}}, {NarrativeText, Email  test with two text attachments\r\n\r\nCheers,\r\n\r\n, {sent_to -> Danilo Burbano <danilo@johnsnowlabs.com>, sent_from -> Danilo Burbano <danilo@johnsnowlabs.com>, mimeType -> text/plain}}, {NarrativeText, <html>\r\n<head>\r\n<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">\r\n<style type="text/css" style="display:none;"> P {margin-top:0;margin-bottom:0;} </style>\r\n</head>\r\n<body dir="ltr">\r\n<span style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">Email&nbsp; test with two text attachments</span>\r\n<div class="elementToProof" style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">\r\n<br>\r\n</div>\r\n<div class="elementToProof" style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">\r\nCheers,</div>\r\n<div class="elementToProof" style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">\r\n<br>\r\n</div>\r\n</body>\r\n</html>\r\n, {sent_to -> Danilo Burbano <danilo@johnsnowlabs.com>, sent_from -> Danilo Burbano <danilo@johnsnowlabs.com>, mimeType -> text/html}}, {Attachment, filename.txt, {sent_to -> Danilo Burbano <danilo@johnsnowlabs.com>, sent_from -> Danilo Burbano <danilo@johnsnowlabs.com>, contentType -> text/plain; name="filename.txt"}}, {NarrativeText, This is the content of the file.\n, {sent_to -> Danilo Burbano <danilo@johnsnowlabs.com>, sent_from -> Danilo Burbano <danilo@johnsnowlabs.com>, mimeType -> text/plain}}, {Attachment, filename2.txt, {sent_to -> Danilo Burbano <danilo@johnsnowlabs.com>, sent_from -> Danilo Burbano <danilo@johnsnowlabs.com>, contentType -> text/plain; name="filename2.txt"}}, {NarrativeText, This is an additional content file.\n, {sent_to -> Danilo Burbano <danilo@johnsnowlabs.com>, sent_from -> Danilo Burbano <danilo@johnsnowlabs.com>, mimeType -> text/plain}}]|
+    +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    email_df.printSchema()
+    root
+     |-- path: string (nullable = true)
+     |-- content: array (nullable = true)
+     |-- email: array (nullable = true)
+     |    |-- element: struct (containsNull = true)
+     |    |    |-- elementType: string (nullable = true)
+     |    |    |-- content: string (nullable = true)
+     |    |    |-- metadata: map (nullable = true)
+     |    |    |    |-- key: string
+     |    |    |    |-- value: string (valueContainsNull = true)
+
+    """
+
+    def __init__(self, spark, params=None):
+        if params is None:
+            params = {}
+        super(SparkNLPReader, self).__init__("com.johnsnowlabs.reader.SparkNLPReader", params)
+        self.spark = spark
+
+    def html(self, htmlPath):
+        if not isinstance(htmlPath, (str, list)) or (isinstance(htmlPath, list) and not all(isinstance(item, str) for item in htmlPath)):
+            raise TypeError("htmlPath must be a string or a list of strings")
+        jdf = self._java_obj.html(htmlPath)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe
+
+    def email(self, filePath):
+        if not isinstance(filePath, str):
+            raise TypeError("filePath must be a string")
+        jdf = self._java_obj.email(filePath)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe
+
+    def doc(self, docPath):
+        if not isinstance(docPath, str):
+            raise TypeError("docPath must be a string")
+        jdf = self._java_obj.doc(docPath)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe