spark-nlp 4.2.6__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/base/light_pipeline.py

@@ -12,14 +12,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """Contains classes for the LightPipeline."""
-from sparknlp.base.multi_document_assembler import MultiDocumentAssembler
 
 import sparknlp.internal as _internal
-
 from sparknlp.annotation import Annotation
 from sparknlp.annotation_audio import AnnotationAudio
 from sparknlp.annotation_image import AnnotationImage
 from sparknlp.common import AnnotatorApproach, AnnotatorModel
+from sparknlp.internal import AnnotatorTransformer
 
 
 class LightPipeline:
@@ -66,17 +65,17 @@ class LightPipeline:
 
     def __init__(self, pipelineModel, parse_embeddings=False):
         self.pipeline_model = pipelineModel
+        self.parse_embeddings = parse_embeddings
         self._lightPipeline = _internal._LightPipeline(pipelineModel, parse_embeddings).apply()
 
-    def _validateStagesInputCols(self):
-        stages = self.pipeline_model.stages
+    def _validateStagesInputCols(self, stages):
         annotator_types = self._getAnnotatorTypes(stages)
         for stage in stages:
             if isinstance(stage, AnnotatorApproach) or isinstance(stage, AnnotatorModel):
                 input_cols = stage.getInputCols()
                 if type(input_cols) == str:
                     input_cols = [input_cols]
-                input_annotator_types = stage.inputAnnotatorTypes
+                input_annotator_types = stage.inputAnnotatorTypes + stage.optionalInputAnnotatorTypes
                 for input_col in input_cols:
                     annotator_type = annotator_types.get(input_col)
                     if annotator_type is None or annotator_type not in input_annotator_types:
@@ -85,15 +84,27 @@ class LightPipeline:
                                         f" with the right output names and that they have following annotator types:"
                                         f" {input_annotator_types}")
 
+    def _skipPipelineValidation(self, stages):
+        exceptional_pipeline = [stage for stage in stages if self._skipStageValidation(stage)]
+        if len(exceptional_pipeline) >= 1:
+            return True
+        else:
+            return False
+
+    def _skipStageValidation(self, stage):
+        return hasattr(stage, 'skipLPInputColsValidation') and stage.skipLPInputColsValidation
+
     def _getAnnotatorTypes(self, stages):
         annotator_types = {}
         for stage in stages:
-            if isinstance(stage, MultiDocumentAssembler):
+            if hasattr(stage, 'getOutputCols'):
                 output_cols = stage.getOutputCols()
                 for output_col in output_cols:
                     annotator_types[output_col] = stage.outputAnnotatorType
-            else:
-                annotator_types[stage.getOutputCol()] = stage.outputAnnotatorType
+            elif isinstance(stage, AnnotatorApproach) or isinstance(stage, AnnotatorModel) or\
+                    isinstance(stage, AnnotatorTransformer):
+                if stage.outputAnnotatorType is not None:
+                    annotator_types[stage.getOutputCol()] = stage.outputAnnotatorType
         return annotator_types
 
     def _annotationFromJava(self, java_annotations):
@@ -123,13 +134,17 @@ class LightPipeline:
                                     annotation.metadata())
                 )
             else:
+                if self.parse_embeddings:
+                    embeddings = list(annotation.embeddings())
+                else:
+                    embeddings = []
                 annotations.append(
                     Annotation(annotation.annotatorType(),
                                annotation.begin(),
                                annotation.end(),
                                annotation.result(),
                                annotation.metadata(),
-                               [])
+                               embeddings)
                 )
         return annotations
 
@@ -176,7 +191,9 @@ class LightPipeline:
         Annotation(named_entity, 30, 36, B-LOC, {'word': 'Baghdad'}),
         Annotation(named_entity, 37, 37, O, {'word': '.'})]
         """
-        self._validateStagesInputCols()
+        stages = self.pipeline_model.stages
+        if not self._skipPipelineValidation(stages):
+            self._validateStagesInputCols(stages)
 
         if optional_target == "":
             if self.__isTextInput(target):
@@ -260,7 +277,7 @@ class LightPipeline:
 
         return result
 
-    def fullAnnotateImage(self, path_to_image):
+    def fullAnnotateImage(self, path_to_image, text=None):
         """Annotates the data provided into `Annotation` type results.
 
         The data should be either a list or a str.
@@ -270,25 +287,38 @@ class LightPipeline:
         path_to_image : list or str
             Source path of image, list of paths to images
 
+        text: list or str, optional
+           Optional list or str of texts. If None, defaults to empty list if path_to_image is a list, or empty string if path_to_image is a string.
+
         Returns
         -------
         List[AnnotationImage]
             The result of the annotation
         """
-        self._validateStagesInputCols()
+        if not isinstance(path_to_image, (str, list)):
+            raise TypeError("argument for path_to_image must be 'str' or 'list[str]'")
+
+        if text is None:
+            text = "" if isinstance(path_to_image, str) else []
 
-        if type(path_to_image) is str:
+        if type(path_to_image) != type(text):
+            raise ValueError("`path_to_image` and `text` must be of the same type")
+
+        stages = self.pipeline_model.stages
+        if not self._skipPipelineValidation(stages):
+            self._validateStagesInputCols(stages)
+
+        if isinstance(path_to_image, str):
             path_to_image = [path_to_image]
+            text = [text]
 
-        if type(path_to_image) is list:
-            result = []
+        result = []
 
-            for image_result in self._lightPipeline.fullAnnotateImageJava(path_to_image):
-                result.append(self.__buildStages(image_result))
+        for image_result in self._lightPipeline.fullAnnotateImageJava(path_to_image, text):
+            result.append(self.__buildStages(image_result))
+
+        return result
 
-            return result
-        else:
-            raise TypeError("argument for annotation may be 'str' or list[str]")
 
     def __buildStages(self, annotations_result):
         stages = {}
@@ -327,7 +357,9 @@ class LightPipeline:
         def reformat(annotations):
             return {k: list(v) for k, v in annotations.items()}
 
-        self._validateStagesInputCols()
+        stages = self.pipeline_model.stages
+        if not self._skipPipelineValidation(stages):
+            self._validateStagesInputCols(stages)
 
         if optional_target == "":
             if type(target) is str:

sparknlp/base/multi_document_assembler.py

@@ -28,7 +28,7 @@ class MultiDocumentAssembler(AnnotatorTransformer):
     refer the parameters section.
 
     For more extended examples on document pre-processing see the
-    `Spark NLP Workshop <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb>`__.
+    `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/document-assembler/Loading_Multiple_Documents.ipynb>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type
@@ -38,18 +38,15 @@ class MultiDocumentAssembler(AnnotatorTransformer):
 
     Parameters
     ----------
-    inputCols
-        Input column name
-    outputCols
-        Output column name
-    idCol
+    inputCols: str or List[str]
+        Input column name.
+    outputCols: str or List[str]
+        Output column name.
+    idCol: str
         Name of String type column for row id.
-    metadataCol
+    metadataCol: str
         Name of Map type column with metadata information
-    calculationsCol
-        Name of float vector map column to use for embeddings and other
-        representations.
-    cleanupMode
+    cleanupMode: str
         How to cleanup the document , by default disabled.
         Possible values: ``disabled, inplace, inplace_full, shrink, shrink_full,
         each, each_full, delete_full``
@@ -89,7 +86,6 @@ class MultiDocumentAssembler(AnnotatorTransformer):
     outputCols = Param(Params._dummy(), "outputCols", "output finished annotation cols", typeConverter=TypeConverters.toListString)
     idCol = Param(Params._dummy(), "idCol", "column for setting an id to such string in row", typeConverter=TypeConverters.toString)
     metadataCol = Param(Params._dummy(), "metadataCol", "String to String map column to use as metadata", typeConverter=TypeConverters.toString)
-    calculationsCol = Param(Params._dummy(), "calculationsCol", "String to Float vector map column to use as embeddigns and other representations", typeConverter=TypeConverters.toString)
     cleanupMode = Param(Params._dummy(), "cleanupMode", "possible values: disabled, inplace, inplace_full, shrink, shrink_full, each, each_full, delete_full", typeConverter=TypeConverters.toString)
     name = 'MultiDocumentAssembler'
 
@@ -108,7 +104,7 @@ class MultiDocumentAssembler(AnnotatorTransformer):
 
         Parameters
         ----------
-        *value : str
+        *value : List[str]
             Input columns for the annotator
         """
         if len(value) == 1 and type(value[0]) == list:

sparknlp/base/prompt_assembler.py

@@ -0,0 +1,207 @@
+#  Copyright 2017-2024 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the PromptAssembler."""
+
+from pyspark import keyword_only
+from pyspark.ml.param import TypeConverters, Params, Param
+
+from sparknlp.common import AnnotatorType
+from sparknlp.internal import AnnotatorTransformer
+
+
+class PromptAssembler(AnnotatorTransformer):
+    """Assembles a sequence of messages into a single string using a template. These strings can then
+    be used as prompts for large language models.
+
+    This annotator expects an array of two-tuples as the type of the input column (one array of
+    tuples per row). The first element of the tuples should be the role and the second element is
+    the text of the message. Possible roles are "system", "user" and "assistant".
+
+    An assistant header can be added to the end of the generated string by using
+    ``setAddAssistant(True)``.
+
+    At the moment, this annotator uses llama.cpp as a backend to parse and apply the templates.
+    llama.cpp uses basic pattern matching to determine the type of the template, then applies a
+    basic version of the template to the messages. This means that more advanced templates are not
+    supported.
+
+    For an extended example see the
+    `example notebook <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/llama.cpp/PromptAssember_with_AutoGGUFModel.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``NONE``               ``DOCUMENT``
+    ====================== ======================
+
+    Parameters
+    ----------
+    inputCol
+        Input column name
+    outputCol
+        Output column name
+    chatTemplate
+        Template used for the chat
+    addAssistant
+        Whether to add an assistant header to the end of the generated string
+
+    Examples
+    --------
+    >>> from sparknlp.base import *
+    >>> messages = [
+    ...     [
+    ...         ("system", "You are a helpful assistant."),
+    ...         ("assistant", "Hello there, how can I help you?"),
+    ...         ("user", "I need help with organizing my room."),
+    ...     ]
+    ... ]
+    >>> df = spark.createDataFrame([messages]).toDF("messages")
+    >>> template = (
+    ...     "{{- bos_token }} {%- if custom_tools is defined %} {%- set tools = custom_tools %} {%- "
+    ...     "endif %} {%- if not tools_in_user_message is defined %} {%- set tools_in_user_message = true %} {%- "
+    ...     'endif %} {%- if not date_string is defined %} {%- set date_string = "26 Jul 2024" %} {%- endif %} '
+    ...     "{%- if not tools is defined %} {%- set tools = none %} {%- endif %} {#- This block extracts the "
+    ...     "system message, so we can slot it into the right place. #} {%- if messages[0]['role'] == 'system' %}"
+    ...     " {%- set system_message = messages[0]['content']|trim %} {%- set messages = messages[1:] %} {%- else"
+    ...     ' %} {%- set system_message = "" %} {%- endif %} {#- System message + builtin tools #} {{- '
+    ...     '"<|start_header_id|>system<|end_header_id|>\\n\\n" }} {%- if builtin_tools is defined or tools is '
+    ...     'not none %} {{- "Environment: ipython\\n" }} {%- endif %} {%- if builtin_tools is defined %} {{- '
+    ...     '"Tools: " + builtin_tools | reject(\\'equalto\', \\'code_interpreter\\') | join(", ") + "\\n\\n"}} '
+    ...     '{%- endif %} {{- "Cutting Knowledge Date: December 2023\\n" }} {{- "Today Date: " + date_string '
+    ...     '+ "\\n\\n" }} {%- if tools is not none and not tools_in_user_message %} {{- "You have access to '
+    ...     'the following functions. To call a function, please respond with JSON for a function call." }} {{- '
+    ...     '\\'Respond in the format {"name": function name, "parameters": dictionary of argument name and its'
+    ...     ' value}.\\' }} {{- "Do not use variables.\\n\\n" }} {%- for t in tools %} {{- t | tojson(indent=4) '
+    ...     '}} {{- "\\n\\n" }} {%- endfor %} {%- endif %} {{- system_message }} {{- "<|eot_id|>" }} {#- '
+    ...     "Custom tools are passed in a user message with some extra guidance #} {%- if tools_in_user_message "
+    ...     "and not tools is none %} {#- Extract the first user message so we can plug it in here #} {%- if "
+    ...     "messages | length != 0 %} {%- set first_user_message = messages[0]['content']|trim %} {%- set "
+    ...     'messages = messages[1:] %} {%- else %} {{- raise_exception("Cannot put tools in the first user '
+    ...     "message when there's no first user message!\\") }} {%- endif %} {{- "
+    ...     "'<|start_header_id|>user<|end_header_id|>\\n\\n' -}} {{- \\"Given the following functions, please "
+    ...     'respond with a JSON for a function call " }} {{- "with its proper arguments that best answers the '
+    ...     'given prompt.\\n\\n" }} {{- \\'Respond in the format {"name": function name, "parameters": '
+    ...     'dictionary of argument name and its value}.\\' }} {{- "Do not use variables.\\n\\n" }} {%- for t in '
+    ...     'tools %} {{- t | tojson(indent=4) }} {{- "\\n\\n" }} {%- endfor %} {{- first_user_message + '
+    ...     "\\"<|eot_id|>\\"}} {%- endif %} {%- for message in messages %} {%- if not (message.role == 'ipython' "
+    ...     "or message.role == 'tool' or 'tool_calls' in message) %} {{- '<|start_header_id|>' + message['role']"
+    ...     " + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }} {%- elif 'tool_calls' in "
+    ...     'message %} {%- if not message.tool_calls|length == 1 %} {{- raise_exception("This model only '
+    ...     'supports single tool-calls at once!") }} {%- endif %} {%- set tool_call = message.tool_calls[0]'
+    ...     ".function %} {%- if builtin_tools is defined and tool_call.name in builtin_tools %} {{- "
+    ...     "'<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}} {{- \\"<|python_tag|>\\" + tool_call.name + "
+    ...     '".call(" }} {%- for arg_name, arg_val in tool_call.arguments | items %} {{- arg_name + \\'="\\' + '
+    ...     'arg_val + \\'"\\' }} {%- if not loop.last %} {{- ", " }} {%- endif %} {%- endfor %} {{- ")" }} {%- '
+    ...     "else %} {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}} {{- '{\\"name\": \\"' + "
+    ...     'tool_call.name + \\'", \\' }} {{- \\'"parameters": \\' }} {{- tool_call.arguments | tojson }} {{- "}" '
+    ...     "}} {%- endif %} {%- if builtin_tools is defined %} {#- This means we're in ipython mode #} {{- "
+    ...     '"<|eom_id|>" }} {%- else %} {{- "<|eot_id|>" }} {%- endif %} {%- elif message.role == "tool" '
+    ...     'or message.role == "ipython" %} {{- "<|start_header_id|>ipython<|end_header_id|>\\n\\n" }} {%- '
+    ...     "if message.content is mapping or message.content is iterable %} {{- message.content | tojson }} {%- "
+    ...     'else %} {{- message.content }} {%- endif %} {{- "<|eot_id|>" }} {%- endif %} {%- endfor %} {%- if '
+    ...     "add_generation_prompt %} {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }} {%- endif %} "
+    ... )
+    >>> prompt_assembler = (
+    ...     PromptAssembler()
+    ...     .setInputCol("messages")
+    ...     .setOutputCol("prompt")
+    ...     .setChatTemplate(template)
+    ... )
+    >>> prompt_assembler.transform(df).select("prompt.result").show(truncate=False)
+    +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |result                                                                                                                                                                                                                                                                                                                      |
+    +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |[<|start_header_id|>system<|end_header_id|>\\n\\nYou are a helpful assistant.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\nHello there, how can I help you?<|eot_id|><|start_header_id|>user<|end_header_id|>\\n\\nI need help with organizing my room.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\n]|
+    +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    """
+
+    outputAnnotatorType = AnnotatorType.DOCUMENT
+
+    inputCol = Param(
+        Params._dummy(),
+        "inputCol",
+        "input column name",
+        typeConverter=TypeConverters.toString,
+    )
+    outputCol = Param(
+        Params._dummy(),
+        "outputCol",
+        "output column name",
+        typeConverter=TypeConverters.toString,
+    )
+    chatTemplate = Param(
+        Params._dummy(),
+        "chatTemplate",
+        "Template used for the chat",
+        typeConverter=TypeConverters.toString,
+    )
+    addAssistant = Param(
+        Params._dummy(),
+        "addAssistant",
+        "Whether to add an assistant header to the end of the generated string",
+        typeConverter=TypeConverters.toBoolean,
+    )
+    name = "PromptAssembler"
+
+    @keyword_only
+    def __init__(self):
+        super(PromptAssembler, self).__init__(
+            classname="com.johnsnowlabs.nlp.PromptAssembler"
+        )
+        self._setDefault(outputCol="prompt", addAssistant=True)
+
+    @keyword_only
+    def setParams(self):
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    def setInputCol(self, value):
+        """Sets input column name.
+
+        Parameters
+        ----------
+        value : str
+            Name of the input column
+        """
+        return self._set(inputCol=value)
+
+    def setOutputCol(self, value):
+        """Sets output column name.
+
+        Parameters
+        ----------
+        value : str
+            Name of the Output Column
+        """
+        return self._set(outputCol=value)
+
+    def setChatTemplate(self, value):
+        """Sets the chat template.
+
+        Parameters
+        ----------
+        value : str
+            Template used for the chat
+        """
+        return self._set(chatTemplate=value)
+
+    def setAddAssistant(self, value):
+        """Sets whether to add an assistant header to the end of the generated string.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to add an assistant header to the end of the generated string
+        """
+        return self._set(addAssistant=value)

sparknlp/base/token_assembler.py

@@ -29,7 +29,7 @@ class TokenAssembler(AnnotatorTransformer, AnnotatorProperties):
     annotators. Requires ``DOCUMENT`` and ``TOKEN`` type annotations as input.
 
     For more extended examples on document pre-processing see the
-    `Spark NLP Workshop <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb>`__.
+    `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/token-assembler/Assembling_Tokens_to_Documents.ipynb>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type
@@ -122,4 +122,3 @@ class TokenAssembler(AnnotatorTransformer, AnnotatorProperties):
             Name of the Id Column
         """
         return self._set(preservePosition=value)
-

sparknlp/common/__init__.py

@@ -22,3 +22,5 @@ from sparknlp.common.recursive_annotator_approach import *
 from sparknlp.common.storage import *
 from sparknlp.common.utils import *
 from sparknlp.common.annotator_type import *
+from sparknlp.common.match_strategy import *
+from sparknlp.common.completion_post_processing import *

sparknlp/common/annotator_type.py

@@ -35,3 +35,4 @@ class AnnotatorType(object):
     NODE = "node"
     TABLE = "table"
     DUMMY = "dummy"
+    DOC_SIMILARITY_RANKINGS = "doc_similarity_rankings"

sparknlp/common/completion_post_processing.py

@@ -0,0 +1,37 @@
+#  Copyright 2017-2025 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from pyspark.ml.param import Param, Params, TypeConverters
+
+
+class CompletionPostProcessing:
+    removeThinkingTag = Param(
+        Params._dummy(),
+        "removeThinkingTag",
+        "Set a thinking tag (e.g. think) to be removed from output. Will match <TAG>...</TAG>",
+        typeConverter=TypeConverters.toString,
+    )
+
+    def setRemoveThinkingTag(self, value: str):
+        """Set a thinking tag (e.g. `think`) to be removed from output.
+        Will produce the regex: `(?s)<$TAG>.+?</$TAG>`
+        """
+        self._set(removeThinkingTag=value)
+        return self
+
+    def getRemoveThinkingTag(self):
+        """Get the thinking tag to be removed from output."""
+        value = None
+        if self.removeThinkingTag in self._paramMap:
+            value = self._paramMap[self.removeThinkingTag]
+        return value

sparknlp/common/match_strategy.py

@@ -0,0 +1,33 @@
+#  Copyright 2017-2023 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Allowed strategies for RuleFactory applications regarding replacement"""
+
+
+class MatchStrategy(object):
+    """Object that contains constants for how for matched strategies used in RuleFactory.
+
+    Possible values are:
+
+    ================================== ===============================================================================
+    Value                              Description
+    ================================== ===============================================================================
+    ``MatchStrategy.MATCH_ALL``        This strategy matches all occurrences of all rules in the given text.
+    ``MatchStrategy.MATCH_FIRST``      This strategy matches only the first occurrence of each rule in the given text.
+    ``MatchStrategy.MATCH_COMPLETE``   This strategy matches only the first occurrence of each rule in the given text.
+    ================================== ===============================================================================
+    """
+    MATCH_ALL = "MATCH_ALL"
+    MATCH_FIRST = "MATCH_FIRST"
+    MATCH_COMPLETE = "MATCH_COMPLETE"