spark-nlp 5.5.2__py2.py3-none-any.whl → 6.0.0__py2.py3-none-any.whl

sparknlp/internal/__init__.py

@@ -67,6 +67,15 @@ class _AlbertForZeroShotClassificationLoader(ExtendedJavaWrapper):
         )
 
 
+class _AlbertMultipleChoiceLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark):
+        super(_AlbertMultipleChoiceLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.classifier.dl.AlbertForMultipleChoice.loadSavedModel",
+            path,
+            jspark,
+        )
+
+
 class _BertLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark, use_openvino=False):
         super(_BertLoader, self).__init__(
@@ -121,6 +130,15 @@ class _BertMultipleChoiceLoader(ExtendedJavaWrapper):
             jspark,
         )
 
+class _CoHereLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark, use_openvino=False):
+        super(_CoHereLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.seq2seq.CoHereTransformer.loadSavedModel",
+            path,
+            jspark,
+            use_openvino,
+        )
+
 class _DeBERTaLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark):
         super(_DeBERTaLoader, self).__init__(
@@ -211,6 +229,15 @@ class _DistilBertQuestionAnsweringLoader(ExtendedJavaWrapper):
         )
 
 
+class _DistilBertMultipleChoiceLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark):
+        super(_DistilBertMultipleChoiceLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.classifier.dl.DistilBertForMultipleChoice.loadSavedModel",
+            path,
+            jspark,
+        )
+
+
 class _ElmoLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark):
         super(_ElmoLoader, self).__init__(
@@ -245,6 +272,14 @@ class _GPT2Loader(ExtendedJavaWrapper):
             jspark,
         )
 
+class _JanusForMultiModalLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark, use_openvino=False):
+        super(_JanusForMultiModalLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.cv.JanusForMultiModal.loadSavedModel",
+            path,
+            jspark,
+            use_openvino
+        )
 
 class _LLAMA2Loader(ExtendedJavaWrapper):
     def __init__(self, path, jspark, use_openvino=False):
@@ -299,6 +334,14 @@ class _LongformerQuestionAnsweringLoader(ExtendedJavaWrapper):
             jspark,
         )
 
+class _LLAVAForMultiModalLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark, use_openvino=False):
+        super(_LLAVAForMultiModalLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.cv.LLAVAForMultiModal.loadSavedModel",
+            path,
+            jspark,
+            use_openvino
+        )
 
 class _M2M100Loader(ExtendedJavaWrapper):
     def __init__(self, path, jspark, use_openvino=False):
@@ -318,6 +361,14 @@ class _MistralLoader(ExtendedJavaWrapper):
             use_openvino,
         )
 
+class _MLLamaForMultimodalLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark, use_openvino=False):
+        super(_MLLamaForMultimodalLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.cv.MLLamaForMultimodal.loadSavedModel",
+            path,
+            jspark,
+            use_openvino
+        )
 
 class _NLLBLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark,  use_openvino=False):
@@ -345,6 +396,10 @@ class _MPNetLoader(ExtendedJavaWrapper):
         )
 
 
+class _OLMoLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark):
+        super(_OLMoLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.seq2seq.OLMoTransformer.loadSavedModel", path, jspark)
 class _Phi2Loader(ExtendedJavaWrapper):
     def __init__(self, path, jspark, use_openvino=False):
         super(_Phi2Loader, self).__init__(
@@ -363,6 +418,15 @@ class _Phi3Loader(ExtendedJavaWrapper):
             use_openvino,
         )
 
+class _Phi3VisionLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark, use_openvino=False):
+        super(_Phi3VisionLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.cv.Phi3Vision.loadSavedModel",
+            path,
+            jspark,
+            use_openvino
+        )
+
 class _RoBertaLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark, use_openvino=False):
         super(_RoBertaLoader, self).__init__(
@@ -409,6 +473,15 @@ class _RoBertaQuestionAnsweringLoader(ExtendedJavaWrapper):
         )
 
 
+class _RoBertaMultipleChoiceLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark):
+        super(_RoBertaMultipleChoiceLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.classifier.dl.RoBertaForMultipleChoice.loadSavedModel",
+            path,
+            jspark,
+        )
+
+
 class _StarCoderLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark, use_openvino=False):
         super(_StarCoderLoader, self).__init__(
@@ -504,6 +577,15 @@ class _XlmRoBertaQuestionAnsweringLoader(ExtendedJavaWrapper):
         )
 
 
+class _XlmRoBertaMultipleChoiceLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark):
+        super(_XlmRoBertaMultipleChoiceLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.classifier.dl.XlmRoBertaForMultipleChoice.loadSavedModel",
+            path,
+            jspark,
+        )
+
+
 class _XlnetLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark):
         super(_XlnetLoader, self).__init__(
@@ -992,8 +1074,8 @@ class _AutoGGUFLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark):
         super(_AutoGGUFLoader, self).__init__(
             "com.johnsnowlabs.nlp.annotators.seq2seq.AutoGGUFModel.loadSavedModel", path, jspark)
-        
-        
+
+
 class _MxbaiEmbeddingsLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark):
         super(_MxbaiEmbeddingsLoader, self).__init__(
@@ -1021,3 +1103,19 @@ class _BLIPForQuestionAnswering(ExtendedJavaWrapper):
             path,
             jspark,
         )
+
+
+class _AutoGGUFVisionLoader(ExtendedJavaWrapper):
+    def __init__(self, modelPath, mmprojPath, jspark):
+        super(_AutoGGUFVisionLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.seq2seq.AutoGGUFVisionModel.loadSavedModel", modelPath, mmprojPath, jspark)
+        
+               
+class _Qwen2VLTransformerLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark, use_openvino=False):
+        super(_Qwen2VLTransformerLoader, self).__init__(
+            "com.johnsnowlabs.nlp.annotators.cv.Qwen2VLTransformer.loadSavedModel",
+            path,
+            jspark,
+            use_openvino,
+        )

sparknlp/reader/pdf_to_text.py

@@ -0,0 +1,65 @@
+from pyspark import keyword_only
+from pyspark.ml.param import Param, Params, TypeConverters
+from pyspark.ml.param.shared import HasInputCol, HasOutputCol
+from pyspark.ml.util import JavaMLReadable, JavaMLWritable
+from pyspark.ml.wrapper import JavaTransformer
+
+
+class PdfToText(JavaTransformer, HasInputCol, HasOutputCol,
+                JavaMLReadable, JavaMLWritable):
+    """
+    Extract text from Pdf document to single string or to several strings per each page.
+    Input is a column with binary representation of PDF document.
+    As output generate column with text and page number.
+    Explode each page as separate row if split to page enabled.
+    """
+    pageNumCol = Param(Params._dummy(), "pageNumCol",
+                       "Page number output column name.",
+                       typeConverter=TypeConverters.toString)
+
+    partitionNum = Param(Params._dummy(), "partitionNum",
+                         "Number of partitions.",
+                         typeConverter=TypeConverters.toInt)
+
+    storeSplittedPdf = Param(Params._dummy(), "storeSplittedPdf",
+                             "Force to store splitted pdf.",
+                             typeConverter=TypeConverters.toBoolean)
+
+    @keyword_only
+    def __init__(self):
+        """
+        __init__(self)
+        """
+        super(PdfToText, self).__init__()
+        self._java_obj = self._new_java_obj("com.johnsnowlabs.reader.PdfToText", self.uid)
+
+
+    def setInputCol(self, value):
+        """
+        Sets the value of :py:attr:`inputCol`.
+        """
+        return self._set(inputCol=value)
+
+    def setOutputCol(self, value):
+        """
+        Sets the value of :py:attr:`outputCol`.
+        """
+        return self._set(outputCol=value)
+
+    def setPageNumCol(self, value):
+        """
+        Sets the value of :py:attr:`pageNumCol`.
+        """
+        return self._set(pageNumCol=value)
+
+    def setPartitionNum(self, value):
+        """
+        Sets the value of :py:attr:`partitionNum`.
+        """
+        return self._set(partitionNum=value)
+
+    def setStoreSplittedPdf(self, value):
+        """
+        Sets the value of :py:attr:`storeSplittedPdf`.
+        """
+        return self._set(storeSplittedPdf=value)