spark-nlp 5.5.1__py2.py3-none-any.whl → 5.5.3__py2.py3-none-any.whl

{spark_nlp-5.5.1.dist-info → spark_nlp-5.5.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spark-nlp
-Version: 5.5.1
+Version: 5.5.3
 Summary: John Snow Labs Spark NLP is a natural language processing library built on top of Apache Spark ML. It provides simple, performant & accurate NLP annotations for machine learning pipelines, that scale easily in a distributed environment.
 Home-page: https://github.com/JohnSnowLabs/spark-nlp
 Author: John Snow Labs
@@ -95,7 +95,7 @@ $ java -version
 $ conda create -n sparknlp python=3.7 -y
 $ conda activate sparknlp
 # spark-nlp by default is based on pyspark 3.x
-$ pip install spark-nlp==5.5.1 pyspark==3.3.1
+$ pip install spark-nlp==5.5.3 pyspark==3.3.1
 ```
 
 In Python console or Jupyter `Python3` kernel:
@@ -161,7 +161,7 @@ For a quick example of using pipelines and models take a look at our official [d
 
 ### Apache Spark Support
 
-Spark NLP *5.5.1* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x
+Spark NLP *5.5.3* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x
 
 | Spark NLP | Apache Spark 3.5.x | Apache Spark 3.4.x | Apache Spark 3.3.x | Apache Spark 3.2.x | Apache Spark 3.1.x | Apache Spark 3.0.x | Apache Spark 2.4.x | Apache Spark 2.3.x |
 |-----------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|
@@ -189,7 +189,7 @@ Find out more about 4.x `SparkNLP` versions in our official [documentation](http
 
 ### Databricks Support
 
-Spark NLP 5.5.1 has been tested and is compatible with the following runtimes:
+Spark NLP 5.5.3 has been tested and is compatible with the following runtimes:
 
 | **CPU**            | **GPU**            |
 |--------------------|--------------------|
@@ -206,7 +206,7 @@ We are compatible with older runtimes. For a full list check databricks support
 
 ### EMR Support
 
-Spark NLP 5.5.1 has been tested and is compatible with the following EMR releases:
+Spark NLP 5.5.3 has been tested and is compatible with the following EMR releases:
 
 | **EMR Release**    |
 |--------------------|
@@ -237,7 +237,7 @@ deployed to Maven central. To add any of our packages as a dependency in your ap
 from our official documentation.
 
 If you are interested, there is a simple SBT project for Spark NLP to guide you on how to use it in your
-projects [Spark NLP SBT S5.5.1r](https://github.com/maziyarpanahi/spark-nlp-starter)
+projects [Spark NLP SBT S5.5.3r](https://github.com/maziyarpanahi/spark-nlp-starter)
 
 ### Python
 
@@ -282,7 +282,7 @@ In Spark NLP we can define S3 locations to:
 
 Please check [these instructions](https://sparknlp.org/docs/en/install#s3-integration) from our official documentation.
 
-## Document5.5.1
+## Document5.5.3
 
 ### Examples
 
@@ -315,7 +315,7 @@ the Spark NLP library:
     keywords = {Spark, Natural language processing, Deep learning, Tensorflow, Cluster},
     abstract = {Spark NLP is a Natural Language Processing (NLP) library built on top of Apache Spark ML. It provides simple, performant & accurate NLP annotations for machine learning pipelines that can scale easily in a distributed environment. Spark NLP comes with 1100+ pretrained pipelines and models in more than 192+ languages. It supports nearly all the NLP tasks and modules that can be used seamlessly in a cluster. Downloaded more than 2.7 million times and experiencing 9x growth since January 2020, Spark NLP is used by 54% of healthcare organizations as the world’s most widely used NLP library in the enterprise.}
     }
-}5.5.1
+}5.5.3
 ```
 
 ## Community support

{spark_nlp-5.5.1.dist-info → spark_nlp-5.5.3.dist-info}/RECORD

@@ -3,7 +3,7 @@ com/johnsnowlabs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 com/johnsnowlabs/ml/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 com/johnsnowlabs/ml/ai/__init__.py,sha256=YQiK2M7U4d8y5irPy_HB8ae0mSpqS9583MH44pnKJXc,295
 com/johnsnowlabs/nlp/__init__.py,sha256=DPIVXtONO5xXyOk-HB0-sNiHAcco17NN13zPS_6Uw8c,294
-sparknlp/__init__.py,sha256=26U34YGDYCBbYHr3rpxvy71snUYuoFhl5XfUvxkPv7M,13638
+sparknlp/__init__.py,sha256=Wmw9AZuFatQEjZ0WucHWPO4yF4HTsEZOVZ27IaEAbok,13783
 sparknlp/annotation.py,sha256=I5zOxG5vV2RfPZfqN9enT1i4mo6oBcn3Lrzs37QiOiA,5635
 sparknlp/annotation_audio.py,sha256=iRV_InSVhgvAwSRe9NTbUH9v6OGvTM-FPCpSAKVu0mE,1917
 sparknlp/annotation_image.py,sha256=xhCe8Ko-77XqWVuuYHFrjKqF6zPd8Z-RY_rmZXNwCXU,2547
@@ -75,7 +75,8 @@ sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py,sha256=CI9
 sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py,sha256=SndQpIfslsSYEOX-myLjpUS6-wVIeDG8MOhJYcu2_7M,6739
 sparknlp/annotator/coref/__init__.py,sha256=SG8MAaVxQpoYYAsyKaoOlvlHjorDzj3DHt4nnEdBWm8,53
 sparknlp/annotator/coref/spanbert_coref.py,sha256=AXWJhvVquY2uoApO_Np1fz7_KyJhxnZB4i-xk78sBfc,8407
-sparknlp/annotator/cv/__init__.py,sha256=LYbR0X-Wl-NkoWpSzovmhSQzk8zcCBAhqZfVd6YLnKs,943
+sparknlp/annotator/cv/__init__.py,sha256=194aJ5N5eE3HOYRzAAdroHTTQ0o1qyCrgyRLddvqBp0,1006
+sparknlp/annotator/cv/blip_for_question_answering.py,sha256=At7L5pPBNDR1r-JGLKM5b3dTrq5Ecz9r0M1gToUVZTs,6551
 sparknlp/annotator/cv/clip_for_zero_shot_classification.py,sha256=_1pLc9BiFrFN10eJPCDJLJT-vdnTSG9OnB25Y_kKJIA,7528
 sparknlp/annotator/cv/convnext_for_image_classification.py,sha256=KzaAlYW5M2l73zUozzgg8_p14eGDz9k9PYVAUZLN25k,11874
 sparknlp/annotator/cv/swin_for_image_classification.py,sha256=iZ1KY0GInbQmGzkmuNbds4PGPwCheLXc-Syv2HRmqug,10694
@@ -84,11 +85,12 @@ sparknlp/annotator/cv/vit_for_image_classification.py,sha256=D2V3pxAd3rBi1817lxV
 sparknlp/annotator/dependency/__init__.py,sha256=eV43oXAGaYl2N1XKIEAAZJLNP8gpHm8VxuXDeDlQzR4,774
 sparknlp/annotator/dependency/dependency_parser.py,sha256=SxyvHPp8Hs1Xnm5X1nLTMi095XoQMtfL8pbys15mYAI,11212
 sparknlp/annotator/dependency/typed_dependency_parser.py,sha256=60vPdYkbFk9MPGegg3m9Uik9cMXpMZd8tBvXG39gNww,12456
-sparknlp/annotator/embeddings/__init__.py,sha256=WifjEILUN6lZs4_WIX80xl5hrrsSrBK-4oVaLh-tONc,2343
+sparknlp/annotator/embeddings/__init__.py,sha256=KHDCHb8SMlkSGGSu69SfKneUDDUlBdMGdMzDrYp_cis,2408
 sparknlp/annotator/embeddings/albert_embeddings.py,sha256=6Rd1LIn8oFIpq_ALcJh-RUjPEO7Ht8wsHY6JHSFyMkw,9995
+sparknlp/annotator/embeddings/auto_gguf_embeddings.py,sha256=ngqjiXUqkL3xOrmt42bY8pp7azgbIWqXGfbKud1CijM,19981
 sparknlp/annotator/embeddings/bert_embeddings.py,sha256=HVUjkg56kBcpGZCo-fmPG5uatMDF3swW_lnbpy1SgSI,8463
 sparknlp/annotator/embeddings/bert_sentence_embeddings.py,sha256=NQy9KuXT9aKsTpYCR5RAeoFWI2YqEGorbdYrf_0KKmw,9148
-sparknlp/annotator/embeddings/bge_embeddings.py,sha256=hXFFd9HOru1w2L9N5YGSZlaKyxqMsZccpaI4Z8-bNUE,7919
+sparknlp/annotator/embeddings/bge_embeddings.py,sha256=Y4b6QzRJGc_Z9_R6SYq-P5NxcvI9XzJlBzwCLLHJpRo,8103
 sparknlp/annotator/embeddings/camembert_embeddings.py,sha256=dBTXas-2Tas_JUR9Xt_GtHLcyqi_cdvT5EHRnyVrSSQ,8817
 sparknlp/annotator/embeddings/chunk_embeddings.py,sha256=WUmkJimSuFkdcLJnvcxOV0QlCLgGlhub29ZTrZb70WE,6052
 sparknlp/annotator/embeddings/deberta_embeddings.py,sha256=_b5nzLb7heFQNN-uT2oBNO6-YmM8bHmAdnGXg47HOWw,8649
@@ -100,7 +102,7 @@ sparknlp/annotator/embeddings/instructor_embeddings.py,sha256=CTKmbuBOx_KBM4JM-Y
 sparknlp/annotator/embeddings/longformer_embeddings.py,sha256=jS4fxB5O0-d9ta9VKv8ai-17n5YHt5rML8QxUw7K4Io,8754
 sparknlp/annotator/embeddings/mpnet_embeddings.py,sha256=7d6E4lS7jjkppDPvty1UHNNrbykkriFiysrxZ_RzL0U,7875
 sparknlp/annotator/embeddings/mxbai_embeddings.py,sha256=kCaYcM3lLYJjhElLK5isdxzJqIvoGZlUKKNkySMUkE8,6017
-sparknlp/annotator/embeddings/nomic_embeddings.py,sha256=SfiTTpx0MqeHGC_nyoFNxJbfEQL4v-PrNH6hAOFsd8c,7338
+sparknlp/annotator/embeddings/nomic_embeddings.py,sha256=WTllH3htx9wDD2Le8pZgKVPM_U8XNmroJb6f4PeVeP8,7347
 sparknlp/annotator/embeddings/roberta_embeddings.py,sha256=q_WHby2lDcPc5bVHkGc6X_GwT3qyDUBLUVz5ZW4HCSY,9229
 sparknlp/annotator/embeddings/roberta_sentence_embeddings.py,sha256=KVrD4z_tIU-sphK6dmbbnHBBt8-Y89C_BFQAkN99kZo,8181
 sparknlp/annotator/embeddings/sentence_embeddings.py,sha256=azuA1FKMtTJ9suwJqTEHeWHumT6kYdfURTe_1fsqcB8,5402
@@ -146,19 +148,19 @@ sparknlp/annotator/sentiment/__init__.py,sha256=Lq3vKaZS1YATLMg0VNXSVtkWL5q5G9ta
 sparknlp/annotator/sentiment/sentiment_detector.py,sha256=m545NGU0Xzg_PO6_qIfpli1uZj7JQcyFgqe9R6wAPFI,8154
 sparknlp/annotator/sentiment/vivekn_sentiment.py,sha256=4rpXWDgzU6ddnbrSCp9VdLb2epCc9oZ3c6XcqxEw8nk,9655
 sparknlp/annotator/seq2seq/__init__.py,sha256=Fdz1zsxpB6vM2a0sKuGCSMD1ZgqeVqAez0-AtppMGB4,1541
-sparknlp/annotator/seq2seq/auto_gguf_model.py,sha256=IrhVFUh8SFodJBl5k_V6_7qCu_Ib1l223t3dzp3HsZI,39402
+sparknlp/annotator/seq2seq/auto_gguf_model.py,sha256=pTQq3KztHQq3fybdCmXEq5wTlb0t-5ANCfdQ_-7oQRg,38343
 sparknlp/annotator/seq2seq/bart_transformer.py,sha256=I1flM4yeCzEAKOdQllBC30XuedxVJ7ferkFhZ6gwEbE,18481
-sparknlp/annotator/seq2seq/cpm_transformer.py,sha256=zRbw_xAsaRnhuyYAW8UAGhz7mIyhpqk09nCsJb8-7rg,13298
+sparknlp/annotator/seq2seq/cpm_transformer.py,sha256=0CnBFMlxMu0pD2QZMHyoGtIYgXqfUQm68vr6zEAa6Eg,13290
 sparknlp/annotator/seq2seq/gpt2_transformer.py,sha256=Oz95R_NRR4tWHu_bW6Ak2832ZILXycp3ify7LfRSi8o,15310
 sparknlp/annotator/seq2seq/llama2_transformer.py,sha256=3LzTR0VerFdFmOizsrs2Q7HTnjELJ5WtfUgx5XnOqGM,13898
 sparknlp/annotator/seq2seq/llama3_transformer.py,sha256=dA3rIEVOLmlnJwhqkYmL_GrrcRVpoUY_i7QIyA5N2jM,14920
 sparknlp/annotator/seq2seq/m2m100_transformer.py,sha256=uIL9RZuuryTIdAy9TbJf9wbz6RekhW8S079bJhaB6i4,16116
 sparknlp/annotator/seq2seq/marian_transformer.py,sha256=mQ4Ylh7ZzXAOue8f-x0gqzfS3vAz3XUdD7eQ2XhcEs4,13781
 sparknlp/annotator/seq2seq/mistral_transformer.py,sha256=PJegrSQts_58rkt96xaHlqU1fKIaz8hxt7DTPkGS10A,14254
-sparknlp/annotator/seq2seq/nllb_transformer.py,sha256=hbE2k5YDAZUWPk0qyx6-5xIZi3nBFeFqLcr6lEU9LZ8,19474
+sparknlp/annotator/seq2seq/nllb_transformer.py,sha256=hOmdJOgl_-_PxoADrV-tVYmlfFrqNwvn6Vn2RC4siZM,19534
 sparknlp/annotator/seq2seq/phi2_transformer.py,sha256=WwKCUOH8qGFv62YF63HjuT7bMVldh06gHvaZH3tbSDk,13787
-sparknlp/annotator/seq2seq/phi3_transformer.py,sha256=rIFSS0sit9kUazUvMvwGFSRRsIuFNqpLH28bVKkFzx4,14219
-sparknlp/annotator/seq2seq/qwen_transformer.py,sha256=UmxF84gQsqFVyofuki9TxaPTS_fP71WTj-ylCMLKsYY,14624
+sparknlp/annotator/seq2seq/phi3_transformer.py,sha256=arIcw5NDMv3ubBwWz3KYRdLMsspTiEI8vk4s00lyq1c,14293
+sparknlp/annotator/seq2seq/qwen_transformer.py,sha256=cOpOlz5r_apmVHZgp7uFjybSzVj2yxv8QYlYcGwFyKg,14645
 sparknlp/annotator/seq2seq/starcoder_transformer.py,sha256=BTXbSMRpXnDvrfh-6iFS5k6g6EcPV9zBl4U-SSC19wA,14293
 sparknlp/annotator/seq2seq/t5_transformer.py,sha256=wDVxNLluIU1HGZFqaKKc4YTt4l-elPlAtQ7EEa0f5tg,17308
 sparknlp/annotator/similarity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -183,8 +185,8 @@ sparknlp/base/finisher.py,sha256=V4wkMm9Ug09q4zTQc9T9Wr-awmu2Hu-eNaJ039YgZXM,858
 sparknlp/base/graph_finisher.py,sha256=a8fxk3ei2YQw6s0Y9Yy8oMOF1i1XUrgqaiwVE0VPt4w,4834
 sparknlp/base/has_recursive_fit.py,sha256=P55rSHLIXhihXWS2bOC_DskcQTc3njieVD1JkjS2bcA,849
 sparknlp/base/has_recursive_transform.py,sha256=UkGNgo4LMsjQC-Coeefg4bJcg7FoPcPiG382zEa6Ywk,841
-sparknlp/base/image_assembler.py,sha256=HytRoYJTLMqGtvScHoFnp6CasG9IVNYAHYiT2_rrmeE,3719
-sparknlp/base/light_pipeline.py,sha256=Jk2DLpT4PLHCANlOo_WetTdPba_5lYs3ywiyY3lM-PE,16577
+sparknlp/base/image_assembler.py,sha256=HH7ZJ-iZCXnBXVXekQLb1ei_HJuVxhYNVb94OrVLmeY,4068
+sparknlp/base/light_pipeline.py,sha256=2lOstyyK0o6L3BHPIZWQBpIKtJ7LcSz3Pvgo6eZDs5U,17023
 sparknlp/base/multi_document_assembler.py,sha256=4htET1fRAeOB6zhsNXsBq5rKZvn-LGD4vrFRjPZeqow,7070
 sparknlp/base/prompt_assembler.py,sha256=ysU4Vbmnuv2UBHK0JBkYrxiZiJ7_GTcVMip1-QRmheI,11570
 sparknlp/base/recursive_pipeline.py,sha256=V9rTnu8KMwgjoceykN9pF1mKGtOkkuiC_n9v8dE3LDk,4279
@@ -197,12 +199,12 @@ sparknlp/common/annotator_properties.py,sha256=7B1os7pBUfHo6b7IPQAXQ-nir0u3tQLzD
 sparknlp/common/annotator_type.py,sha256=ash2Ip1IOOiJamPVyy_XQj8Ja_DRHm0b9Vj4Ni75oKM,1225
 sparknlp/common/coverage_result.py,sha256=No4PSh1HSs3PyRI1zC47x65tWgfirqPI290icHQoXEI,823
 sparknlp/common/match_strategy.py,sha256=kt1MUPqU1wCwk5qCdYk6jubHbU-5yfAYxb9jjAOrdnY,1678
-sparknlp/common/properties.py,sha256=454BAfebYhg_l7lfjXSCKPWzmCgmU3IT-r2yLGG22DI,22912
+sparknlp/common/properties.py,sha256=TMUpY0EQ3b-GXO9iuctkKrunLhRYePqu2fbmHfocr2w,23870
 sparknlp/common/read_as.py,sha256=imxPGwV7jr4Li_acbo0OAHHRGCBbYv-akzEGaBWEfcY,1226
 sparknlp/common/recursive_annotator_approach.py,sha256=vqugBw22cE3Ff7PIpRlnYFuOlchgL0nM26D8j-NdpqU,1449
 sparknlp/common/storage.py,sha256=D91H3p8EIjNspjqAYu6ephRpCUtdcAir4_PrAbkIQWE,4842
 sparknlp/common/utils.py,sha256=Yne6yYcwKxhOZC-U4qfYoDhWUP_6BIaAjI5X_P_df1E,1306
-sparknlp/internal/__init__.py,sha256=ljEf4IUraCdKU7gKFxNwFxlX-FHcnkG6sqs1MxEhLSQ,33967
+sparknlp/internal/__init__.py,sha256=BttGS21n2-LGjx8udi7f4_nNt_BeUnfif9WpeZchuFE,34502
 sparknlp/internal/annotator_java_ml.py,sha256=UGPoThG0rGXUOXGSQnDzEDW81Mu1s5RPF29v7DFyE3c,1187
 sparknlp/internal/annotator_transformer.py,sha256=fXmc2IWXGybqZpbEU9obmbdBYPc798y42zvSB4tqV9U,1448
 sparknlp/internal/extended_java_wrapper.py,sha256=hwP0133-hDiDf5sBF-P3MtUsuuDj1PpQbtGZQIRwzfk,2240
@@ -214,6 +216,8 @@ sparknlp/pretrained/__init__.py,sha256=GV-x9UBK8F2_IR6zYatrzFcVJtkSUIMbxqWsxRUeP
 sparknlp/pretrained/pretrained_pipeline.py,sha256=lquxiaABuA68Rmu7csamJPqBoRJqMUO0oNHsmEZDAIs,5740
 sparknlp/pretrained/resource_downloader.py,sha256=8_-rpvO2LsX_Lq4wMPif2ca3RlJZWEabt8pDm2xymiI,7806
 sparknlp/pretrained/utils.py,sha256=T1MrvW_DaWk_jcOjVLOea0NMFE9w8fe0ZT_5urZ_nEY,1099
+sparknlp/reader/__init__.py,sha256=-Toj3AIBki-zXPpV8ezFTI2LX1yP_rK2bhpoa8nBkTw,685
+sparknlp/reader/sparknlp_reader.py,sha256=cMliB2zDcmhxp44mu8aRcm5nFK2BXeFCuGgVUkhI8YQ,3825
 sparknlp/training/__init__.py,sha256=qREi9u-5Vc2VjpL6-XZsyvu5jSEIdIhowW7_kKaqMqo,852
 sparknlp/training/conll.py,sha256=wKBiSTrjc6mjsl7Nyt6B8f4yXsDJkZb-sn8iOjix9cE,6961
 sparknlp/training/conllu.py,sha256=8r3i-tmyrLsyk1DtZ9uo2mMDCWb1yw2Y5W6UsV13MkY,4953
@@ -244,8 +248,8 @@ sparknlp/training/_tf_graph_builders_1x/ner_dl/dataset_encoder.py,sha256=R4yHFN3
 sparknlp/training/_tf_graph_builders_1x/ner_dl/ner_model.py,sha256=EoCSdcIjqQ3wv13MAuuWrKV8wyVBP0SbOEW41omHlR0,23189
 sparknlp/training/_tf_graph_builders_1x/ner_dl/ner_model_saver.py,sha256=k5CQ7gKV6HZbZMB8cKLUJuZxoZWlP_DFWdZ--aIDwsc,2356
 sparknlp/training/_tf_graph_builders_1x/ner_dl/sentence_grouper.py,sha256=pAxjWhjazSX8Vg0MFqJiuRVw1IbnQNSs-8Xp26L4nko,870
-spark_nlp-5.5.1.dist-info/.uuid,sha256=1f6hF51aIuv9yCvh31NU9lOpS34NE-h3a0Et7R9yR6A,36
-spark_nlp-5.5.1.dist-info/METADATA,sha256=Y7Y0nf18tO2RfHzagHWWZpn4QRrF50d5wP3hXG1eFyw,19156
-spark_nlp-5.5.1.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-spark_nlp-5.5.1.dist-info/top_level.txt,sha256=uuytur4pyMRw2H_txNY2ZkaucZHUs22QF8-R03ch_-E,13
-spark_nlp-5.5.1.dist-info/RECORD,,
+spark_nlp-5.5.3.dist-info/.uuid,sha256=1f6hF51aIuv9yCvh31NU9lOpS34NE-h3a0Et7R9yR6A,36
+spark_nlp-5.5.3.dist-info/METADATA,sha256=rZJcS1xIcl3Vota-hC2wHauvrHO45e9c8Y86MjVt4go,19156
+spark_nlp-5.5.3.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+spark_nlp-5.5.3.dist-info/top_level.txt,sha256=uuytur4pyMRw2H_txNY2ZkaucZHUs22QF8-R03ch_-E,13
+spark_nlp-5.5.3.dist-info/RECORD,,

sparknlp/__init__.py

@@ -12,17 +12,20 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import sys
 import subprocess
+import sys
 import threading
+
+from pyspark.conf import SparkConf
+from pyspark.context import SparkContext
+from pyspark.java_gateway import launch_gateway
 from pyspark.sql import SparkSession
+
 from sparknlp import annotator
 # Must be declared here one by one or else PretrainedPipeline will fail with AttributeError
 from sparknlp.base import DocumentAssembler, MultiDocumentAssembler, Finisher, EmbeddingsFinisher, TokenAssembler, \
     Doc2Chunk, AudioAssembler, GraphFinisher, ImageAssembler, TableAssembler
-from pyspark.conf import SparkConf
-from pyspark.context import SparkContext
-from pyspark.java_gateway import launch_gateway
+from sparknlp.reader import SparkNLPReader
 
 sys.modules['com.johnsnowlabs.nlp.annotators'] = annotator
 sys.modules['com.johnsnsowlabs.nlp.annotators.tokenizer'] = annotator
@@ -129,7 +132,7 @@ def start(gpu=False,
         The initiated Spark session.
 
     """
-    current_version = "5.5.1"
+    current_version = "5.5.3"
 
     if params is None:
         params = {}
@@ -301,6 +304,9 @@ def start(gpu=False,
         spark_session = start_without_realtime_output()
         return spark_session
 
+def read(params=None):
+    spark_session = start()
+    return SparkNLPReader(spark_session, params)
 
 def version():
     """Returns the current Spark NLP version.
@@ -310,4 +316,4 @@ def version():
     str
         The current Spark NLP version.
     """
-    return '5.5.1'
+    return '5.5.3'

sparknlp/annotator/cv/__init__.py

@@ -16,3 +16,4 @@ from sparknlp.annotator.cv.swin_for_image_classification import *
 from sparknlp.annotator.cv.convnext_for_image_classification import *
 from sparknlp.annotator.cv.vision_encoder_decoder_for_image_captioning import *
 from sparknlp.annotator.cv.clip_for_zero_shot_classification import *
+from sparknlp.annotator.cv.blip_for_question_answering import *

sparknlp/annotator/cv/blip_for_question_answering.py

@@ -0,0 +1,172 @@
+#  Copyright 2017-2024 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from sparknlp.common import *
+
+class BLIPForQuestionAnswering(AnnotatorModel,
+                               HasBatchedAnnotateImage,
+                               HasImageFeatureProperties,
+                               HasEngine,
+                               HasCandidateLabelsProperties,
+                               HasRescaleFactor):
+    """BLIPForQuestionAnswering can load BLIP models  for visual question answering.
+    The model consists of a vision encoder, a text encoder as well as a text decoder.
+    The vision encoder will encode the input image, the text encoder will encode the input question together
+    with the encoding of the image, and the text decoder will output the answer to the question.
+
+    Pretrained models can be loaded with :meth:`.pretrained` of the companion
+    object:
+
+    >>> visualQAClassifier = BLIPForQuestionAnswering.pretrained() \\
+    ...     .setInputCols(["image_assembler"]) \\
+    ...     .setOutputCol("answer")
+
+    The default model is ``"blip_vqa_base"``, if no name is
+    provided.
+
+    For available pretrained models please see the `Models Hub
+    <https://sparknlp.org/models?task=Question+Answering>`__.
+
+    To see which models are compatible and how to import them see
+    `Import Transformers into Spark NLP 🚀
+    <https://github.com/JohnSnowLabs/spark-nlp/discussions/5669>`_.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``IMAGE``              ``DOCUMENT``
+    ====================== ======================
+
+    Parameters
+    ----------
+    batchSize
+        Batch size. Large values allows faster processing but requires more
+        memory, by default 2
+    configProtoBytes
+        ConfigProto from tensorflow, serialized into byte array.
+    maxSentenceLength
+        Max sentence length to process, by default 50
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> image_df = SparkSessionForTest.spark.read.format("image").load(path=images_path)
+    >>> test_df = image_df.withColumn("text", lit("What's this picture about?"))
+    >>> imageAssembler = ImageAssembler() \\
+    ...     .setInputCol("image") \\
+    ...     .setOutputCol("image_assembler")
+    >>> visualQAClassifier = BLIPForQuestionAnswering.pretrained() \\
+    ...     .setInputCols("image_assembler") \\
+    ...     .setOutputCol("answer") \\
+    ...     .setSize(384)
+    >>> pipeline = Pipeline().setStages([
+    ...     imageAssembler,
+    ...     visualQAClassifier
+    ... ])
+    >>> result = pipeline.fit(test_df).transform(test_df)
+    >>> result.select("image_assembler.origin", "answer.result").show(false)
+    +--------------------------------------+------+
+    |origin                                |result|
+    +--------------------------------------+------+
+    |[file:///content/images/cat_image.jpg]|[cats]|
+    +--------------------------------------+------+
+    """
+
+    name = "BLIPForQuestionAnswering"
+
+    inputAnnotatorTypes = [AnnotatorType.IMAGE]
+
+    outputAnnotatorType = AnnotatorType.DOCUMENT
+
+    configProtoBytes = Param(Params._dummy(),
+                             "configProtoBytes",
+                             "ConfigProto from tensorflow, serialized into byte array. Get with "
+                             "config_proto.SerializeToString()",
+                             TypeConverters.toListInt)
+
+    maxSentenceLength = Param(Params._dummy(),
+                            "maxSentenceLength",
+                            "Maximum sentence length that the annotator will process. Above this, the sentence is skipped",
+                            typeConverter=TypeConverters.toInt)
+
+    def setMaxSentenceSize(self, value):
+        """Sets Maximum sentence length that the annotator will process, by
+        default 50.
+
+        Parameters
+        ----------
+        value : int
+            Maximum sentence length that the annotator will process
+        """
+        return self._set(maxSentenceLength=value)
+
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.cv.BLIPForQuestionAnswering",
+                 java_model=None):
+        super(BLIPForQuestionAnswering, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+        self._setDefault(
+            batchSize=2,
+            size=384,
+            maxSentenceLength=50
+        )
+
+    @staticmethod
+    def loadSavedModel(folder, spark_session):
+        """Loads a locally saved model.
+
+        Parameters
+        ----------
+        folder : str
+            Folder of the saved model
+        spark_session : pyspark.sql.SparkSession
+            The current SparkSession
+
+        Returns
+        -------
+        CLIPForZeroShotClassification
+            The restored model
+        """
+        from sparknlp.internal import _BLIPForQuestionAnswering
+        jModel = _BLIPForQuestionAnswering(folder, spark_session._jsparkSession)._java_obj
+        return BLIPForQuestionAnswering(java_model=jModel)
+
+    @staticmethod
+    def pretrained(name="blip_vqa_base", lang="en", remote_loc=None):
+        """Downloads and loads a pretrained model.
+
+        Parameters
+        ----------
+        name : str, optional
+            Name of the pretrained model, by default
+            "blip_vqa_tf"
+        lang : str, optional
+            Language of the pretrained model, by default "en"
+        remote_loc : str, optional
+            Optional remote address of the resource, by default None. Will use
+            Spark NLPs repositories otherwise.
+
+        Returns
+        -------
+        CLIPForZeroShotClassification
+            The restored model
+        """
+        from sparknlp.pretrained import ResourceDownloader
+        return ResourceDownloader.downloadModel(BLIPForQuestionAnswering, name, lang, remote_loc)

sparknlp/annotator/embeddings/__init__.py

@@ -40,3 +40,4 @@ from sparknlp.annotator.embeddings.uae_embeddings import *
 from sparknlp.annotator.embeddings.mxbai_embeddings import *
 from sparknlp.annotator.embeddings.snowflake_embeddings import *
 from sparknlp.annotator.embeddings.nomic_embeddings import *
+from sparknlp.annotator.embeddings.auto_gguf_embeddings import *