spark-nlp 5.3.2__py2.py3-none-any.whl → 5.4.0__py2.py3-none-any.whl

com/johnsnowlabs/ml/ai/__init__.py

@@ -0,0 +1,10 @@
+import sys
+
+if sys.version_info[0] == 2:
+    raise ImportError(
+        "Spark NLP for Python 2.x is deprecated since version >= 4.0. "
+        "Please use an older versions to use it with this Python version."
+    )
+else:
+    import sparknlp
+    sys.modules['com.johnsnowlabs.ml.ai'] = sparknlp

{spark_nlp-5.3.2.dist-info → spark_nlp-5.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spark-nlp
-Version: 5.3.2
+Version: 5.4.0
 Summary: John Snow Labs Spark NLP is a natural language processing library built on top of Apache Spark ML. It provides simple, performant & accurate NLP annotations for machine learning pipelines, that scale easily in a distributed environment.
 Home-page: https://github.com/JohnSnowLabs/spark-nlp
 Author: John Snow Labs
@@ -146,6 +146,7 @@ documentation and examples
 - INSTRUCTOR Embeddings (HuggingFace models)
 - E5 Embeddings (HuggingFace models)
 - MPNet Embeddings (HuggingFace models)
+- UAE Embeddings (HuggingFace models)
 - OpenAI Embeddings
 - Sentence & Chunk Embeddings
 - Unsupervised keywords extraction
@@ -170,7 +171,7 @@ documentation and examples
 - Text-To-Text Transfer Transformer (Google T5)
 - Generative Pre-trained Transformer 2 (OpenAI GPT2)
 - Seq2Seq for NLG, Translation, and Comprehension (Facebook BART)
-- Chat and Conversational LLMs (Facebook Llama-22)
+- Chat and Conversational LLMs (Facebook Llama-2)
 - Vision Transformer (Google ViT)
 - Swin Image Classification (Microsoft Swin Transformer)
 - ConvNext Image Classification (Facebook ConvNext)
@@ -180,10 +181,10 @@ documentation and examples
 - Automatic Speech Recognition (HuBERT)
 - Automatic Speech Recognition (OpenAI Whisper)
 - Named entity recognition (Deep learning)
-- Easy ONNX and TensorFlow integrations
+- Easy ONNX, OpenVINO, and TensorFlow integrations
 - GPU Support
 - Full integration with Spark ML functions
-- +30000 pre-trained models in +200 languages!
+- +31000 pre-trained models in +200 languages!
 - +6000 pre-trained pipelines in +200 languages!
 - Multi-lingual NER models: Arabic, Bengali, Chinese, Danish, Dutch, English, Finnish, French, German, Hebrew, Italian,
   Japanese, Korean, Norwegian, Persian, Polish, Portuguese, Russian, Spanish, Swedish, Urdu, and more.
@@ -197,7 +198,7 @@ To use Spark NLP you need the following requirements:
 
 **GPU (optional):**
 
-Spark NLP 5.3.2 is built with ONNX 1.17.0 and TensorFlow 2.7.1 deep learning engines. The minimum following NVIDIA® software are only required for GPU support:
+Spark NLP 5.4.0 is built with ONNX 1.17.0 and TensorFlow 2.7.1 deep learning engines. The minimum following NVIDIA® software are only required for GPU support:
 
 - NVIDIA® GPU drivers version 450.80.02 or higher
 - CUDA® Toolkit 11.2
@@ -213,7 +214,7 @@ $ java -version
 $ conda create -n sparknlp python=3.7 -y
 $ conda activate sparknlp
 # spark-nlp by default is based on pyspark 3.x
-$ pip install spark-nlp==5.3.2 pyspark==3.3.1
+$ pip install spark-nlp==5.4.0 pyspark==3.3.1
 ```
 
 In Python console or Jupyter `Python3` kernel:
@@ -258,10 +259,11 @@ For more examples, you can visit our dedicated [examples](https://github.com/Joh
 
 ## Apache Spark Support
 
-Spark NLP *5.3.2* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x
+Spark NLP *5.4.0* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x
 
 | Spark NLP | Apache Spark 3.5.x | Apache Spark 3.4.x | Apache Spark 3.3.x | Apache Spark 3.2.x | Apache Spark 3.1.x | Apache Spark 3.0.x | Apache Spark 2.4.x | Apache Spark 2.3.x |
 |-----------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|
+| 5.4.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 5.3.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 5.2.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 5.1.x     | Partially          | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
@@ -271,12 +273,6 @@ Spark NLP *5.3.2* has been built on top of Apache Spark 3.4 while fully supports
 | 4.2.x     | NO                 | NO                 | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 4.1.x     | NO                 | NO                 | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 4.0.x     | NO                 | NO                 | YES                | YES                | YES                | YES                | NO                 | NO                 |
-| 3.4.x     | NO                 | NO                 | N/A                | Partially          | YES                | YES                | YES                | YES                |
-| 3.3.x     | NO                 | NO                 | NO                 | NO                 | YES                | YES                | YES                | YES                |
-| 3.2.x     | NO                 | NO                 | NO                 | NO                 | YES                | YES                | YES                | YES                |
-| 3.1.x     | NO                 | NO                 | NO                 | NO                 | YES                | YES                | YES                | YES                |
-| 3.0.x     | NO                 | NO                 | NO                 | NO                 | YES                | YES                | YES                | YES                |
-| 2.7.x     | NO                 | NO                 | NO                 | NO                 | NO                 | NO                 | YES                | YES                |
 
 Find out more about `Spark NLP` versions from our [release notes](https://github.com/JohnSnowLabs/spark-nlp/releases).
 
@@ -293,16 +289,10 @@ Find out more about `Spark NLP` versions from our [release notes](https://github
 | 4.2.x     | YES        | YES        | YES        | YES        | YES        | NO         | YES        |
 | 4.1.x     | YES        | YES        | YES        | YES        | NO         | NO         | YES        |
 | 4.0.x     | YES        | YES        | YES        | YES        | NO         | NO         | YES        |
-| 3.4.x     | YES        | YES        | YES        | YES        | NO         | YES        | YES        |
-| 3.3.x     | YES        | YES        | YES        | NO         | NO         | YES        | YES        |
-| 3.2.x     | YES        | YES        | YES        | NO         | NO         | YES        | YES        |
-| 3.1.x     | YES        | YES        | YES        | NO         | NO         | YES        | YES        |
-| 3.0.x     | YES        | YES        | YES        | NO         | NO         | YES        | YES        |
-| 2.7.x     | YES        | YES        | NO         | NO         | NO         | YES        | NO         |
 
 ## Databricks Support
 
-Spark NLP 5.3.2 has been tested and is compatible with the following runtimes:
+Spark NLP 5.4.0 has been tested and is compatible with the following runtimes:
 
 **CPU:**
 
@@ -375,7 +365,7 @@ Spark NLP 5.3.2 has been tested and is compatible with the following runtimes:
 
 ## EMR Support
 
-Spark NLP 5.3.2 has been tested and is compatible with the following EMR releases:
+Spark NLP 5.4.0 has been tested and is compatible with the following EMR releases:
 
 - emr-6.2.0
 - emr-6.3.0
@@ -425,11 +415,11 @@ Spark NLP supports all major releases of Apache Spark 3.0.x, Apache Spark 3.1.x,
 ```sh
 # CPU
 
-spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2
+spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0
 
-pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2
+pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0
 
-spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2
+spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0
 ```
 
 The `spark-nlp` has been published to
@@ -438,11 +428,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s
 ```sh
 # GPU
 
-spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.2
+spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.4.0
 
-pyspark --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.2
+pyspark --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.4.0
 
-spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.2
+spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.4.0
 
 ```
 
@@ -452,11 +442,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s
 ```sh
 # AArch64
 
-spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.2
+spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.4.0
 
-pyspark --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.2
+pyspark --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.4.0
 
-spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.2
+spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.4.0
 
 ```
 
@@ -466,11 +456,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s
 ```sh
 # M1/M2 (Apple Silicon)
 
-spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.2
+spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.4.0
 
-pyspark --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.2
+pyspark --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.4.0
 
-spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.2
+spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.4.0
 
 ```
 
@@ -484,7 +474,7 @@ set in your SparkSession:
 spark-shell \
   --driver-memory 16g \
   --conf spark.kryoserializer.buffer.max=2000M \
-  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2
+  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0
 ```
 
 ## Scala
@@ -502,7 +492,7 @@ coordinates:
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp_2.12</artifactId>
-    <version>5.3.2</version>
+    <version>5.4.0</version>
 </dependency>
 ```
 
@@ -513,7 +503,7 @@ coordinates:
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp-gpu_2.12</artifactId>
-    <version>5.3.2</version>
+    <version>5.4.0</version>
 </dependency>
 ```
 
@@ -524,7 +514,7 @@ coordinates:
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp-aarch64_2.12</artifactId>
-    <version>5.3.2</version>
+    <version>5.4.0</version>
 </dependency>
 ```
 
@@ -535,7 +525,7 @@ coordinates:
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp-silicon_2.12</artifactId>
-    <version>5.3.2</version>
+    <version>5.4.0</version>
 </dependency>
 ```
 
@@ -545,28 +535,28 @@ coordinates:
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.2"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.4.0"
 ```
 
 **spark-nlp-gpu:**
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-gpu
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.2"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.4.0"
 ```
 
 **spark-nlp-aarch64:**
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-aarch64
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.2"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.4.0"
 ```
 
 **spark-nlp-silicon:**
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-silicon
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.2"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.4.0"
 ```
 
 Maven
@@ -588,7 +578,7 @@ If you installed pyspark through pip/conda, you can install `spark-nlp` through
 Pip:
 
 ```bash
-pip install spark-nlp==5.3.2
+pip install spark-nlp==5.4.0
 ```
 
 Conda:
@@ -617,7 +607,7 @@ spark = SparkSession.builder
     .config("spark.driver.memory", "16G")
     .config("spark.driver.maxResultSize", "0")
     .config("spark.kryoserializer.buffer.max", "2000M")
-    .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2")
+    .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0")
     .getOrCreate()
 ```
 
@@ -688,7 +678,7 @@ Use either one of the following options
 - Add the following Maven Coordinates to the interpreter's library list
 
 ```bash
-com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2
+com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0
 ```
 
 - Add a path to pre-built jar from [here](#compiled-jars) in the interpreter's library list making sure the jar is
@@ -699,7 +689,7 @@ com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2
 Apart from the previous step, install the python module through pip
 
 ```bash
-pip install spark-nlp==5.3.2
+pip install spark-nlp==5.4.0
 ```
 
 Or you can install `spark-nlp` from inside Zeppelin by using Conda:
@@ -727,7 +717,7 @@ launch the Jupyter from the same Python environment:
 $ conda create -n sparknlp python=3.8 -y
 $ conda activate sparknlp
 # spark-nlp by default is based on pyspark 3.x
-$ pip install spark-nlp==5.3.2 pyspark==3.3.1 jupyter
+$ pip install spark-nlp==5.4.0 pyspark==3.3.1 jupyter
 $ jupyter notebook
 ```
 
@@ -744,7 +734,7 @@ export PYSPARK_PYTHON=python3
 export PYSPARK_DRIVER_PYTHON=jupyter
 export PYSPARK_DRIVER_PYTHON_OPTS=notebook
 
-pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2
+pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0
 ```
 
 Alternatively, you can mix in using `--jars` option for pyspark + `pip install spark-nlp`
@@ -771,7 +761,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi
 # -s is for spark-nlp
 # -g will enable upgrading libcudnn8 to 8.1.0 on Google Colab for GPU usage
 # by default they are set to the latest
-!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.2
+!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.4.0
 ```
 
 [Spark NLP quick start on Google Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/quick_start_google_colab.ipynb)
@@ -794,7 +784,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi
 # -s is for spark-nlp
 # -g will enable upgrading libcudnn8 to 8.1.0 on Kaggle for GPU usage
 # by default they are set to the latest
-!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.2
+!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.4.0
 ```
 
 [Spark NLP quick start on Kaggle Kernel](https://www.kaggle.com/mozzie/spark-nlp-named-entity-recognition) is a live
@@ -813,9 +803,9 @@ demo on Kaggle Kernel that performs named entity recognitions by using Spark NLP
 
 3. In `Libraries` tab inside your cluster you need to follow these steps:
 
-   3.1. Install New -> PyPI -> `spark-nlp==5.3.2` -> Install
+   3.1. Install New -> PyPI -> `spark-nlp==5.4.0` -> Install
 
-   3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2` -> Install
+   3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0` -> Install
 
 4. Now you can attach your notebook to the cluster and use Spark NLP!
 
@@ -866,7 +856,7 @@ A sample of your software configuration in JSON on S3 (must be public access):
       "spark.kryoserializer.buffer.max": "2000M",
       "spark.serializer": "org.apache.spark.serializer.KryoSerializer",
       "spark.driver.maxResultSize": "0",
-      "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2"
+      "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0"
     }
 }]
 ```
@@ -875,7 +865,7 @@ A sample of AWS CLI to launch EMR cluster:
 
 ```.sh
 aws emr create-cluster \
---name "Spark NLP 5.3.2" \
+--name "Spark NLP 5.4.0" \
 --release-label emr-6.2.0 \
 --applications Name=Hadoop Name=Spark Name=Hive \
 --instance-type m4.4xlarge \
@@ -939,7 +929,7 @@ gcloud dataproc clusters create ${CLUSTER_NAME} \
   --enable-component-gateway \
   --metadata 'PIP_PACKAGES=spark-nlp spark-nlp-display google-cloud-bigquery google-cloud-storage' \
   --initialization-actions gs://goog-dataproc-initialization-actions-${REGION}/python/pip-install.sh \
-  --properties spark:spark.serializer=org.apache.spark.serializer.KryoSerializer,spark:spark.driver.maxResultSize=0,spark:spark.kryoserializer.buffer.max=2000M,spark:spark.jars.packages=com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2
+  --properties spark:spark.serializer=org.apache.spark.serializer.KryoSerializer,spark:spark.driver.maxResultSize=0,spark:spark.kryoserializer.buffer.max=2000M,spark:spark.jars.packages=com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0
 ```
 
 2. On an existing one, you need to install spark-nlp and spark-nlp-display packages from PyPI.
@@ -982,7 +972,7 @@ spark = SparkSession.builder
     .config("spark.kryoserializer.buffer.max", "2000m")
     .config("spark.jsl.settings.pretrained.cache_folder", "sample_data/pretrained")
     .config("spark.jsl.settings.storage.cluster_tmp_dir", "sample_data/storage")
-    .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2")
+    .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0")
     .getOrCreate()
 ```
 
@@ -996,7 +986,7 @@ spark-shell \
   --conf spark.kryoserializer.buffer.max=2000M \
   --conf spark.jsl.settings.pretrained.cache_folder="sample_data/pretrained" \
   --conf spark.jsl.settings.storage.cluster_tmp_dir="sample_data/storage" \
-  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2
+  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0
 ```
 
 **pyspark:**
@@ -1009,7 +999,7 @@ pyspark \
   --conf spark.kryoserializer.buffer.max=2000M \
   --conf spark.jsl.settings.pretrained.cache_folder="sample_data/pretrained" \
   --conf spark.jsl.settings.storage.cluster_tmp_dir="sample_data/storage" \
-  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.2
+  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.4.0
 ```
 
 **Databricks:**
@@ -1281,7 +1271,7 @@ spark = SparkSession.builder
     .config("spark.driver.memory", "16G")
     .config("spark.driver.maxResultSize", "0")
     .config("spark.kryoserializer.buffer.max", "2000M")
-    .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.2.jar")
+    .config("spark.jars", "/tmp/spark-nlp-assembly-5.4.0.jar")
     .getOrCreate()
 ```
 
@@ -1290,7 +1280,7 @@ spark = SparkSession.builder
   version (3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x)
 - If you are local, you can load the Fat JAR from your local FileSystem, however, if you are in a cluster setup you need
   to put the Fat JAR on a distributed FileSystem such as HDFS, DBFS, S3, etc. (
-  i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.2.jar`)
+  i.e., `hdfs:///tmp/spark-nlp-assembly-5.4.0.jar`)
 
 Example of using pretrained Models and Pipelines in offline:
 

{spark_nlp-5.3.2.dist-info → spark_nlp-5.4.0.dist-info}/RECORD

@@ -1,7 +1,9 @@
 com/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 com/johnsnowlabs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+com/johnsnowlabs/ml/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+com/johnsnowlabs/ml/ai/__init__.py,sha256=YQiK2M7U4d8y5irPy_HB8ae0mSpqS9583MH44pnKJXc,295
 com/johnsnowlabs/nlp/__init__.py,sha256=DPIVXtONO5xXyOk-HB0-sNiHAcco17NN13zPS_6Uw8c,294
-sparknlp/__init__.py,sha256=N74jXyWIf6cZnJADb-aBShj-r4eCjcLGDPmXcRNEBN0,13588
+sparknlp/__init__.py,sha256=lRQR3K0noT97MQlXrjnJEgvD4QIvuUUMrbC7VCND4w4,13638
 sparknlp/annotation.py,sha256=I5zOxG5vV2RfPZfqN9enT1i4mo6oBcn3Lrzs37QiOiA,5635
 sparknlp/annotation_audio.py,sha256=iRV_InSVhgvAwSRe9NTbUH9v6OGvTM-FPCpSAKVu0mE,1917
 sparknlp/annotation_image.py,sha256=xhCe8Ko-77XqWVuuYHFrjKqF6zPd8Z-RY_rmZXNwCXU,2547
@@ -28,7 +30,7 @@ sparknlp/annotator/audio/__init__.py,sha256=dXjtvi5c0aTZFq1Q_JciUd1uFTBVSJoUdcq0
 sparknlp/annotator/audio/hubert_for_ctc.py,sha256=76PfwPZZvOHU5kfDqLueCFbmqa4W8pMNRGoCvOqjsEA,7859
 sparknlp/annotator/audio/wav2vec2_for_ctc.py,sha256=K78P1U6vA4O1UufsLYzy0H7arsKNmwPcIV7kzDFsA5Q,6210
 sparknlp/annotator/audio/whisper_for_ctc.py,sha256=uII51umuohqwnAW0Q7VdxEFyr_j5LMnfpcRlf8TbetA,9800
-sparknlp/annotator/classifier_dl/__init__.py,sha256=tGg78A8LUgobZFre_3ySN51KGNyl0Zx0inxT9yfL_g8,3686
+sparknlp/annotator/classifier_dl/__init__.py,sha256=dsbceLBdAsk0VlvgcCcGANHMcyFMKi7-sdyu-Eg41ws,3763
 sparknlp/annotator/classifier_dl/albert_for_question_answering.py,sha256=LG2dL6Fky1T35yXTUZBfIihIIGnkRFQ7ECQ3HRXXEG8,6517
 sparknlp/annotator/classifier_dl/albert_for_sequence_classification.py,sha256=kWx7f9pcKE2qw319gn8FN0Md5dX38gbmfeoY9gWCLNk,7842
 sparknlp/annotator/classifier_dl/albert_for_token_classification.py,sha256=5rdsjWnsAVmtP-idU7ATKJ8lkH2rtlKZLnpi4Mq27eI,6839
@@ -54,6 +56,7 @@ sparknlp/annotator/classifier_dl/longformer_for_sequence_classification.py,sha25
 sparknlp/annotator/classifier_dl/longformer_for_token_classification.py,sha256=RmiFuBRhIAoJoQ8Rgcu997-PxBK1hhWuLVlS1qztMyk,6848
 sparknlp/annotator/classifier_dl/mpnet_for_question_answering.py,sha256=w9hHLrQbDIUHAdCKiXNDneAbohMKopixAKU2wkYkqbs,5522
 sparknlp/annotator/classifier_dl/mpnet_for_sequence_classification.py,sha256=M__giFElL6Q3I88QD6OoXDzdQDk_Zp5sS__Kh_XpLdo,7308
+sparknlp/annotator/classifier_dl/mpnet_for_token_classification.py,sha256=SgFAJcv7ZE3BmJOehK_CjAaueqaaK6PR33zA5aE9-Ww,6754
 sparknlp/annotator/classifier_dl/multi_classifier_dl.py,sha256=ylKQzS7ROyeKeiOF4BZiIkQV1sfrnfUUQ9LXFSFK_Vo,16045
 sparknlp/annotator/classifier_dl/roberta_for_question_answering.py,sha256=WRxu1uhXnY9C4UHdtJ8qiVGhPSX7sCdSaML0AWHOdJw,6471
 sparknlp/annotator/classifier_dl/roberta_for_sequence_classification.py,sha256=z97uH5WkG8kPX1Y9qtpLwD7egl0kzbVmxtq4xzZgNNI,7857
@@ -63,7 +66,7 @@ sparknlp/annotator/classifier_dl/sentiment_dl.py,sha256=6Z7X3-ykxoaUz6vz-YIXkv2m
 sparknlp/annotator/classifier_dl/tapas_for_question_answering.py,sha256=2YBODMDUZT-j5ceOFTixrEkOqrztIM1kU-tsW_wao18,6317
 sparknlp/annotator/classifier_dl/xlm_roberta_for_question_answering.py,sha256=t_zCnKGCjDccKNj_2mjRkysOaNCWNBMKXehbuFSphQc,6538
 sparknlp/annotator/classifier_dl/xlm_roberta_for_sequence_classification.py,sha256=sudgwa8_QZQzaYvEMSt6J1bDDwyK2Hp1VFhh98P08hY,7930
-sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py,sha256=pe4Y1XDxDMQs1q32bwhbPC5_oKcJ4n5JFu-dsofLdSA,6850
+sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py,sha256=ub5mMiZYKP4eBmXRzjkjfv_FFFR8E01XJs0RC__RxPo,6808
 sparknlp/annotator/classifier_dl/xlm_roberta_for_zero_shot_classification.py,sha256=4dBzpPj-VJcZul5hGcyjYkVMQ1PiaXZEGwvEaob3rss,8899
 sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py,sha256=CI9Ah2lyHkqwDHWGCbkk_gPbQd0NudpC7oXiHtWOucs,7811
 sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py,sha256=SndQpIfslsSYEOX-myLjpUS6-wVIeDG8MOhJYcu2_7M,6739
@@ -78,28 +81,29 @@ sparknlp/annotator/cv/vit_for_image_classification.py,sha256=D2V3pxAd3rBi1817lxV
 sparknlp/annotator/dependency/__init__.py,sha256=eV43oXAGaYl2N1XKIEAAZJLNP8gpHm8VxuXDeDlQzR4,774
 sparknlp/annotator/dependency/dependency_parser.py,sha256=SxyvHPp8Hs1Xnm5X1nLTMi095XoQMtfL8pbys15mYAI,11212
 sparknlp/annotator/dependency/typed_dependency_parser.py,sha256=60vPdYkbFk9MPGegg3m9Uik9cMXpMZd8tBvXG39gNww,12456
-sparknlp/annotator/embeddings/__init__.py,sha256=od9aVMywyLf0KYBueoTeUjFbbCnh4UIuIGbsXwGtOAQ,2097
+sparknlp/annotator/embeddings/__init__.py,sha256=XQ6-UMsfvH54u3f0yceKiM8XJOAugIT3jwHE3ExoppI,2156
 sparknlp/annotator/embeddings/albert_embeddings.py,sha256=6Rd1LIn8oFIpq_ALcJh-RUjPEO7Ht8wsHY6JHSFyMkw,9995
-sparknlp/annotator/embeddings/bert_embeddings.py,sha256=uExpIlJNkQpuoZ3J_Zc2b2dV0hDNCRCAujNR4Lckly4,8369
-sparknlp/annotator/embeddings/bert_sentence_embeddings.py,sha256=XHls9qOkurwg9o6nDuwk77KSMNJmv1n4L5pcU22alWA,9054
-sparknlp/annotator/embeddings/bge_embeddings.py,sha256=FNmYxcynM1iLJvg5ZNmrZKkyIF0Gtr7G-CgZ72mrVyU,7842
+sparknlp/annotator/embeddings/bert_embeddings.py,sha256=HVUjkg56kBcpGZCo-fmPG5uatMDF3swW_lnbpy1SgSI,8463
+sparknlp/annotator/embeddings/bert_sentence_embeddings.py,sha256=NQy9KuXT9aKsTpYCR5RAeoFWI2YqEGorbdYrf_0KKmw,9148
+sparknlp/annotator/embeddings/bge_embeddings.py,sha256=hXFFd9HOru1w2L9N5YGSZlaKyxqMsZccpaI4Z8-bNUE,7919
 sparknlp/annotator/embeddings/camembert_embeddings.py,sha256=dBTXas-2Tas_JUR9Xt_GtHLcyqi_cdvT5EHRnyVrSSQ,8817
 sparknlp/annotator/embeddings/chunk_embeddings.py,sha256=WUmkJimSuFkdcLJnvcxOV0QlCLgGlhub29ZTrZb70WE,6052
 sparknlp/annotator/embeddings/deberta_embeddings.py,sha256=_b5nzLb7heFQNN-uT2oBNO6-YmM8bHmAdnGXg47HOWw,8649
 sparknlp/annotator/embeddings/distil_bert_embeddings.py,sha256=4pyMCsbvvXYeTGIMVUir9wCDKR_1f_HKtXZrTDO1Thc,9275
 sparknlp/annotator/embeddings/doc2vec.py,sha256=Xk3MdEkXatX9lRgbFbAdnIDrLgIxzUIGWFBZeo9BTq0,13226
-sparknlp/annotator/embeddings/e5_embeddings.py,sha256=dfPHCAYpayCUMxXtol0t68cDs8-JVu0M4EslimwNS0Q,7684
+sparknlp/annotator/embeddings/e5_embeddings.py,sha256=Esuvrq9JlogGaSSzFVVDkOFMwgYwFwr17I62ZiCDm0k,7858
 sparknlp/annotator/embeddings/elmo_embeddings.py,sha256=KV-KPs0Pq_OpPaHsnqBz2k_S7VdzyFZ4632IeFNKqJ8,9858
 sparknlp/annotator/embeddings/instructor_embeddings.py,sha256=CTKmbuBOx_KBM4JM-Y1U5LyR-6rrnpoBGbgGE_axS1c,8670
 sparknlp/annotator/embeddings/longformer_embeddings.py,sha256=jS4fxB5O0-d9ta9VKv8ai-17n5YHt5rML8QxUw7K4Io,8754
-sparknlp/annotator/embeddings/mpnet_embeddings.py,sha256=2sabImn5spYGzfNwBSH2zUU90Wjqrm2btCVbDbtsqPg,7796
-sparknlp/annotator/embeddings/roberta_embeddings.py,sha256=V4HGDUK2YBHhAZd1ygJEGUmxDgul0MrpKDm1UQcNqTs,9135
+sparknlp/annotator/embeddings/mpnet_embeddings.py,sha256=7d6E4lS7jjkppDPvty1UHNNrbykkriFiysrxZ_RzL0U,7875
+sparknlp/annotator/embeddings/roberta_embeddings.py,sha256=q_WHby2lDcPc5bVHkGc6X_GwT3qyDUBLUVz5ZW4HCSY,9229
 sparknlp/annotator/embeddings/roberta_sentence_embeddings.py,sha256=KVrD4z_tIU-sphK6dmbbnHBBt8-Y89C_BFQAkN99kZo,8181
 sparknlp/annotator/embeddings/sentence_embeddings.py,sha256=azuA1FKMtTJ9suwJqTEHeWHumT6kYdfURTe_1fsqcB8,5402
+sparknlp/annotator/embeddings/uae_embeddings.py,sha256=sqTT67vcegVxcyoATISLPJSmOnA6J_otB6iREKOb6e4,8794
 sparknlp/annotator/embeddings/universal_sentence_encoder.py,sha256=_fTo-K78RjxiIKptpsI32mpW87RFCdXM16epHv4RVQY,8571
 sparknlp/annotator/embeddings/word2vec.py,sha256=UBhA4qUczQOx1t82Eu51lxx1-wJ_RLnCb__ncowSNhk,13229
 sparknlp/annotator/embeddings/word_embeddings.py,sha256=CQxjx2yDdmSM9s8D-bzsbUQhT8t1cqC4ynxlf9INpMU,15388
-sparknlp/annotator/embeddings/xlm_roberta_embeddings.py,sha256=t-Bg1bQcqI_fIqUWQbHt9rHK2_tyq0YXiq3uMw4xb94,9488
+sparknlp/annotator/embeddings/xlm_roberta_embeddings.py,sha256=S2HHXOrSFXMAyloZUXJFNXL0-9wrZ32blsAhLB3Za1w,9582
 sparknlp/annotator/embeddings/xlm_roberta_sentence_embeddings.py,sha256=ojxD3H2VgDEn-RzDdCz0X485pojHBAFrlzsNemI05bY,8602
 sparknlp/annotator/embeddings/xlnet_embeddings.py,sha256=hJrlsJeO3D7uz54xiEiqqXEbq24YGuWz8U652PV9fNE,9336
 sparknlp/annotator/er/__init__.py,sha256=eF9Z-PanVfZWSVN2HSFbE7QjCDb6NYV5ESn6geYKlek,692
@@ -123,7 +127,7 @@ sparknlp/annotator/ner/ner_overwriter.py,sha256=en5OxXIP46yTXokIE96YDP9kcHA9oxiR
 sparknlp/annotator/ner/zero_shot_ner_model.py,sha256=DohhnkGSG-JxjW72t8AOx3GY7R_qT-LA3I0KF9TBz-Y,7501
 sparknlp/annotator/openai/__init__.py,sha256=u6SpV_xS8UpBE95WnTl0IefOI5TrTRl7ZHuYoeTetiA,759
 sparknlp/annotator/openai/openai_completion.py,sha256=OqDODelDAxlS66a4mAqJqXMFlEhaeiKZD4XBzR98k-g,16859
-sparknlp/annotator/openai/openai_embeddings.py,sha256=TJgd6sLfUWqJz6fd3jGfoKb-j2nrzzJbhr1S-e-71MI,109860
+sparknlp/annotator/openai/openai_embeddings.py,sha256=i1ABDRmK6vMzzWP1rVxFiWnvXG4zfrTGGDjq4lvWQeE,108802
 sparknlp/annotator/param/__init__.py,sha256=MKBZs6NWRKxrpeof3Jr4PVmoa75wyRSdWzSt0A9lpfY,750
 sparknlp/annotator/param/classifier_encoder.py,sha256=PDyOdUX2GOFVr6MLtB7RUPBdtDrzDNJNRe_r9bY5JpE,3005
 sparknlp/annotator/param/evaluation_dl_params.py,sha256=qxMP_98zaKbO1Y20yOvvarmrTCiU24VskJRo8NNI9CA,4998
@@ -135,12 +139,14 @@ sparknlp/annotator/sentence/sentence_detector_dl.py,sha256=-Osj9Bm9KyZRTAWkOsK9c
 sparknlp/annotator/sentiment/__init__.py,sha256=Lq3vKaZS1YATLMg0VNXSVtkWL5q5G9taGBvdrvSwnfg,766
 sparknlp/annotator/sentiment/sentiment_detector.py,sha256=m545NGU0Xzg_PO6_qIfpli1uZj7JQcyFgqe9R6wAPFI,8154
 sparknlp/annotator/sentiment/vivekn_sentiment.py,sha256=4rpXWDgzU6ddnbrSCp9VdLb2epCc9oZ3c6XcqxEw8nk,9655
-sparknlp/annotator/seq2seq/__init__.py,sha256=UQK-_3wLkUdW1piGudCx1_k3Tg3tERZJYOBnfMRj8pA,1011
+sparknlp/annotator/seq2seq/__init__.py,sha256=3pF-b9ubgAs8ofggiNyuc1NQseq_oe231UVjVkZWTmU,1130
 sparknlp/annotator/seq2seq/bart_transformer.py,sha256=I1flM4yeCzEAKOdQllBC30XuedxVJ7ferkFhZ6gwEbE,18481
 sparknlp/annotator/seq2seq/gpt2_transformer.py,sha256=Oz95R_NRR4tWHu_bW6Ak2832ZILXycp3ify7LfRSi8o,15310
-sparknlp/annotator/seq2seq/llama2_transformer.py,sha256=YPge5f4qfv7XZY_LoH2HRzvbZ--XoTTY_BupxxYaCd8,13862
-sparknlp/annotator/seq2seq/m2m100_transformer.py,sha256=fTFGFWaFfJt5kaLvnYknf_23PVyjBuha48asFsE_NaE,16082
+sparknlp/annotator/seq2seq/llama2_transformer.py,sha256=3LzTR0VerFdFmOizsrs2Q7HTnjELJ5WtfUgx5XnOqGM,13898
+sparknlp/annotator/seq2seq/m2m100_transformer.py,sha256=uIL9RZuuryTIdAy9TbJf9wbz6RekhW8S079bJhaB6i4,16116
 sparknlp/annotator/seq2seq/marian_transformer.py,sha256=mQ4Ylh7ZzXAOue8f-x0gqzfS3vAz3XUdD7eQ2XhcEs4,13781
+sparknlp/annotator/seq2seq/mistral_transformer.py,sha256=hq5-Emut7qYnwFolYQ6cFOEY4j5-8PdlPi2Vs72qCig,14254
+sparknlp/annotator/seq2seq/phi2_transformer.py,sha256=YuqEcvJunKKZMmfqD3thXHR5FsPbqjjwbHFExWjbDWk,13796
 sparknlp/annotator/seq2seq/t5_transformer.py,sha256=wDVxNLluIU1HGZFqaKKc4YTt4l-elPlAtQ7EEa0f5tg,17308
 sparknlp/annotator/similarity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sparknlp/annotator/similarity/document_similarity_ranker.py,sha256=OFAXEBuALFJglwThsGK8YaJ_pgW1tcevB7jVq-8SyKM,14991
@@ -182,7 +188,7 @@ sparknlp/common/read_as.py,sha256=imxPGwV7jr4Li_acbo0OAHHRGCBbYv-akzEGaBWEfcY,12
 sparknlp/common/recursive_annotator_approach.py,sha256=vqugBw22cE3Ff7PIpRlnYFuOlchgL0nM26D8j-NdpqU,1449
 sparknlp/common/storage.py,sha256=D91H3p8EIjNspjqAYu6ephRpCUtdcAir4_PrAbkIQWE,4842
 sparknlp/common/utils.py,sha256=Yne6yYcwKxhOZC-U4qfYoDhWUP_6BIaAjI5X_P_df1E,1306
-sparknlp/internal/__init__.py,sha256=g4REY_0X2Sr05szDb9681oiPqRWlT4KaOpcAOj3q32A,26496
+sparknlp/internal/__init__.py,sha256=X38S3vTHB0c4EkzczDv-J7hpJl0g6A9Xe_3u8jGJTCU,30239
 sparknlp/internal/annotator_java_ml.py,sha256=UGPoThG0rGXUOXGSQnDzEDW81Mu1s5RPF29v7DFyE3c,1187
 sparknlp/internal/annotator_transformer.py,sha256=fXmc2IWXGybqZpbEU9obmbdBYPc798y42zvSB4tqV9U,1448
 sparknlp/internal/extended_java_wrapper.py,sha256=hwP0133-hDiDf5sBF-P3MtUsuuDj1PpQbtGZQIRwzfk,2240
@@ -192,7 +198,7 @@ sparknlp/logging/__init__.py,sha256=DoROFF5KLZe4t4Q-OHxqk1nhqbw9NQ-wb64y8icNwgw,
 sparknlp/logging/comet.py,sha256=_ZBi9-hlilCAnd4lvdYMWiq4Vqsppv8kow3k0cf-NG4,15958
 sparknlp/pretrained/__init__.py,sha256=GV-x9UBK8F2_IR6zYatrzFcVJtkSUIMbxqWsxRUePmQ,793
 sparknlp/pretrained/pretrained_pipeline.py,sha256=lquxiaABuA68Rmu7csamJPqBoRJqMUO0oNHsmEZDAIs,5740
-sparknlp/pretrained/resource_downloader.py,sha256=XKnx9Mu_K3R7Quj2X1EHVUzY5fJ6rvVnK-JChrWPaRY,7820
+sparknlp/pretrained/resource_downloader.py,sha256=8_-rpvO2LsX_Lq4wMPif2ca3RlJZWEabt8pDm2xymiI,7806
 sparknlp/pretrained/utils.py,sha256=T1MrvW_DaWk_jcOjVLOea0NMFE9w8fe0ZT_5urZ_nEY,1099
 sparknlp/training/__init__.py,sha256=qREi9u-5Vc2VjpL6-XZsyvu5jSEIdIhowW7_kKaqMqo,852
 sparknlp/training/conll.py,sha256=wKBiSTrjc6mjsl7Nyt6B8f4yXsDJkZb-sn8iOjix9cE,6961
@@ -224,8 +230,8 @@ sparknlp/training/_tf_graph_builders_1x/ner_dl/dataset_encoder.py,sha256=R4yHFN3
 sparknlp/training/_tf_graph_builders_1x/ner_dl/ner_model.py,sha256=EoCSdcIjqQ3wv13MAuuWrKV8wyVBP0SbOEW41omHlR0,23189
 sparknlp/training/_tf_graph_builders_1x/ner_dl/ner_model_saver.py,sha256=k5CQ7gKV6HZbZMB8cKLUJuZxoZWlP_DFWdZ--aIDwsc,2356
 sparknlp/training/_tf_graph_builders_1x/ner_dl/sentence_grouper.py,sha256=pAxjWhjazSX8Vg0MFqJiuRVw1IbnQNSs-8Xp26L4nko,870
-spark_nlp-5.3.2.dist-info/.uuid,sha256=1f6hF51aIuv9yCvh31NU9lOpS34NE-h3a0Et7R9yR6A,36
-spark_nlp-5.3.2.dist-info/METADATA,sha256=lvuYdEbmUUpC9QYY4YIfuMhBUIhy5axZrrX3XgPc0uQ,57087
-spark_nlp-5.3.2.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-spark_nlp-5.3.2.dist-info/top_level.txt,sha256=uuytur4pyMRw2H_txNY2ZkaucZHUs22QF8-R03ch_-E,13
-spark_nlp-5.3.2.dist-info/RECORD,,
+spark_nlp-5.4.0.dist-info/.uuid,sha256=1f6hF51aIuv9yCvh31NU9lOpS34NE-h3a0Et7R9yR6A,36
+spark_nlp-5.4.0.dist-info/METADATA,sha256=fzEL08vmQeHH_Y9OCF3QfU_CWtGtWk5bexyFxXOGoSs,55595
+spark_nlp-5.4.0.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+spark_nlp-5.4.0.dist-info/top_level.txt,sha256=uuytur4pyMRw2H_txNY2ZkaucZHUs22QF8-R03ch_-E,13
+spark_nlp-5.4.0.dist-info/RECORD,,

sparknlp/__init__.py

@@ -58,6 +58,7 @@ sys.modules['com.johnsnowlabs.nlp.annotators.er'] = annotator
 sys.modules['com.johnsnowlabs.nlp.annotators.coref'] = annotator
 sys.modules['com.johnsnowlabs.nlp.annotators.cv'] = annotator
 sys.modules['com.johnsnowlabs.nlp.annotators.audio'] = annotator
+sys.modules['com.johnsnowlabs.ml.ai'] = annotator
 
 annotators = annotator
 embeddings = annotator
@@ -128,7 +129,7 @@ def start(gpu=False,
         The initiated Spark session.
 
     """
-    current_version = "5.3.2"
+    current_version = "5.4.0"
 
     if params is None:
         params = {}
@@ -309,4 +310,4 @@ def version():
     str
         The current Spark NLP version.
     """
-    return '5.3.2'
+    return '5.4.0'

sparknlp/annotator/classifier_dl/__init__.py

@@ -51,3 +51,4 @@ from sparknlp.annotator.classifier_dl.bart_for_zero_shot_classification import *
 from sparknlp.annotator.classifier_dl.deberta_for_zero_shot_classification import *
 from sparknlp.annotator.classifier_dl.mpnet_for_sequence_classification import *
 from sparknlp.annotator.classifier_dl.mpnet_for_question_answering import *
+from sparknlp.annotator.classifier_dl.mpnet_for_token_classification import *