spark-nlp 5.3.1__py2.py3-none-any.whl → 5.3.3__py2.py3-none-any.whl

spark_nlp-5.3.3.dist-info/.uuid

@@ -0,0 +1 @@
+90f78083-0ee0-43e9-8240-7263731b6707

{spark_nlp-5.3.1.dist-info → spark_nlp-5.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spark-nlp
-Version: 5.3.1
+Version: 5.3.3
 Summary: John Snow Labs Spark NLP is a natural language processing library built on top of Apache Spark ML. It provides simple, performant & accurate NLP annotations for machine learning pipelines, that scale easily in a distributed environment.
 Home-page: https://github.com/JohnSnowLabs/spark-nlp
 Author: John Snow Labs
@@ -197,7 +197,7 @@ To use Spark NLP you need the following requirements:
 
 **GPU (optional):**
 
-Spark NLP 5.3.1 is built with ONNX 1.17.0 and TensorFlow 2.7.1 deep learning engines. The minimum following NVIDIA® software are only required for GPU support:
+Spark NLP 5.3.3 is built with ONNX 1.17.0 and TensorFlow 2.7.1 deep learning engines. The minimum following NVIDIA® software are only required for GPU support:
 
 - NVIDIA® GPU drivers version 450.80.02 or higher
 - CUDA® Toolkit 11.2
@@ -213,7 +213,7 @@ $ java -version
 $ conda create -n sparknlp python=3.7 -y
 $ conda activate sparknlp
 # spark-nlp by default is based on pyspark 3.x
-$ pip install spark-nlp==5.3.1 pyspark==3.3.1
+$ pip install spark-nlp==5.3.3 pyspark==3.3.1
 ```
 
 In Python console or Jupyter `Python3` kernel:
@@ -258,7 +258,7 @@ For more examples, you can visit our dedicated [examples](https://github.com/Joh
 
 ## Apache Spark Support
 
-Spark NLP *5.3.1* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x
+Spark NLP *5.3.3* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x
 
 | Spark NLP | Apache Spark 3.5.x | Apache Spark 3.4.x | Apache Spark 3.3.x | Apache Spark 3.2.x | Apache Spark 3.1.x | Apache Spark 3.0.x | Apache Spark 2.4.x | Apache Spark 2.3.x |
 |-----------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|
@@ -302,7 +302,7 @@ Find out more about `Spark NLP` versions from our [release notes](https://github
 
 ## Databricks Support
 
-Spark NLP 5.3.1 has been tested and is compatible with the following runtimes:
+Spark NLP 5.3.3 has been tested and is compatible with the following runtimes:
 
 **CPU:**
 
@@ -375,7 +375,7 @@ Spark NLP 5.3.1 has been tested and is compatible with the following runtimes:
 
 ## EMR Support
 
-Spark NLP 5.3.1 has been tested and is compatible with the following EMR releases:
+Spark NLP 5.3.3 has been tested and is compatible with the following EMR releases:
 
 - emr-6.2.0
 - emr-6.3.0
@@ -425,11 +425,11 @@ Spark NLP supports all major releases of Apache Spark 3.0.x, Apache Spark 3.1.x,
 ```sh
 # CPU
 
-spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1
+spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3
 
-pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1
+pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3
 
-spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1
+spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3
 ```
 
 The `spark-nlp` has been published to
@@ -438,11 +438,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s
 ```sh
 # GPU
 
-spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1
+spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.3
 
-pyspark --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1
+pyspark --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.3
 
-spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1
+spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.3
 
 ```
 
@@ -452,11 +452,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s
 ```sh
 # AArch64
 
-spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1
+spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.3
 
-pyspark --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1
+pyspark --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.3
 
-spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1
+spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.3
 
 ```
 
@@ -466,11 +466,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s
 ```sh
 # M1/M2 (Apple Silicon)
 
-spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1
+spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.3
 
-pyspark --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1
+pyspark --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.3
 
-spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1
+spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.3
 
 ```
 
@@ -484,7 +484,7 @@ set in your SparkSession:
 spark-shell \
   --driver-memory 16g \
   --conf spark.kryoserializer.buffer.max=2000M \
-  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1
+  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3
 ```
 
 ## Scala
@@ -502,7 +502,7 @@ coordinates:
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp_2.12</artifactId>
-    <version>5.3.1</version>
+    <version>5.3.3</version>
 </dependency>
 ```
 
@@ -513,7 +513,7 @@ coordinates:
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp-gpu_2.12</artifactId>
-    <version>5.3.1</version>
+    <version>5.3.3</version>
 </dependency>
 ```
 
@@ -524,7 +524,7 @@ coordinates:
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp-aarch64_2.12</artifactId>
-    <version>5.3.1</version>
+    <version>5.3.3</version>
 </dependency>
 ```
 
@@ -535,7 +535,7 @@ coordinates:
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp-silicon_2.12</artifactId>
-    <version>5.3.1</version>
+    <version>5.3.3</version>
 </dependency>
 ```
 
@@ -545,28 +545,28 @@ coordinates:
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.1"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.3"
 ```
 
 **spark-nlp-gpu:**
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-gpu
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.1"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.3"
 ```
 
 **spark-nlp-aarch64:**
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-aarch64
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.1"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.3"
 ```
 
 **spark-nlp-silicon:**
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-silicon
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.1"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.3"
 ```
 
 Maven
@@ -588,7 +588,7 @@ If you installed pyspark through pip/conda, you can install `spark-nlp` through
 Pip:
 
 ```bash
-pip install spark-nlp==5.3.1
+pip install spark-nlp==5.3.3
 ```
 
 Conda:
@@ -617,7 +617,7 @@ spark = SparkSession.builder
     .config("spark.driver.memory", "16G")
     .config("spark.driver.maxResultSize", "0")
     .config("spark.kryoserializer.buffer.max", "2000M")
-    .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1")
+    .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3")
     .getOrCreate()
 ```
 
@@ -688,7 +688,7 @@ Use either one of the following options
 - Add the following Maven Coordinates to the interpreter's library list
 
 ```bash
-com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1
+com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3
 ```
 
 - Add a path to pre-built jar from [here](#compiled-jars) in the interpreter's library list making sure the jar is
@@ -699,7 +699,7 @@ com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1
 Apart from the previous step, install the python module through pip
 
 ```bash
-pip install spark-nlp==5.3.1
+pip install spark-nlp==5.3.3
 ```
 
 Or you can install `spark-nlp` from inside Zeppelin by using Conda:
@@ -727,7 +727,7 @@ launch the Jupyter from the same Python environment:
 $ conda create -n sparknlp python=3.8 -y
 $ conda activate sparknlp
 # spark-nlp by default is based on pyspark 3.x
-$ pip install spark-nlp==5.3.1 pyspark==3.3.1 jupyter
+$ pip install spark-nlp==5.3.3 pyspark==3.3.1 jupyter
 $ jupyter notebook
 ```
 
@@ -744,7 +744,7 @@ export PYSPARK_PYTHON=python3
 export PYSPARK_DRIVER_PYTHON=jupyter
 export PYSPARK_DRIVER_PYTHON_OPTS=notebook
 
-pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1
+pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3
 ```
 
 Alternatively, you can mix in using `--jars` option for pyspark + `pip install spark-nlp`
@@ -771,7 +771,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi
 # -s is for spark-nlp
 # -g will enable upgrading libcudnn8 to 8.1.0 on Google Colab for GPU usage
 # by default they are set to the latest
-!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.1
+!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.3
 ```
 
 [Spark NLP quick start on Google Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/quick_start_google_colab.ipynb)
@@ -794,7 +794,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi
 # -s is for spark-nlp
 # -g will enable upgrading libcudnn8 to 8.1.0 on Kaggle for GPU usage
 # by default they are set to the latest
-!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.1
+!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.3
 ```
 
 [Spark NLP quick start on Kaggle Kernel](https://www.kaggle.com/mozzie/spark-nlp-named-entity-recognition) is a live
@@ -813,9 +813,9 @@ demo on Kaggle Kernel that performs named entity recognitions by using Spark NLP
 
 3. In `Libraries` tab inside your cluster you need to follow these steps:
 
-   3.1. Install New -> PyPI -> `spark-nlp==5.3.1` -> Install
+   3.1. Install New -> PyPI -> `spark-nlp==5.3.3` -> Install
 
-   3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1` -> Install
+   3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3` -> Install
 
 4. Now you can attach your notebook to the cluster and use Spark NLP!
 
@@ -866,7 +866,7 @@ A sample of your software configuration in JSON on S3 (must be public access):
       "spark.kryoserializer.buffer.max": "2000M",
       "spark.serializer": "org.apache.spark.serializer.KryoSerializer",
       "spark.driver.maxResultSize": "0",
-      "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1"
+      "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3"
     }
 }]
 ```
@@ -875,7 +875,7 @@ A sample of AWS CLI to launch EMR cluster:
 
 ```.sh
 aws emr create-cluster \
---name "Spark NLP 5.3.1" \
+--name "Spark NLP 5.3.3" \
 --release-label emr-6.2.0 \
 --applications Name=Hadoop Name=Spark Name=Hive \
 --instance-type m4.4xlarge \
@@ -939,7 +939,7 @@ gcloud dataproc clusters create ${CLUSTER_NAME} \
   --enable-component-gateway \
   --metadata 'PIP_PACKAGES=spark-nlp spark-nlp-display google-cloud-bigquery google-cloud-storage' \
   --initialization-actions gs://goog-dataproc-initialization-actions-${REGION}/python/pip-install.sh \
-  --properties spark:spark.serializer=org.apache.spark.serializer.KryoSerializer,spark:spark.driver.maxResultSize=0,spark:spark.kryoserializer.buffer.max=2000M,spark:spark.jars.packages=com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1
+  --properties spark:spark.serializer=org.apache.spark.serializer.KryoSerializer,spark:spark.driver.maxResultSize=0,spark:spark.kryoserializer.buffer.max=2000M,spark:spark.jars.packages=com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3
 ```
 
 2. On an existing one, you need to install spark-nlp and spark-nlp-display packages from PyPI.
@@ -982,7 +982,7 @@ spark = SparkSession.builder
     .config("spark.kryoserializer.buffer.max", "2000m")
     .config("spark.jsl.settings.pretrained.cache_folder", "sample_data/pretrained")
     .config("spark.jsl.settings.storage.cluster_tmp_dir", "sample_data/storage")
-    .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1")
+    .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3")
     .getOrCreate()
 ```
 
@@ -996,7 +996,7 @@ spark-shell \
   --conf spark.kryoserializer.buffer.max=2000M \
   --conf spark.jsl.settings.pretrained.cache_folder="sample_data/pretrained" \
   --conf spark.jsl.settings.storage.cluster_tmp_dir="sample_data/storage" \
-  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1
+  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3
 ```
 
 **pyspark:**
@@ -1009,7 +1009,7 @@ pyspark \
   --conf spark.kryoserializer.buffer.max=2000M \
   --conf spark.jsl.settings.pretrained.cache_folder="sample_data/pretrained" \
   --conf spark.jsl.settings.storage.cluster_tmp_dir="sample_data/storage" \
-  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1
+  --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.3
 ```
 
 **Databricks:**
@@ -1281,7 +1281,7 @@ spark = SparkSession.builder
     .config("spark.driver.memory", "16G")
     .config("spark.driver.maxResultSize", "0")
     .config("spark.kryoserializer.buffer.max", "2000M")
-    .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.1.jar")
+    .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.3.jar")
     .getOrCreate()
 ```
 
@@ -1290,7 +1290,7 @@ spark = SparkSession.builder
   version (3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x)
 - If you are local, you can load the Fat JAR from your local FileSystem, however, if you are in a cluster setup you need
   to put the Fat JAR on a distributed FileSystem such as HDFS, DBFS, S3, etc. (
-  i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.1.jar`)
+  i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.3.jar`)
 
 Example of using pretrained Models and Pipelines in offline:
 

{spark_nlp-5.3.1.dist-info → spark_nlp-5.3.3.dist-info}/RECORD

@@ -1,7 +1,7 @@
 com/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 com/johnsnowlabs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 com/johnsnowlabs/nlp/__init__.py,sha256=DPIVXtONO5xXyOk-HB0-sNiHAcco17NN13zPS_6Uw8c,294
-sparknlp/__init__.py,sha256=fP6mNHdeh0JvNOydT4WGsALtBOM5HLWx5Kz9MplSS8s,13588
+sparknlp/__init__.py,sha256=ZUkW_iY3tWQwa5XvLKprnbvY0_hTCOHJSYWb-KNrvmE,13588
 sparknlp/annotation.py,sha256=I5zOxG5vV2RfPZfqN9enT1i4mo6oBcn3Lrzs37QiOiA,5635
 sparknlp/annotation_audio.py,sha256=iRV_InSVhgvAwSRe9NTbUH9v6OGvTM-FPCpSAKVu0mE,1917
 sparknlp/annotation_image.py,sha256=xhCe8Ko-77XqWVuuYHFrjKqF6zPd8Z-RY_rmZXNwCXU,2547
@@ -78,7 +78,7 @@ sparknlp/annotator/cv/vit_for_image_classification.py,sha256=D2V3pxAd3rBi1817lxV
 sparknlp/annotator/dependency/__init__.py,sha256=eV43oXAGaYl2N1XKIEAAZJLNP8gpHm8VxuXDeDlQzR4,774
 sparknlp/annotator/dependency/dependency_parser.py,sha256=SxyvHPp8Hs1Xnm5X1nLTMi095XoQMtfL8pbys15mYAI,11212
 sparknlp/annotator/dependency/typed_dependency_parser.py,sha256=60vPdYkbFk9MPGegg3m9Uik9cMXpMZd8tBvXG39gNww,12456
-sparknlp/annotator/embeddings/__init__.py,sha256=od9aVMywyLf0KYBueoTeUjFbbCnh4UIuIGbsXwGtOAQ,2097
+sparknlp/annotator/embeddings/__init__.py,sha256=XQ6-UMsfvH54u3f0yceKiM8XJOAugIT3jwHE3ExoppI,2156
 sparknlp/annotator/embeddings/albert_embeddings.py,sha256=6Rd1LIn8oFIpq_ALcJh-RUjPEO7Ht8wsHY6JHSFyMkw,9995
 sparknlp/annotator/embeddings/bert_embeddings.py,sha256=uExpIlJNkQpuoZ3J_Zc2b2dV0hDNCRCAujNR4Lckly4,8369
 sparknlp/annotator/embeddings/bert_sentence_embeddings.py,sha256=XHls9qOkurwg9o6nDuwk77KSMNJmv1n4L5pcU22alWA,9054
@@ -96,6 +96,7 @@ sparknlp/annotator/embeddings/mpnet_embeddings.py,sha256=2sabImn5spYGzfNwBSH2zUU
 sparknlp/annotator/embeddings/roberta_embeddings.py,sha256=V4HGDUK2YBHhAZd1ygJEGUmxDgul0MrpKDm1UQcNqTs,9135
 sparknlp/annotator/embeddings/roberta_sentence_embeddings.py,sha256=KVrD4z_tIU-sphK6dmbbnHBBt8-Y89C_BFQAkN99kZo,8181
 sparknlp/annotator/embeddings/sentence_embeddings.py,sha256=azuA1FKMtTJ9suwJqTEHeWHumT6kYdfURTe_1fsqcB8,5402
+sparknlp/annotator/embeddings/uae_embeddings.py,sha256=sqTT67vcegVxcyoATISLPJSmOnA6J_otB6iREKOb6e4,8794
 sparknlp/annotator/embeddings/universal_sentence_encoder.py,sha256=_fTo-K78RjxiIKptpsI32mpW87RFCdXM16epHv4RVQY,8571
 sparknlp/annotator/embeddings/word2vec.py,sha256=UBhA4qUczQOx1t82Eu51lxx1-wJ_RLnCb__ncowSNhk,13229
 sparknlp/annotator/embeddings/word_embeddings.py,sha256=CQxjx2yDdmSM9s8D-bzsbUQhT8t1cqC4ynxlf9INpMU,15388
@@ -182,7 +183,7 @@ sparknlp/common/read_as.py,sha256=imxPGwV7jr4Li_acbo0OAHHRGCBbYv-akzEGaBWEfcY,12
 sparknlp/common/recursive_annotator_approach.py,sha256=vqugBw22cE3Ff7PIpRlnYFuOlchgL0nM26D8j-NdpqU,1449
 sparknlp/common/storage.py,sha256=D91H3p8EIjNspjqAYu6ephRpCUtdcAir4_PrAbkIQWE,4842
 sparknlp/common/utils.py,sha256=Yne6yYcwKxhOZC-U4qfYoDhWUP_6BIaAjI5X_P_df1E,1306
-sparknlp/internal/__init__.py,sha256=g4REY_0X2Sr05szDb9681oiPqRWlT4KaOpcAOj3q32A,26496
+sparknlp/internal/__init__.py,sha256=ymZxTXlIf6e_wWEBCVI727zq2EP4nD5z97BWmJDuKlo,26725
 sparknlp/internal/annotator_java_ml.py,sha256=UGPoThG0rGXUOXGSQnDzEDW81Mu1s5RPF29v7DFyE3c,1187
 sparknlp/internal/annotator_transformer.py,sha256=fXmc2IWXGybqZpbEU9obmbdBYPc798y42zvSB4tqV9U,1448
 sparknlp/internal/extended_java_wrapper.py,sha256=hwP0133-hDiDf5sBF-P3MtUsuuDj1PpQbtGZQIRwzfk,2240
@@ -192,7 +193,7 @@ sparknlp/logging/__init__.py,sha256=DoROFF5KLZe4t4Q-OHxqk1nhqbw9NQ-wb64y8icNwgw,
 sparknlp/logging/comet.py,sha256=_ZBi9-hlilCAnd4lvdYMWiq4Vqsppv8kow3k0cf-NG4,15958
 sparknlp/pretrained/__init__.py,sha256=GV-x9UBK8F2_IR6zYatrzFcVJtkSUIMbxqWsxRUePmQ,793
 sparknlp/pretrained/pretrained_pipeline.py,sha256=lquxiaABuA68Rmu7csamJPqBoRJqMUO0oNHsmEZDAIs,5740
-sparknlp/pretrained/resource_downloader.py,sha256=XKnx9Mu_K3R7Quj2X1EHVUzY5fJ6rvVnK-JChrWPaRY,7820
+sparknlp/pretrained/resource_downloader.py,sha256=8_-rpvO2LsX_Lq4wMPif2ca3RlJZWEabt8pDm2xymiI,7806
 sparknlp/pretrained/utils.py,sha256=T1MrvW_DaWk_jcOjVLOea0NMFE9w8fe0ZT_5urZ_nEY,1099
 sparknlp/training/__init__.py,sha256=qREi9u-5Vc2VjpL6-XZsyvu5jSEIdIhowW7_kKaqMqo,852
 sparknlp/training/conll.py,sha256=wKBiSTrjc6mjsl7Nyt6B8f4yXsDJkZb-sn8iOjix9cE,6961
@@ -224,7 +225,8 @@ sparknlp/training/_tf_graph_builders_1x/ner_dl/dataset_encoder.py,sha256=R4yHFN3
 sparknlp/training/_tf_graph_builders_1x/ner_dl/ner_model.py,sha256=EoCSdcIjqQ3wv13MAuuWrKV8wyVBP0SbOEW41omHlR0,23189
 sparknlp/training/_tf_graph_builders_1x/ner_dl/ner_model_saver.py,sha256=k5CQ7gKV6HZbZMB8cKLUJuZxoZWlP_DFWdZ--aIDwsc,2356
 sparknlp/training/_tf_graph_builders_1x/ner_dl/sentence_grouper.py,sha256=pAxjWhjazSX8Vg0MFqJiuRVw1IbnQNSs-8Xp26L4nko,870
-spark_nlp-5.3.1.dist-info/METADATA,sha256=cfK1KW9iG7FnwuiQH9bBTakLsWE7H_1zHTnMPOICjE8,57087
-spark_nlp-5.3.1.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-spark_nlp-5.3.1.dist-info/top_level.txt,sha256=uuytur4pyMRw2H_txNY2ZkaucZHUs22QF8-R03ch_-E,13
-spark_nlp-5.3.1.dist-info/RECORD,,
+spark_nlp-5.3.3.dist-info/.uuid,sha256=1f6hF51aIuv9yCvh31NU9lOpS34NE-h3a0Et7R9yR6A,36
+spark_nlp-5.3.3.dist-info/METADATA,sha256=YSJq8MiAoRizhOjb8zUeMBqNzNAL1rDEVW5MWy_Q37c,57087
+spark_nlp-5.3.3.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+spark_nlp-5.3.3.dist-info/top_level.txt,sha256=uuytur4pyMRw2H_txNY2ZkaucZHUs22QF8-R03ch_-E,13
+spark_nlp-5.3.3.dist-info/RECORD,,

sparknlp/__init__.py

@@ -128,7 +128,7 @@ def start(gpu=False,
         The initiated Spark session.
 
     """
-    current_version = "5.3.1"
+    current_version = "5.3.3"
 
     if params is None:
         params = {}
@@ -309,4 +309,4 @@ def version():
     str
         The current Spark NLP version.
     """
-    return '5.3.1'
+    return '5.3.3'

sparknlp/annotator/embeddings/__init__.py

@@ -36,3 +36,4 @@ from sparknlp.annotator.embeddings.xlm_roberta_embeddings import *
 from sparknlp.annotator.embeddings.xlm_roberta_sentence_embeddings import *
 from sparknlp.annotator.embeddings.xlnet_embeddings import *
 from sparknlp.annotator.embeddings.bge_embeddings import *
+from sparknlp.annotator.embeddings.uae_embeddings import *

sparknlp/annotator/embeddings/uae_embeddings.py

@@ -0,0 +1,211 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for UAEEmbeddings."""
+
+from sparknlp.common import *
+
+
+class UAEEmbeddings(AnnotatorModel,
+                    HasEmbeddingsProperties,
+                    HasCaseSensitiveProperties,
+                    HasStorageRef,
+                    HasBatchedAnnotate,
+                    HasMaxSentenceLengthLimit):
+    """Sentence embeddings using Universal AnglE Embedding (UAE).
+
+    UAE is a novel angle-optimized text embedding model, designed to improve semantic textual
+    similarity tasks, which are crucial for Large Language Model (LLM) applications. By
+    introducing angle optimization in a complex space, AnglE effectively mitigates saturation of
+    the cosine similarity function.
+
+    Pretrained models can be loaded with :meth:`.pretrained` of the companion
+    object:
+
+    >>> embeddings = UAEEmbeddings.pretrained() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("UAE_embeddings")
+
+
+    The default model is ``"uae_large_v1"``, if no name is provided.
+
+    For available pretrained models please see the
+    `Models Hub <https://sparknlp.org/models?q=UAE>`__.
+
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``            ``SENTENCE_EMBEDDINGS``
+    ====================== ======================
+
+    Parameters
+    ----------
+    batchSize
+        Size of every batch , by default 8
+    dimension
+        Number of embedding dimensions, by default 768
+    caseSensitive
+        Whether to ignore case in tokens for embeddings matching, by default False
+    maxSentenceLength
+        Max sentence length to process, by default 512
+    configProtoBytes
+        ConfigProto from tensorflow, serialized into byte array.
+
+    References
+    ----------
+
+    `AnglE-optimized Text Embeddings <https://arxiv.org/abs/2309.12871>`__
+    `UAE Github Repository <https://github.com/baochi0212/uae-embedding>`__
+
+    **Paper abstract**
+
+    *High-quality text embedding is pivotal in improving semantic textual similarity (STS) tasks,
+    which are crucial components in Large Language Model (LLM) applications. However, a common
+    challenge existing text embedding models face is the problem of vanishing gradients, primarily
+    due to their reliance on the cosine function in the optimization objective, which has
+    saturation zones. To address this issue, this paper proposes a novel angle-optimized text
+    embedding model called AnglE. The core idea of AnglE is to introduce angle optimization in a
+    complex space. This novel approach effectively mitigates the adverse effects of the saturation
+    zone in the cosine function, which can impede gradient and hinder optimization processes. To
+    set up a comprehensive STS evaluation, we experimented on existing short-text STS datasets and
+    a newly collected long-text STS dataset from GitHub Issues. Furthermore, we examine
+    domain-specific STS scenarios with limited labeled data and explore how AnglE works with
+    LLM-annotated data. Extensive experiments were conducted on various tasks including short-text
+    STS, long-text STS, and domain-specific STS tasks. The results show that AnglE outperforms the
+    state-of-the-art (SOTA) STS models that ignore the cosine saturation zone. These findings
+    demonstrate the ability of AnglE to generate high-quality text embeddings and the usefulness
+    of angle optimization in STS.*
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> embeddings = UAEEmbeddings.pretrained() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("embeddings")
+    >>> embeddingsFinisher = EmbeddingsFinisher() \\
+    ...     .setInputCols("embeddings") \\
+    ...     .setOutputCols("finished_embeddings") \\
+    ...     .setOutputAsVector(True)
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     embeddings,
+    ...     embeddingsFinisher
+    ... ])
+    >>> data = spark.createDataFrame([["hello world", "hello moon"]]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("explode(finished_embeddings) as result").show(5, 80)
+    +--------------------------------------------------------------------------------+
+    |                                                                          result|
+    +--------------------------------------------------------------------------------+
+    |[0.50387806, 0.5861606, 0.35129607, -0.76046336, -0.32446072, -0.117674336, 0...|
+    |[0.6660665, 0.961762, 0.24854276, -0.1018044, -0.6569202, 0.027635604, 0.1915...|
+    +--------------------------------------------------------------------------------+
+    """
+
+    name = "UAEEmbeddings"
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.SENTENCE_EMBEDDINGS
+    poolingStrategy = Param(Params._dummy(),
+                            "poolingStrategy",
+                            "Pooling strategy to use for sentence embeddings",
+                            TypeConverters.toString)
+
+    def setPoolingStrategy(self, value):
+        """Pooling strategy to use for sentence embeddings.
+
+        Available pooling strategies for sentence embeddings are:
+          - `"cls"`: leading `[CLS]` token
+          - `"cls_avg"`: leading `[CLS]` token + mean of all other tokens
+          - `"last"`: embeddings of the last token in the sequence
+          - `"avg"`: mean of all tokens
+          - `"max"`: max of all embedding features of the entire token sequence
+          - `"int"`: An integer number, which represents the index of the token to use as the
+            embedding
+
+        Parameters
+        ----------
+        value : str
+            Pooling strategy to use for sentence embeddings
+        """
+
+        valid_strategies = {"cls", "cls_avg", "last", "avg", "max"}
+        if value in valid_strategies or value.isdigit():
+            return self._set(poolingStrategy=value)
+        else:
+            raise ValueError(f"Invalid pooling strategy: {value}. "
+                             f"Valid strategies are: {', '.join(self.valid_strategies)} or an integer.")
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.nlp.embeddings.UAEEmbeddings", java_model=None):
+        super(UAEEmbeddings, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+        self._setDefault(
+            dimension=1024,
+            batchSize=8,
+            maxSentenceLength=512,
+            caseSensitive=False,
+            poolingStrategy="cls"
+        )
+
+    @staticmethod
+    def loadSavedModel(folder, spark_session):
+        """Loads a locally saved model.
+
+        Parameters
+        ----------
+        folder : str
+            Folder of the saved model
+        spark_session : pyspark.sql.SparkSession
+            The current SparkSession
+
+        Returns
+        -------
+        UAEEmbeddings
+            The restored model
+        """
+        from sparknlp.internal import _UAEEmbeddingsLoader
+        jModel = _UAEEmbeddingsLoader(folder, spark_session._jsparkSession)._java_obj
+        return UAEEmbeddings(java_model=jModel)
+
+    @staticmethod
+    def pretrained(name="uae_large_v1", lang="en", remote_loc=None):
+        """Downloads and loads a pretrained model.
+
+        Parameters
+        ----------
+        name : str, optional
+            Name of the pretrained model, by default "UAE_small"
+        lang : str, optional
+            Language of the pretrained model, by default "en"
+        remote_loc : str, optional
+            Optional remote address of the resource, by default None. Will use
+            Spark NLPs repositories otherwise.
+
+        Returns
+        -------
+        UAEEmbeddings
+            The restored model
+        """
+        from sparknlp.pretrained import ResourceDownloader
+        return ResourceDownloader.downloadModel(UAEEmbeddings, name, lang, remote_loc)

sparknlp/internal/__init__.py

@@ -158,11 +158,13 @@ class _GPT2Loader(ExtendedJavaWrapper):
         super(_GPT2Loader, self).__init__(
             "com.johnsnowlabs.nlp.annotators.seq2seq.GPT2Transformer.loadSavedModel", path, jspark)
 
+
 class _LLAMA2Loader(ExtendedJavaWrapper):
     def __init__(self, path, jspark):
         super(_LLAMA2Loader, self).__init__(
             "com.johnsnowlabs.nlp.annotators.seq2seq.LLAMA2Transformer.loadSavedModel", path, jspark)
 
+
 class _LongformerLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark):
         super(_LongformerLoader, self).__init__("com.johnsnowlabs.nlp.embeddings.LongformerEmbeddings.loadSavedModel",
@@ -601,8 +603,8 @@ class _DeBertaForZeroShotClassification(ExtendedJavaWrapper):
         super(_DeBertaForZeroShotClassification, self).__init__(
             "com.johnsnowlabs.nlp.annotators.classifier.dl.DeBertaForZeroShotClassification.loadSavedModel", path,
             jspark)
-        
-        
+
+
 class _MPNetForSequenceClassificationLoader(ExtendedJavaWrapper):
     def __init__(self, path, jspark):
         super(_MPNetForSequenceClassificationLoader, self).__init__(
@@ -615,3 +617,10 @@ class _MPNetForQuestionAnsweringLoader(ExtendedJavaWrapper):
         super(_MPNetForQuestionAnsweringLoader, self).__init__(
             "com.johnsnowlabs.nlp.annotators.classifier.dl.MPNetForQuestionAnswering.loadSavedModel", path,
             jspark)
+
+
+class _UAEEmbeddingsLoader(ExtendedJavaWrapper):
+    def __init__(self, path, jspark):
+        super(_UAEEmbeddingsLoader, self).__init__(
+            "com.johnsnowlabs.nlp.embeddings.UAEEmbeddings.loadSavedModel", path,
+            jspark)

sparknlp/pretrained/resource_downloader.py

@@ -58,7 +58,6 @@ class ResourceDownloader(object):
 
     """
 
-
     @staticmethod
     def downloadModel(reader, name, language, remote_loc=None, j_dwn='PythonResourceDownloader'):
         """Downloads and loads a model with the default downloader. Usually this method
@@ -67,8 +66,8 @@ class ResourceDownloader(object):
 
         Parameters
         ----------
-        reader : str
-            Name of the class to read the model for
+        reader : obj
+           Class to read the model for
         name : str
             Name of the pretrained model
         language : str