spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/util.py

@@ -1,4 +1,23 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains various utilities."""
+
+
 import sparknlp.internal as _internal
+import numpy as np
+from pyspark.sql import Row
+from pyspark.sql.types import StructType, StructField, StringType, IntegerType, BinaryType
 
 
 def get_config_path():
@@ -7,9 +26,36 @@ def get_config_path():
 
 class CoNLLGenerator:
     @staticmethod
-    def exportConllFiles(spark, files_path, pipeline, output_path):
-        _internal._CoNLLGeneratorExport(spark, files_path, pipeline, output_path).apply()
+    def exportConllFiles(*args):
+        num_args = len(args)
+        if num_args == 2:
+            _internal._CoNLLGeneratorExportFromDataFrame(*args).apply()
+        elif num_args == 3:
+            _internal._CoNLLGeneratorExportFromDataFrameAndField(*args).apply()
+        elif num_args == 4:
+            _internal._CoNLLGeneratorExportFromTargetAndPipeline(*args).apply()
+        else:
+            raise NotImplementedError(f"No exportConllFiles alternative takes {num_args} parameters")
 
-    @staticmethod
-    def exportConllFiles(dataframe, output_path):
-        _internal._CoNLLGeneratorExport(dataframe, output_path).apply()
+
+class EmbeddingsDataFrameUtils:
+    """
+    Utility for creating DataFrames compatible with multimodal embedding models (e.g., E5VEmbeddings) for text-only scenarios.
+    Provides:
+      - imageSchema: the expected schema for Spark image DataFrames
+      - emptyImageRow: a dummy image row for text-only embedding
+    """
+    imageSchema = StructType([
+        StructField(
+            "image",
+            StructType([
+                StructField("origin", StringType(), True),
+                StructField("height", IntegerType(), True),
+                StructField("width", IntegerType(), True),
+                StructField("nChannels", IntegerType(), True),
+                StructField("mode", IntegerType(), True),
+                StructField("data", BinaryType(), True),
+            ]),
+        )
+    ])
+    emptyImageRow = Row(Row("", 0, 0, 0, 0, bytes()))

spark_nlp-2.6.3rc1.dist-info/METADATA

@@ -1,36 +0,0 @@
-Metadata-Version: 2.1
-Name: spark-nlp
-Version: 2.6.3rc1
-Summary: John Snow Labs Spark NLP is a natural language processing library built on top of Apache Spark ML. It provides simple, performant & accurate NLP annotations for machine learning pipelines, that scale easily in a distributed environment.
-Home-page: http://nlp.johnsnowlabs.com
-Author: John Snow Labs
-License: UNKNOWN
-Keywords: NLP spark development
-Platform: UNKNOWN
-Classifier: Development Status :: 5 - Production/Stable
-Classifier: Intended Audience :: Developers
-Classifier: Topic :: Software Development :: Build Tools
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 2
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.4
-Classifier: Programming Language :: Python :: 3.5
-Classifier: Programming Language :: Python :: 3.6
-Description-Content-Type: text/markdown
-
-# Spark-NLP
-John Snow Labs Spark NLP is a natural language processing library built on top of Apache Spark ML. It provides simple, performant & accurate NLP annotations for machine learning pipelines, that scale easily in a distributed environment.
-
-## Requirements
-Spark NLP is built on top of Apache Spark 2.4.4 and works with any user provided Spark 2.x.x it is advised to have basic knowledge of the framework and a working environment before using Spark NLP.
-
-# Spark-NLP for Python
-
-Dependencies on `python3-devel` and `wheel` python module
-
-Build python package with `python3 setup.py sdist bdist_wheel`
-
-Install with `python3 -m pip install --force-reinstall --user dist/spark_nlp-2.2.2-py2.py3-none-any.whl`
-
-

spark_nlp-2.6.3rc1.dist-info/RECORD

@@ -1,48 +0,0 @@
-com/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-com/__init__.pyc,sha256=hLEyuwbCEI-_KJ-HaauMjaJ7_Eova3f2qlUtg7wbZrc,170
-com/__pycache__/__init__.cpython-36.pyc,sha256=urKd549TaG3Rs3c1uaIi-IaQZECP9IOqchonF2WaDzo,166
-com/johnsnowlabs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-com/johnsnowlabs/__init__.pyc,sha256=bhTbkZlXKxXRMmbNNd2yekBL1C8FCS5hTX4-gocBSvQ,183
-com/johnsnowlabs/__pycache__/__init__.cpython-36.pyc,sha256=LdHBGR1z6S4-qxQSfqD4rXfTDucnAM-MLuIjERw163k,179
-com/johnsnowlabs/nlp/__init__.py,sha256=gMSxTenGVOtckW0NxAv1P-19QIMyUrjJv5dnGX03cgQ,209
-com/johnsnowlabs/nlp/__init__.pyc,sha256=uaxKCZLcbjekEaN71rRDNYoKsbRkBLd2-u48MN4N1DY,411
-com/johnsnowlabs/nlp/__pycache__/__init__.cpython-36.pyc,sha256=k4w8UDG2BbV03Lbq5FXs0gNDABbjV7RRnZ_6L21OvUk,353
-sparknlp/__init__.py,sha256=ni0RVqb4SKfSdOGmc7-sbZhL5cMCCH4yisDd_F6fC54,3332
-sparknlp/__init__.pyc,sha256=nVwWsqKvIazlucT_z3PIrNz2XsU_AXRP53HYP5SMFu4,3561
-sparknlp/annotation.py,sha256=juoczyZFnFNVzsiZRKYObcINr7P_Ip2U5WmoPlsPKmk,1151
-sparknlp/annotation.pyc,sha256=1pEno03-yOWvSALAxZmt-9YW8MVfikCn8Sco8QbWta8,2058
-sparknlp/annotator.py,sha256=5FkjHNQo4zwqGf4nvWazfZnq0uCXotCd_fmou-TsFlI,116896
-sparknlp/annotator.pyc,sha256=PUQNs7guLUc06EbctLHr_5qT-W5B7J4sYlcocreqcEo,148679
-sparknlp/base.py,sha256=-85zXkkQdaDMaCNbfUU-nMSVnqW6HggOf9xcYryVVeA,14057
-sparknlp/base.pyc,sha256=EcT6yLv3eqemHhICMkoc7Oyqadg7wBeUWAoHz2eMXPg,20729
-sparknlp/common.py,sha256=GVykc3sXgz6Dgdi44ofra4uLpXUjXRyVwg_BaaqyCUE,6627
-sparknlp/common.pyc,sha256=qy_9mRVn8jc0RtdgMKSUjOD19Wun_iUnn29LZ2Y6MwE,12042
-sparknlp/embeddings.py,sha256=vzXs8ZI9_cppUxPQkhuCzLckESnL0ayRu06ZBAixeNQ,1291
-sparknlp/embeddings.pyc,sha256=kXR7XOsmxYfCpfs5uYCYuu1vVxTWN5aFmUugRWVJ--4,2709
-sparknlp/functions.py,sha256=NlnyhpT-tXAkkjR-ul_21wSMDalqbOdSSl9In_EHC7I,953
-sparknlp/internal.py,sha256=LB5_3kQkBSrttbda4sP3ep4RDm6fjAuoXjw_2vOZGXo,11379
-sparknlp/internal.pyc,sha256=kGVic2zcIkrE9Pxf9RdOGhGv97IQ_YcSXTvhO91HsSY,22311
-sparknlp/pretrained.py,sha256=-SOiKd_1Su2d2QdN4wnmJy6_e9jN2cOSxIk8GXNsGDE,4517
-sparknlp/pretrained.pyc,sha256=KXdsluyIkxsYZXQmTMHU57yso1HLpqB1_hCLxaNqmew,6663
-sparknlp/storage.py,sha256=X4UNOUxt5Ia2FDFSZHA1xxArvKNjD8u2xMCa2kBO_xI,872
-sparknlp/storage.pyc,sha256=q03gTcxlAfq35ejKIIlUpsBZca3dGJaAy0vdhi5LcJI,2063
-sparknlp/training.py,sha256=6T5RYlrv7DQ9KpxpRZnN5RHjkkj13rLeD4Qi-4eLDrc,2215
-sparknlp/training.pyc,sha256=oPZ6qCF6DDXmOPzm0SmR7JUdvXRk2zTDiUdtnNlm2Ts,3222
-sparknlp/util.py,sha256=J5XhwhnaslqdtFQK6Sd9p4MTJsTC3su9yE3SBzqik2w,455
-sparknlp/util.pyc,sha256=WbmCQ3jssVdZNYhhqmkfEV0OTX7ezmFHPjQ91OnfuSc,1275
-sparknlp/__pycache__/__init__.cpython-36.pyc,sha256=_TjoLI9fCqNQN8MdK86RgkFYh16KQZRBn1z_uoyBMiE,2888
-sparknlp/__pycache__/annotation.cpython-36.pyc,sha256=8Cs2Xl2qQ_YbQ83wVrgE4hPiafq4yQiuEGd3HgZ9sMw,1425
-sparknlp/__pycache__/annotator.cpython-36.pyc,sha256=9xhdxdwc2JsOzI_etgMQG3_arI180BnZnpdmNH8PgJE,112153
-sparknlp/__pycache__/base.cpython-36.pyc,sha256=uKQ9gTFZzch9tUUBxfFmmq_Z9BOoqp6V9ZHW9Lcx0h0,15366
-sparknlp/__pycache__/common.cpython-36.pyc,sha256=hHNRFQd4F8Evq-Oc22-Y2O1C37nIryR1_bii841zCh8,8634
-sparknlp/__pycache__/embeddings.cpython-36.pyc,sha256=Y8KqPtGmEfJaOUFrFDvRIGlmzdlA0hXJoy2IS6kNH40,1948
-sparknlp/__pycache__/functions.cpython-36.pyc,sha256=T0F-HDDZWFwfijd-cyVxZgZH79-XGVbt9Q0AFI6aHo8,1646
-sparknlp/__pycache__/internal.cpython-36.pyc,sha256=1a0YbyewntnY6mJifOpwjMH2QXsATL-8UXyA8ngF6O4,16122
-sparknlp/__pycache__/pretrained.cpython-36.pyc,sha256=WuvTW4FbpY3yj__2ou851z8XF2bKAP9mQ9L-sVlR5bY,4981
-sparknlp/__pycache__/storage.cpython-36.pyc,sha256=qlrlSHWXN2bRJ5cUTWNzb4xk2I4-IkBVIvs-OU0DRP8,1494
-sparknlp/__pycache__/training.cpython-36.pyc,sha256=V0JrDYafhBQ7K17cwzNdNy7dEQPIm1jABg5CJEgs5gU,2361
-sparknlp/__pycache__/util.cpython-36.pyc,sha256=oE_Rgp3Eu3mTqoeAvXiMGm2XUdrwEnlTNyzhRHeIWAs,889
-spark_nlp-2.6.3rc1.dist-info/METADATA,sha256=3nht_gCw84Wr3P0vkOm-440Y4ZOzBBMEu2x4_4e0tuI,1707
-spark_nlp-2.6.3rc1.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110
-spark_nlp-2.6.3rc1.dist-info/top_level.txt,sha256=uuytur4pyMRw2H_txNY2ZkaucZHUs22QF8-R03ch_-E,13
-spark_nlp-2.6.3rc1.dist-info/RECORD,,