spark-nlp 4.2.6__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/reader/sparknlp_reader.py

@@ -0,0 +1,461 @@
+#  Copyright 2017-2024 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from sparknlp.internal import ExtendedJavaWrapper
+
+
+class SparkNLPReader(ExtendedJavaWrapper):
+    """Instantiates class to read documents in various formats.
+
+    Parameters
+    ----------
+    params : spark
+        Spark session
+    params : dict, optional
+        Parameter with custom configuration
+
+    Notes
+    -----
+    This class can read HTML, email, PDF, MS Word, Excel, PowerPoint, and text files.
+
+    Examples
+    --------
+    >>> from sparknlp.reader import SparkNLPReader
+    >>> reader = SparkNLPReader(spark)
+
+    Reading HTML
+
+    >>> html_df = reader.html("https://www.wikipedia.org")
+    >>> # Or with shorthand
+    >>> import sparknlp
+    >>> html_df = sparknlp.read().html("https://www.wikipedia.org")
+
+    Reading PDF
+
+    >>> pdf_df = reader.pdf("home/user/pdfs-directory")
+    >>> # Or with shorthand
+    >>> pdf_df = sparknlp.read().pdf("home/user/pdfs-directory")
+
+    Reading Email
+    
+    >>> email_df = reader.email("home/user/emails-directory")
+    >>> # Or with shorthand
+    >>> email_df = sparknlp.read().email("home/user/emails-directory")
+    """
+
+    def __init__(self, spark, params=None, headers=None):
+        if params is None:
+            params = {}
+        super(SparkNLPReader, self).__init__("com.johnsnowlabs.reader.SparkNLPReader", params, headers)
+        self.spark = spark
+
+    def html(self, htmlPath):
+        """Reads HTML files or URLs and returns a Spark DataFrame.
+
+        Parameters
+        ----------
+        htmlPath : str or list of str
+            Path(s) to HTML file(s) or a list of URLs.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            A DataFrame containing the parsed HTML content.
+
+        Examples
+        --------
+        >>> from sparknlp.reader import SparkNLPReader
+        >>> html_df = SparkNLPReader().html("https://www.wikipedia.org")
+
+        You can also use SparkNLP to simplify the process:
+
+        >>> import sparknlp
+        >>> html_df = sparknlp.read().html("https://www.wikipedia.org")
+        >>> html_df.show(truncate=False)
+
+        +--------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+        |url                 |html                                                                                                                                                                                                                                                                                                                            |
+        +--------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+        |https://example.com/|[{Title, Example Domain, {pageNumber -> 1}}, {NarrativeText, 0, This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission., {pageNumber -> 1}}, {NarrativeText, 0, More information... More information..., {pageNumber -> 1}}]   |
+        +--------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+        >>> html_df.printSchema()
+        root
+         |-- url: string (nullable = true)
+         |-- html: array (nullable = true)
+         |    |-- element: struct (containsNull = true)
+         |    |    |-- elementType: string (nullable = true)
+         |    |    |-- content: string (nullable = true)
+         |    |    |-- metadata: map (nullable = true)
+         |    |    |    |-- key: string
+         |    |    |    |-- value: string (valueContainsNull = true)
+        """
+        if not isinstance(htmlPath, (str, list)) or (isinstance(htmlPath, list) and not all(isinstance(item, str) for item in htmlPath)):
+            raise TypeError("htmlPath must be a string or a list of strings")
+        jdf = self._java_obj.html(htmlPath)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe
+
+    def email(self, filePath):
+        """Reads email files and returns a Spark DataFrame.
+
+        Parameters
+        ----------
+        filePath : str
+            Path to an email file or a directory containing emails.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            A DataFrame containing parsed email data.
+
+        Examples
+        --------
+        >>> from sparknlp.reader import SparkNLPReader
+        >>> email_df = SparkNLPReader(spark).email("home/user/emails-directory")
+
+        You can also use SparkNLP to simplify the process:
+
+        >>> import sparknlp
+        >>> email_df = sparknlp.read().email("home/user/emails-directory")
+        >>> email_df.show()
+        +---------------------------------------------------+
+        |email                                              |
+        +---------------------------------------------------+
+        |[{Title, Email Text Attachments, {sent_to -> Danilo|
+        +---------------------------------------------------+
+        >>> email_df.printSchema()
+        root
+         |-- path: string (nullable = true)
+         |-- content: array (nullable = true)
+         |-- email: array (nullable = true)
+         |    |-- element: struct (containsNull = true)
+         |    |    |-- elementType: string (nullable = true)
+         |    |    |-- content: string (nullable = true)
+         |    |    |-- metadata: map (nullable = true)
+         |    |    |    |-- key: string
+         |    |    |    |-- value: string (valueContainsNull = true)
+
+        """
+        if not isinstance(filePath, str):
+            raise TypeError("filePath must be a string")
+        jdf = self._java_obj.email(filePath)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe
+
+    def doc(self, docPath):
+        """Reads word document files and returns a Spark DataFrame.
+
+        Parameters
+        ----------
+        docPath : str
+            Path to a word document file.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            A DataFrame containing parsed document content.
+
+        Examples
+        --------
+        >>> from sparknlp.reader import SparkNLPReader
+        >>> doc_df = SparkNLPReader().doc(spark, "home/user/word-directory")
+
+        You can use SparkNLP for one line of code
+
+        >>> import sparknlp
+        >>> doc_df = sparknlp.read().doc("home/user/word-directory")
+        >>> doc_df.show()
+        +-------------------------------------------------+
+        |doc                                              |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+        +-------------------------------------------------+
+        |[{Table, Header Col 1, {}}, {Table, Header Col 2,|
+        +-------------------------------------------------+
+
+        >>> doc_df.printSchema()
+        root
+         |-- path: string (nullable = true)
+         |-- content: array (nullable = true)
+         |-- doc: array (nullable = true)
+         |    |-- element: struct (containsNull = true)
+         |    |    |-- elementType: string (nullable = true)
+         |    |    |-- content: string (nullable = true)
+         |    |    |-- metadata: map (nullable = true)
+         |    |    |    |-- key: string
+         |    |    |    |-- value: string (valueContainsNull = true)
+
+        """
+        if not isinstance(docPath, str):
+            raise TypeError("docPath must be a string")
+        jdf = self._java_obj.doc(docPath)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe
+
+    def pdf(self, pdfPath):
+        if not isinstance(pdfPath, str):
+            raise TypeError("docPath must be a string")
+        jdf = self._java_obj.pdf(pdfPath)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe
+
+    def xls(self, docPath):
+        """Reads excel document files and returns a Spark DataFrame.
+
+        Parameters
+        ----------
+        docPath : str
+            Path to an excel document file.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            A DataFrame containing parsed document content.
+
+        Examples
+        --------
+        >>> from sparknlp.reader import SparkNLPReader
+        >>> xlsDf = SparkNLPReader().xls(spark, "home/user/excel-directory")
+
+        You can use SparkNLP for one line of code
+
+        >>> import sparknlp
+        >>> xlsDf = sparknlp.read().xls("home/user/excel-directory")
+        >>> xlsDf.show()
+        +--------------------------------------------+
+        |xls                                         |
+        +--------------------------------------------+
+        |[{Title, Financial performance, {SheetNam}}]|
+        +--------------------------------------------+
+
+        >>> xlsDf.printSchema()
+        root
+         |-- path: string (nullable = true)
+         |-- content: binary (nullable = true)
+         |-- xls: array (nullable = true)
+         |    |-- element: struct (containsNull = true)
+         |    |    |-- elementType: string (nullable = true)
+         |    |    |-- content: string (nullable = true)
+         |    |    |-- metadata: map (nullable = true)
+         |    |    |    |-- key: string
+         |    |    |    |-- value: string (valueContainsNull = true)
+       """
+        if not isinstance(docPath, str):
+            raise TypeError("docPath must be a string")
+        jdf = self._java_obj.xls(docPath)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe
+
+    def ppt(self, docPath):
+        """
+        Reads power point document files and returns a Spark DataFrame.
+
+        Parameters
+        ----------
+        docPath : str
+            Path to an power point document file.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            A DataFrame containing parsed document content.
+
+        Examples
+        --------
+        >>> from sparknlp.reader import SparkNLPReader
+        >>> pptDf = SparkNLPReader().ppt(spark, "home/user/powerpoint-directory")
+
+        You can use SparkNLP for one line of code
+
+        >>> import sparknlp
+        >>> pptDf = sparknlp.read().ppt("home/user/powerpoint-directory")
+        >>> pptDf.show(truncate=False)
+        +-------------------------------------+
+        |ppt                                  |
+        +-------------------------------------+
+        |[{Title, Adding a Bullet Slide, {}},]|
+        +-------------------------------------+
+        """
+        if not isinstance(docPath, str):
+            raise TypeError("docPath must be a string")
+        jdf = self._java_obj.ppt(docPath)
+        dataframe = self.getDataFrame(self.spark, jdf)
+        return dataframe
+
+    def txt(self, docPath):
+        """Reads TXT files and returns a Spark DataFrame.
+
+        Parameters
+        ----------
+        docPath : str
+            Path to a TXT file.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            A DataFrame containing parsed document content.
+
+        Examples
+        --------
+        >>> from sparknlp.reader import SparkNLPReader
+        >>> txtDf = SparkNLPReader().txt(spark, "home/user/txt/files")
+
+        You can use SparkNLP for one line of code
+
+        >>> import sparknlp
+        >>> txtDf = sparknlp.read().txt("home/user/txt/files")
+        >>> txtDf.show(truncate=False)
+        +-----------------------------------------------+
+        |txt                                            |
+        +-----------------------------------------------+
+        |[{Title, BIG DATA ANALYTICS, {paragraph -> 0}}]|
+        +-----------------------------------------------+
+        """
+        if not isinstance(docPath, str):
+            raise TypeError("docPath must be a string")
+        jdf = self._java_obj.txt(docPath)
+        return self.getDataFrame(self.spark, jdf)
+
+    def xml(self, docPath):
+        """Reads XML files and returns a Spark DataFrame.
+
+        Parameters
+        ----------
+        docPath : str
+            Path to an XML file or a directory containing XML files.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            A DataFrame containing parsed XML content.
+
+        Examples
+        --------
+        >>> from sparknlp.reader import SparkNLPReader
+        >>> xml_df = SparkNLPReader(spark).xml("home/user/xml-directory")
+
+        You can use SparkNLP for one line of code
+
+        >>> import sparknlp
+        >>> xml_df = sparknlp.read().xml("home/user/xml-directory")
+        >>> xml_df.show(truncate=False)
+        +-----------------------------------------------------------+
+        |xml                                                       |
+        +-----------------------------------------------------------+
+        |[{Title, John Smith, {elementId -> ..., tag -> title}}]   |
+        +-----------------------------------------------------------+
+
+        >>> xml_df.printSchema()
+        root
+         |-- path: string (nullable = true)
+         |-- xml: array (nullable = true)
+         |    |-- element: struct (containsNull = true)
+         |    |    |-- elementType: string (nullable = true)
+         |    |    |-- content: string (nullable = true)
+         |    |    |-- metadata: map (nullable = true)
+         |    |    |    |-- key: string
+         |    |    |    |-- value: string (valueContainsNull = true)
+        """
+        if not isinstance(docPath, str):
+            raise TypeError("docPath must be a string")
+        jdf = self._java_obj.xml(docPath)
+        return self.getDataFrame(self.spark, jdf)
+
+
+    def md(self, filePath):
+        """Reads Markdown files and returns a Spark DataFrame.
+
+        Parameters
+        ----------
+        filePath : str
+            Path to a Markdown file or a directory containing Markdown files.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            A DataFrame containing parsed Markdown content.
+
+        Examples
+        --------
+        >>> from sparknlp.reader import SparkNLPReader
+        >>> md_df = SparkNLPReader(spark).md("home/user/markdown-directory")
+
+        You can use SparkNLP for one line of code
+
+        >>> import sparknlp
+        >>> md_df = sparknlp.read().md("home/user/markdown-directory")
+        >>> md_df.show(truncate=False)
+        +-----------------------------------------------------------+
+        |md                                                         |
+        +-----------------------------------------------------------+
+        |[{Title, Sample Markdown Document, {elementId -> ..., tag -> title}}]|
+        +-----------------------------------------------------------+
+
+        >>> md_df.printSchema()
+        root
+         |-- path: string (nullable = true)
+         |-- md: array (nullable = true)
+         |    |-- element: struct (containsNull = true)
+         |    |    |-- elementType: string (nullable = true)
+         |    |    |-- content: string (nullable = true)
+         |    |    |-- metadata: map (nullable = true)
+         |    |    |    |-- key: string
+         |    |    |    |-- value: string (valueContainsNull = true)
+        """
+        if not isinstance(filePath, str):
+            raise TypeError("filePath must be a string")
+        jdf = self._java_obj.md(filePath)
+        return self.getDataFrame(self.spark, jdf)
+
+    def csv(self, csvPath):
+        """Reads CSV files and returns a Spark DataFrame.
+
+        Parameters
+        ----------
+        docPath : str
+            Path to an CSV file or a directory containing CSV files.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            A DataFrame containing parsed CSV content.
+
+        Examples
+        --------
+        >>> from sparknlp.reader import SparkNLPReader
+        >>> csv_df = SparkNLPReader(spark).csv("home/user/csv-directory")
+
+        You can use SparkNLP for one line of code
+
+        >>> import sparknlp
+        >>> csv_df = sparknlp.read().csv("home/user/csv-directory")
+        >>> csv_df.show(truncate=False)
+        +-----------------------------------------------------------------------------------------------------------------------------------------+
+        |csv                                                                                                                                      |
+        +-----------------------------------------------------------------------------------------------------------------------------------------+
+        |[{NarrativeText, Alice 100 Bob 95, {}}, {Table, <table><tr><td>Alice</td><td>100</td></tr><tr><td>Bob</td><td>95</td></tr></table>, {}}] |
+        +-----------------------------------------------------------------------------------------------------------------------------------------+
+
+        >>> csv_df.printSchema()
+        root
+         |-- path: string (nullable = true)
+         |-- csv: array (nullable = true)
+         |    |-- element: struct (containsNull = true)
+         |    |    |-- elementType: string (nullable = true)
+         |    |    |-- content: string (nullable = true)
+         |    |    |-- metadata: map (nullable = true)
+         |    |    |    |-- key: string
+         |    |    |    |-- value: string (valueContainsNull = true)
+        """
+        if not isinstance(csvPath, str):
+            raise TypeError("docPath must be a string")
+        jdf = self._java_obj.csv(csvPath)
+        return self.getDataFrame(self.spark, jdf)

sparknlp/training/__init__.py

@@ -17,3 +17,4 @@ from sparknlp.training.conll import *
 from sparknlp.training.conllu import *
 from sparknlp.training.pos import *
 from sparknlp.training.pub_tator import *
+from sparknlp.training.spacy_to_annotation import *

sparknlp/training/conll.py

@@ -65,6 +65,8 @@ class CoNLL(ExtendedJavaWrapper):
         Whether to explode sentences to separate rows, by default True
     delimiter: str, optional
         Delimiter used to separate columns inside CoNLL file
+    includeDocId: bool, optional
+        Whether to try and parse the document id from the third item in the -DOCSTART- line (X if not found)
 
     Examples
     --------
@@ -92,10 +94,12 @@ class CoNLL(ExtendedJavaWrapper):
                  posCol='pos',
                  conllLabelIndex=3,
                  conllPosIndex=1,
+                 conllDocIdCol="doc_id",
                  textCol='text',
                  labelCol='label',
                  explodeSentences=True,
-                 delimiter=' '
+                 delimiter=' ',
+                 includeDocId=False
                  ):
         super(CoNLL, self).__init__("com.johnsnowlabs.nlp.training.CoNLL",
                                     documentCol,
@@ -104,10 +108,12 @@ class CoNLL(ExtendedJavaWrapper):
                                     posCol,
                                     conllLabelIndex,
                                     conllPosIndex,
+                                    conllDocIdCol,
                                     textCol,
                                     labelCol,
                                     explodeSentences,
-                                    delimiter)
+                                    delimiter,
+                                    includeDocId)
 
     def readDataset(self, spark, path, read_as=ReadAs.TEXT, partitions=8, storage_level=pyspark.StorageLevel.DISK_ONLY):
         # ToDo Replace with std pyspark

sparknlp/training/spacy_to_annotation.py

@@ -0,0 +1,57 @@
+#  Copyright 2017-2023 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from pyspark.sql import SparkSession
+
+from sparknlp.internal import ExtendedJavaWrapper
+
+
+class SpacyToAnnotation(ExtendedJavaWrapper):
+
+    """Helper class to load a list of tokens/sentences as JSON to Annotation.
+
+    The JSON will be in this format:
+        [
+         {
+            "tokens": ["Hello", "world", "!", "How", "are", "you", "today", "?", "I", "'m", "fine", "thanks", "."],
+            "token_spaces": [true, false, true, true, true, true, false, true, false, true, true, false, false],
+            "sentence_ends": [2, 7, 12]
+         }
+        ]
+
+    Examples
+    --------
+    >>> from sparknlp.training import SpacyToAnnotation
+    >>> result = SpacyToAnnotation().readDataset(spark, "src/test/resources/spacy-to-annotation/multi_doc_tokens.json")
+    >>> result.show(False)
+    +-------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |document                                                                             |sentence                                                                                                                                                                      |token                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+    +-------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |[{document, 0, 55, John went to the store last night. He bought some bread., {}, []}]|[{document, 0, 33, John went to the store last night., {sentence -> 0}, []}, {document, 35, 55, He bought some bread., {sentence -> 1}, []}]                                  |[{token, 0, 3, John, {sentence -> 0}, []}, {token, 5, 8, went, {sentence -> 0}, []}, {token, 10, 11, to, {sentence -> 0}, []}, {token, 13, 15, the, {sentence -> 0}, []}, {token, 17, 21, store, {sentence -> 0}, []}, {token, 23, 26, last, {sentence -> 0}, []}, {token, 28, 32, night, {sentence -> 0}, []}, {token, 33, 33, ., {sentence -> 0}, []}, {token, 35, 36, He, {sentence -> 1}, []}, {token, 38, 43, bought, {sentence -> 1}, []}, {token, 45, 48, some, {sentence -> 1}, []}, {token, 50, 54, bread, {sentence -> 1}, []}, {token, 55, 55, ., {sentence -> 1}, []}]|
+    |[{document, 0, 47, Hello world! How are you today? I'm fine thanks., {}, []}]        |[{document, 0, 11, Hello world!, {sentence -> 0}, []}, {document, 13, 30, How are you today?, {sentence -> 1}, []}, {document, 32, 47, I'm fine thanks., {sentence -> 2}, []}]|[{token, 0, 4, Hello, {sentence -> 0}, []}, {token, 6, 10, world, {sentence -> 0}, []}, {token, 11, 11, !, {sentence -> 0}, []}, {token, 13, 15, How, {sentence -> 1}, []}, {token, 17, 19, are, {sentence -> 1}, []}, {token, 21, 23, you, {sentence -> 1}, []}, {token, 25, 29, today, {sentence -> 1}, []}, {token, 30, 30, ?, {sentence -> 1}, []}, {token, 32, 32, I, {sentence -> 2}, []}, {token, 33, 34, 'm, {sentence -> 2}, []}, {token, 36, 39, fine, {sentence -> 2}, []}, {token, 41, 46, thanks, {sentence -> 2}, []}, {token, 47, 47, ., {sentence -> 2}, []}]     |
+    +-------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+
+    """
+
+    def __init__(self):
+        super(SpacyToAnnotation, self).__init__("com.johnsnowlabs.nlp.training.SpacyToAnnotation")
+
+    def readJsonFile(self, spark, jsonFilePath, params=None):
+        if params is None:
+            params = {}
+
+        jSession = spark._jsparkSession
+
+        jdf = self._java_obj.readJsonFileJava(jSession, jsonFilePath, params)
+        annotation_dataset = self.getDataFrame(spark, jdf)
+        return annotation_dataset

sparknlp/util.py

@@ -15,6 +15,9 @@
 
 
 import sparknlp.internal as _internal
+import numpy as np
+from pyspark.sql import Row
+from pyspark.sql.types import StructType, StructField, StringType, IntegerType, BinaryType
 
 
 def get_config_path():
@@ -33,3 +36,26 @@ class CoNLLGenerator:
             _internal._CoNLLGeneratorExportFromTargetAndPipeline(*args).apply()
         else:
             raise NotImplementedError(f"No exportConllFiles alternative takes {num_args} parameters")
+
+
+class EmbeddingsDataFrameUtils:
+    """
+    Utility for creating DataFrames compatible with multimodal embedding models (e.g., E5VEmbeddings) for text-only scenarios.
+    Provides:
+      - imageSchema: the expected schema for Spark image DataFrames
+      - emptyImageRow: a dummy image row for text-only embedding
+    """
+    imageSchema = StructType([
+        StructField(
+            "image",
+            StructType([
+                StructField("origin", StringType(), True),
+                StructField("height", IntegerType(), True),
+                StructField("width", IntegerType(), True),
+                StructField("nChannels", IntegerType(), True),
+                StructField("mode", IntegerType(), True),
+                StructField("data", BinaryType(), True),
+            ]),
+        )
+    ])
+    emptyImageRow = Row(Row("", 0, 0, 0, 0, bytes()))