spark-nlp 6.0.1rc1__py2.py3-none-any.whl → 6.0.2__py2.py3-none-any.whl

sparknlp/partition/partition_properties.py

@@ -0,0 +1,257 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for partition properties used in reading various document types."""
+from typing import Dict
+
+from pyspark.ml.param import TypeConverters, Params, Param
+
+
+class HasEmailReaderProperties(Params):
+
+    addAttachmentContent = Param(
+        Params._dummy(),
+        "addAttachmentContent",
+        "Whether to extract and include the textual content of plain-text attachments in the output",
+        typeConverter=TypeConverters.toBoolean
+    )
+
+    def setAddAttachmentContent(self, value):
+        """
+        Sets whether to extract and include the textual content of plain-text attachments in the output.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to include text from plain-text attachments.
+        """
+        return self._set(addAttachmentContent=value)
+
+    def getAddAttachmentContent(self):
+        """
+        Gets whether to extract and include the textual content of plain-text attachments in the output.
+
+        Returns
+        -------
+        bool
+            Whether to include text from plain-text attachments.
+        """
+        return self.getOrDefault(self.addAttachmentContent)
+
+
+class HasExcelReaderProperties(Params):
+
+    cellSeparator = Param(
+        Params._dummy(),
+        "cellSeparator",
+        "String used to join cell values in a row when assembling textual output.",
+        typeConverter=TypeConverters.toString
+    )
+
+    def setCellSeparator(self, value):
+        """
+        Sets the string used to join cell values in a row when assembling textual output.
+
+        Parameters
+        ----------
+        value : str
+            Delimiter used to concatenate cell values.
+        """
+        return self._set(cellSeparator=value)
+
+    def getCellSeparator(self):
+        """
+        Gets the string used to join cell values in a row when assembling textual output.
+
+        Returns
+        -------
+        str
+            Delimiter used to concatenate cell values.
+        """
+        return self.getOrDefault(self.cellSeparator)
+
+    appendCells = Param(
+        Params._dummy(),
+        "appendCells",
+        "Whether to append all rows into a single content block instead of creating separate elements per row.",
+        typeConverter=TypeConverters.toBoolean
+    )
+
+    def setAppendCells(self, value):
+        """
+        Sets whether to append all rows into a single content block.
+
+        Parameters
+        ----------
+        value : bool
+            True to merge rows into one block, False for individual elements.
+        """
+        return self._set(appendCells=value)
+
+    def getAppendCells(self):
+        """
+        Gets whether to append all rows into a single content block.
+
+        Returns
+        -------
+        bool
+            True to merge rows into one block, False for individual elements.
+        """
+        return self.getOrDefault(self.appendCells)
+
+class HasHTMLReaderProperties(Params):
+
+    timeout = Param(
+        Params._dummy(),
+        "timeout",
+        "Timeout value in seconds for reading remote HTML resources. Applied when fetching content from URLs.",
+        typeConverter=TypeConverters.toInt
+    )
+
+    def setTimeout(self, value):
+        """
+        Sets the timeout (in seconds) for reading remote HTML resources.
+
+        Parameters
+        ----------
+        value : int
+            Timeout in seconds for remote content retrieval.
+        """
+        return self._set(timeout=value)
+
+    def getTimeout(self):
+        """
+        Gets the timeout value for reading remote HTML resources.
+
+        Returns
+        -------
+        int
+            Timeout in seconds.
+        """
+        return self.getOrDefault(self.timeout)
+
+    def setHeaders(self, headers: Dict[str, str]):
+        self._call_java("setHeadersPython", headers)
+        return self
+
+
+class HasPowerPointProperties(Params):
+
+    includeSlideNotes = Param(
+        Params._dummy(),
+        "includeSlideNotes",
+        "Whether to extract speaker notes from slides. When enabled, notes are included as narrative text elements.",
+        typeConverter=TypeConverters.toBoolean
+    )
+
+    def setIncludeSlideNotes(self, value):
+        """
+        Sets whether to extract speaker notes from slides.
+
+        Parameters
+        ----------
+        value : bool
+            If True, notes are included as narrative text elements.
+        """
+        return self._set(includeSlideNotes=value)
+
+    def getIncludeSlideNotes(self):
+        """
+        Gets whether to extract speaker notes from slides.
+
+        Returns
+        -------
+        bool
+            True if notes are included as narrative text elements.
+        """
+        return self.getOrDefault(self.includeSlideNotes)
+
+class HasTextReaderProperties(Params):
+
+    titleLengthSize = Param(
+        Params._dummy(),
+        "titleLengthSize",
+        "Maximum character length used to determine if a text block qualifies as a title during parsing.",
+        typeConverter=TypeConverters.toInt
+    )
+
+    def setTitleLengthSize(self, value):
+        return self._set(titleLengthSize=value)
+
+    def getTitleLengthSize(self):
+        return self.getOrDefault(self.titleLengthSize)
+
+    groupBrokenParagraphs = Param(
+        Params._dummy(),
+        "groupBrokenParagraphs",
+        "Whether to merge fragmented lines into coherent paragraphs using heuristics based on line length and structure.",
+        typeConverter=TypeConverters.toBoolean
+    )
+
+    def setGroupBrokenParagraphs(self, value):
+        return self._set(groupBrokenParagraphs=value)
+
+    def getGroupBrokenParagraphs(self):
+        return self.getOrDefault(self.groupBrokenParagraphs)
+
+    paragraphSplit = Param(
+        Params._dummy(),
+        "paragraphSplit",
+        "Regex pattern used to detect paragraph boundaries when grouping broken paragraphs.",
+        typeConverter=TypeConverters.toString
+    )
+
+    def setParagraphSplit(self, value):
+        return self._set(paragraphSplit=value)
+
+    def getParagraphSplit(self):
+        return self.getOrDefault(self.paragraphSplit)
+
+    shortLineWordThreshold = Param(
+        Params._dummy(),
+        "shortLineWordThreshold",
+        "Maximum word count for a line to be considered 'short' during broken paragraph grouping.",
+        typeConverter=TypeConverters.toInt
+    )
+
+    def setShortLineWordThreshold(self, value):
+        return self._set(shortLineWordThreshold=value)
+
+    def getShortLineWordThreshold(self):
+        return self.getOrDefault(self.shortLineWordThreshold)
+
+    maxLineCount = Param(
+        Params._dummy(),
+        "maxLineCount",
+        "Maximum number of lines to evaluate when estimating paragraph layout characteristics.",
+        typeConverter=TypeConverters.toInt
+    )
+
+    def setMaxLineCount(self, value):
+        return self._set(maxLineCount=value)
+
+    def getMaxLineCount(self):
+        return self.getOrDefault(self.maxLineCount)
+
+    threshold = Param(
+        Params._dummy(),
+        "threshold",
+        "Threshold ratio of empty lines used to decide between new line-based or broken-paragraph grouping.",
+        typeConverter=TypeConverters.toFloat
+    )
+
+    def setThreshold(self, value):
+        return self._set(threshold=value)
+
+    def getThreshold(self):
+        return self.getOrDefault(self.threshold)

sparknlp/partition/partition_transformer.py

@@ -0,0 +1,196 @@
+#  Copyright 2017-2025 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains the PartitionTransformer class for reading various types of documents into chunks."""
+from sparknlp.common import *
+from sparknlp.partition.partition_properties import *
+
+class PartitionTransformer(
+    AnnotatorModel,
+    HasEmailReaderProperties,
+    HasExcelReaderProperties,
+    HasHTMLReaderProperties,
+    HasPowerPointProperties,
+    HasTextReaderProperties
+):
+    """
+    The PartitionTransformer annotator allows you to use the Partition feature more smoothly
+    within existing Spark NLP workflows, enabling seamless reuse of your pipelines.
+
+    It supports reading from files, URLs, in-memory strings, or byte arrays, and works
+    within a Spark NLP pipeline.
+
+    Supported formats include:
+    - Plain text
+    - HTML
+    - Word (.doc/.docx)
+    - Excel (.xls/.xlsx)
+    - PowerPoint (.ppt/.pptx)
+    - Email files (.eml, .msg)
+    - PDFs
+
+    Parameters
+    ----------
+    inputCols : list of str
+        Names of input columns (typically from DocumentAssembler).
+    outputCol : str
+        Name of the column to store the output.
+    contentType : str
+        The type of content: e.g., "text", "url", "file", etc.
+    headers : dict, optional
+        Headers to be used if content type is a URL.
+
+    Examples
+    --------
+    >>> dataset = spark.createDataFrame([
+    ...     ("https://www.blizzard.com",),
+    ... ], ["text"])
+
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+
+    >>> partition = PartitionTransformer() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("partition") \\
+    ...     .setContentType("url") \\
+    ...     .setHeaders({"Accept-Language": "es-ES"})
+
+    >>> pipeline = Pipeline(stages=[documentAssembler, partition])
+    >>> pipelineModel = pipeline.fit(dataset)
+    >>> resultDf = pipelineModel.transform(dataset)
+    >>> resultDf.show()
+    +--------------------+--------------------+--------------------+
+    |                text|            document|           partition|
+    +--------------------+--------------------+--------------------+
+    |https://www.blizz...|[{Title, Juegos d...|[{document, 0, 16...|
+    |https://www.googl...|[{Title, Gmail Im...|[{document, 0, 28...|
+    +--------------------+--------------------+--------------------+
+    """
+
+    name = "PartitionTransformer"
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.DOCUMENT
+
+    contentPath = Param(
+        Params._dummy(),
+        "contentPath",
+        "Path to the content source",
+        typeConverter=TypeConverters.toString
+    )
+
+    def setContentPath(self, value):
+        return self._set(contentPath=value)
+
+    def getContentPath(self):
+        return self.getOrDefault(self.contentPath)
+
+    contentType = Param(
+        Params._dummy(),
+        "contentType",
+        "Set the content type to load following MIME specification",
+        typeConverter=TypeConverters.toString
+    )
+
+    def setContentType(self, value):
+        return self._set(contentType=value)
+
+    def getContentType(self):
+        return self.getOrDefault(self.contentType)
+
+    storeContent = Param(
+        Params._dummy(),
+        "storeContent",
+        "Whether to include the raw file content in the output DataFrame as a separate 'content' column, alongside the structured output.",
+        typeConverter=TypeConverters.toBoolean
+    )
+
+    def setStoreContent(self, value):
+        return self._set(storeContent=value)
+
+    def getStoreContent(self):
+        return self.getOrDefault(self.storeContent)
+
+    titleFontSize = Param(
+        Params._dummy(),
+        "titleFontSize",
+        "Minimum font size threshold used as part of heuristic rules to detect title elements based on formatting (e.g., bold, centered, capitalized).",
+        typeConverter=TypeConverters.toInt
+    )
+
+    def setTitleFontSize(self, value):
+        return self._set(titleFontSize=value)
+
+    def getTitleFontSize(self):
+        return self.getOrDefault(self.titleFontSize)
+
+    inferTableStructure = Param(
+        Params._dummy(),
+        "inferTableStructure",
+        "Whether to generate an HTML table representation from structured table content. When enabled, a full <table> element is added alongside cell-level elements, based on row and column layout.",
+        typeConverter=TypeConverters.toBoolean
+    )
+
+    def setInferTableStructure(self, value):
+        return self._set(inferTableStructure=value)
+
+    def getInferTableStructure(self):
+        return self.getOrDefault(self.inferTableStructure)
+
+    includePageBreaks = Param(
+        Params._dummy(),
+        "includePageBreaks",
+        "Whether to detect and tag content with page break metadata. In Word documents, this includes manual and section breaks. In Excel files, this includes page breaks based on column boundaries.",
+        typeConverter=TypeConverters.toBoolean
+    )
+
+    def setIncludePageBreaks(self, value):
+        return self._set(includePageBreaks=value)
+
+    def getIncludePageBreaks(self):
+        return self.getOrDefault(self.includePageBreaks)
+
+    # def setHeaders(self, headers: Dict[str, str]):
+    #     self._call_java("setHeadersPython", headers)
+    #     return self
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.partition.PartitionTransformer",
+                 java_model=None):
+        super(PartitionTransformer, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+        DOUBLE_PARAGRAPH_PATTERN = r"(?:\s*\n\s*){2,}"
+
+        self._setDefault(
+            contentPath="",
+            contentType="text/plain",
+            storeContent=False,
+            titleFontSize = 9,
+            inferTableStructure=False,
+            includePageBreaks=False,
+            addAttachmentContent=False,
+            cellSeparator="\t",
+            appendCells=False,
+            timeout=0,
+            includeSlideNotes=False,
+            titleLengthSize=50,
+            groupBrokenParagraphs=False,
+            paragraphSplit=DOUBLE_PARAGRAPH_PATTERN,
+            shortLineWordThreshold=5,
+            maxLineCount=2000,
+            threshold=0.1
+        )

sparknlp/reader/pdf_to_text.py

@@ -10,10 +10,56 @@ from sparknlp.reader.enums import TextStripperType
 class PdfToText(JavaTransformer, HasInputCol, HasOutputCol,
                 JavaMLReadable, JavaMLWritable):
     """
-    Extract text from Pdf document to single string or to several strings per each page.
-    Input is a column with binary representation of PDF document.
-    As output generate column with text and page number.
-    Explode each page as separate row if split to page enabled.
+    Extract text from PDF documents as either a single string or multiple strings per page.
+    Input is a column with binary content of PDF files. Output is a column with extracted text,
+    with options to include page numbers or split pages.
+
+    Parameters
+    ----------
+    pageNumCol : str, optional
+        Page number output column name.
+    partitionNum : int, optional
+        Number of partitions (default is 0).
+    storeSplittedPdf : bool, optional
+        Whether to store content of split PDFs (default is False).
+    splitPage : bool, optional
+        Enable/disable splitting per page (default is True).
+    onlyPageNum : bool, optional
+        Whether to extract only page numbers (default is False).
+    textStripper : str or TextStripperType, optional
+        Defines layout and formatting type.
+    sort : bool, optional
+        Enable/disable sorting content per page (default is False).
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.reader import *
+    >>> from pyspark.ml import Pipeline
+    >>> pdf_path = "Documents/files/pdf"
+    >>> data_frame = spark.read.format("binaryFile").load(pdf_path)
+    >>> pdf_to_text = PdfToText().setStoreSplittedPdf(True)
+    >>> pipeline = Pipeline(stages=[pdf_to_text])
+    >>> pipeline_model = pipeline.fit(data_frame)
+    >>> pdf_df = pipeline_model.transform(data_frame)
+    >>> pdf_df.show()
+    +--------------------+--------------------+
+    |                path|    modificationTime|
+    +--------------------+--------------------+
+    |file:/Users/paula...|2025-05-15 11:33:...|
+    |file:/Users/paula...|2025-05-15 11:33:...|
+    +--------------------+--------------------+
+    >>> pdf_df.printSchema()
+    root
+     |-- path: string (nullable = true)
+     |-- modificationTime: timestamp (nullable = true)
+     |-- length: long (nullable = true)
+     |-- text: string (nullable = true)
+     |-- height_dimension: integer (nullable = true)
+     |-- width_dimension: integer (nullable = true)
+     |-- content: binary (nullable = true)
+     |-- exception: string (nullable = true)
+     |-- pagenum: integer (nullable = true)
     """
     pageNumCol = Param(Params._dummy(), "pageNumCol",
                        "Page number output column name.",