gllm-docproc-binary 0.1.8__cp312-cp312-macosx_13_0_x86_64.whl

gllm_docproc/model/element_metadata.pyi

@@ -0,0 +1,35 @@
+from pydantic import BaseModel
+
+PDF: str
+DOCX: str
+XLSX: str
+PPTX: str
+CSV: str
+TXT: str
+HTML: str
+AUDIO: str
+IMAGE: str
+VIDEO: str
+
+class ElementMetadata(BaseModel):
+    """Element metadata model.
+
+    This class serves as the Element metadata model for storing element metadata.
+
+    Mandatory Attributes:
+        source (str): The source of the element.
+        source_type (str): The source type of the element.
+        loaded_datetime (datetime): The datetime when the element is loaded.
+    """
+    source: str
+    source_type: str
+    loaded_datetime: str
+    class Config:
+        """Pydantic model configuration.
+
+        This class defines the Pydantic model configuration for the ElementMetadata model.
+
+        Attributes:
+            extra (str): Allow extra fields.
+        """
+        extra: str

gllm_docproc/parser/__init__.pyi

@@ -0,0 +1,4 @@
+from .base_parser import BaseParser as BaseParser
+from .pipeline_parser import PipelineParser as PipelineParser
+
+__all__ = ['BaseParser', 'PipelineParser']

gllm_docproc/parser/base_parser.pyi

@@ -0,0 +1,29 @@
+import abc
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseParser(ABC, metaclass=abc.ABCMeta):
+    """Base class for document parser.
+
+    This class serves as the base for document parser, which will define the structure for every
+    content of document.
+
+    Methods:
+        parse(loaded_elements, **kwargs): Abstract method to parse a document.
+    """
+    @abstractmethod
+    def parse(self, loaded_elements: Any, **kwargs: Any) -> Any:
+        """Parse loaded elements to get element structure.
+
+        This method is abstract and must be implemented in subclasses.
+        It defines the process of parsing a document using loaded elements.
+
+        Args:
+            loaded_elements (Any): The loaded elements from loader. ideally formatted as List[Dict].
+            **kwargs (Any): Additional keyword arguments for customization.
+
+        Returns:
+            Any: The parsed document, ideally formatted as List[Dict]. Each dictionary within
+                the list are recommended to follows the structure of model 'Element',
+                to ensure consistency and ease of use across Document Processing Orchestrator.
+        """

gllm_docproc/parser/document/__init__.pyi

@@ -0,0 +1,6 @@
+from .docx_parser import DOCXParser as DOCXParser
+from .pdf_parser import PDFParser as PDFParser
+from .txt_parser import TXTParser as TXTParser
+from .xlsx_parser import XLSXParser as XLSXParser
+
+__all__ = ['DOCXParser', 'PDFParser', 'XLSXParser', 'TXTParser']

gllm_docproc/parser/document/docx_parser.pyi

@@ -0,0 +1,27 @@
+from gllm_docproc.model.element import Element as Element, FOOTER as FOOTER, HEADER as HEADER, HEADING as HEADING, MAX_HEADING_LEVEL as MAX_HEADING_LEVEL, PARAGRAPH as PARAGRAPH, TITLE as TITLE, UNCATEGORIZED_TEXT as UNCATEGORIZED_TEXT
+from gllm_docproc.parser.base_parser import BaseParser as BaseParser
+from typing import Any
+
+class DOCXParser(BaseParser):
+    """A DOCX parser for parsing DOCX document text structure.
+
+    This class serves as the DOCX parser for parsing DOCX document text structure.
+    It defines the structure for parsing DOCX document text structure from a given loaded_elements.
+
+    Methods:
+        parse(loaded_elements, **kwargs): Parse the document from the loaded elements.
+    """
+    def parse(self, loaded_elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        """Parse the document from the loaded elements.
+
+        This method defines the process of defining text structure from loaded_elements (DOCX Loader output)
+        by their style_name. In cases there's customized style_name, it will be categorized as paragraph.
+        (example: 'Heading', 'Heading Body', 'Title 1', will be categorized as paragraph.)
+
+        Args:
+            loaded_elements (list[dict[str, Any]]): A list of loaded elements containing text content and metadata.
+            **kwargs (Any): Additional keyword arguments for parsing the document.
+
+        Returns:
+            list[dict[str, Any]]: A list of parsed elements containing text content and metadata.
+        """

gllm_docproc/parser/document/pdf_parser.pyi

@@ -0,0 +1,35 @@
+from gllm_docproc.model.element import Element as Element, FOOTER as FOOTER, FOOTNOTE as FOOTNOTE, HEADER as HEADER, HEADING as HEADING, PARAGRAPH as PARAGRAPH, TITLE as TITLE, UNCATEGORIZED_TEXT as UNCATEGORIZED_TEXT
+from gllm_docproc.parser.base_parser import BaseParser as BaseParser
+from typing import Any
+
+HEADER_THRESHOLD_POSITION: int
+FOOTER_THRESHOLD_POSITION: int
+FOOTNOTE_POSITION_RATIO: float
+
+class PDFParser(BaseParser):
+    """A class to parse PDF documents.
+
+    This class serves as a PDF parser for parsing or defining the structure of text within PDF documents
+    based on the text metadata (font size, font family, coordinates, etc.).
+
+    Methods:
+        parse: Parse the loaded elements.
+    """
+    def parse(self, loaded_elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        """Parse the loaded elements.
+
+        This method defines the process of defining text structure of the loaded elements based on metadata
+        for PDF loaded elements.
+
+        Args:
+            loaded_elements (list[dict[str, Any]]): A list of dictionaries containing loaded element
+                content and metadata.
+            **kwargs (Any): Additional keyword arguments.
+
+        Kwargs:
+            header_footer_tolerance (int, optional): An integer value indicating the tolerance for header and footer.
+                Defaults to 0.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries containing parsed element content and metadata.
+        """

gllm_docproc/parser/document/txt_parser.pyi

@@ -0,0 +1,22 @@
+from gllm_docproc.model.element import Element as Element, PARAGRAPH as PARAGRAPH, UNCATEGORIZED_TEXT as UNCATEGORIZED_TEXT
+from gllm_docproc.parser.base_parser import BaseParser as BaseParser
+from typing import Any
+
+class TXTParser(BaseParser):
+    """TXT parser for parsing text files.
+
+    Methods:
+        parse: Parse a list of elements from a text file.
+    """
+    def parse(self, loaded_elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        """Parse a list of elements from a text file.
+
+        all elements with structure UNCATEGORIZED_TEXT will be converted to PARAGRAPH
+
+        Args:
+            loaded_elements (list[dict[str, Any]]): The list of elements that have already been loaded.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            list[dict[str, Any]]: A list of elements.
+        """

gllm_docproc/parser/document/xlsx_parser.pyi

@@ -0,0 +1,26 @@
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.parser import BaseParser as BaseParser
+from typing import Any
+
+DEFAULT_SHEET_NAME_PATTERN: str
+
+class XLSXParser(BaseParser):
+    """A XLSX parser for parsing XLSX document text structure.
+
+    This class serves as the XLSX parser for parsing XLSX document text structure.
+    It defines the structure for parsing XLSX document text structure from a given loaded_elements.
+
+    Methods:
+        parse(loaded_elements, **kwargs): Parse the document from the loaded elements.
+    """
+    def parse(self, loaded_elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        """Parse loaded elements by assigning a structure to each element.
+
+        Args:
+            loaded_elements (list[dict[str, Any]]): A list of dictionaries representing loaded elements.
+            **kwargs (Any): Additional arguments for parsing the document.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries containing parsed elements with assigned structures.
+
+        """

gllm_docproc/parser/html/__init__.pyi

@@ -0,0 +1,4 @@
+from .flat.html_flat_parser import HTMLFlatParser as HTMLFlatParser
+from .nested.html_nested_parser import HTMLNestedParser as HTMLNestedParser
+
+__all__ = ['HTMLFlatParser', 'HTMLNestedParser']

gllm_docproc/parser/html/flat/html_flat_parser.pyi

@@ -0,0 +1,27 @@
+from gllm_docproc.model.element import PARAGRAPH as PARAGRAPH
+from gllm_docproc.parser.base_parser import BaseParser as BaseParser
+from gllm_docproc.utils.html_constants import ItemDataKeys as ItemDataKeys, Structure as Structure
+from typing import Any
+
+class HTMLFlatParser(BaseParser):
+    """This class extends the BaseParser and is specifically designed for parsing elements loaded from web content.
+
+    It assigns a structure to each loaded element based on the HTML tags present in its metadata.
+
+    Attributes:
+        None
+
+    Methods:
+        parse(loaded_elements: list[dict], **kwargs: dict[str, Any]) -> list[dict]:
+            Parses the loaded_elements and assigns a structure to each element based on its HTML tags.
+    """
+    def parse(self, loaded_elements: list[dict[str, Any]], **kwargs: dict[str, Any]) -> list[dict[str, Any]]:
+        """Parses the loaded_elements and assigns a structure to each element based on its HTML tags.
+
+        Args:
+            loaded_elements (list[dict]): The elements loaded from web content to be parsed.
+            **kwargs (dict[str, Any]): Additional keyword arguments.
+
+        Returns:
+            list[dict]: The parsed elements with assigned structures.
+        """

gllm_docproc/parser/html/nested/html_json_processor.pyi

@@ -0,0 +1,158 @@
+from _typeshed import Incomplete
+from gllm_docproc.model.element import PARAGRAPH as PARAGRAPH, TABLE as TABLE, TITLE as TITLE
+from gllm_docproc.parser.html.nested.nested_element import NestedElement as NestedElement
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, ErrorMessage as ErrorMessage, FORMATTING_TAGS as FORMATTING_TAGS, HTMLTags as HTMLTags, ItemDataKeys as ItemDataKeys, MetaDataKeys as MetaDataKeys, SPACING as SPACING, Structure as Structure, TableConstants as TableConstants
+
+class HTMLJsonProcessor:
+    """Processor for processing items scraped by the spider.
+
+    This pipeline processes the raw data scraped by the spider, formats it, and stores it in a JSON format.
+    It also handles errors during the processing and logging of the data.
+
+    Attributes:
+        logger: An instance of a logger, used for logging runtime information.
+        element_id: A counter for the elements processed by the pipeSline.
+        processor_result: A dictionary that holds the processed data.
+    """
+    logger: Incomplete
+    element_id: int
+    processor_result: Incomplete
+    def __init__(self) -> None:
+        """Initialize the HTMLJsonProcessor."""
+    def process_item(self, item: list[dict]):
+        """Processes each item passed by the spider.
+
+        The method formats the raw data and stores it in the processor_result dictionary.
+
+        Args:
+            item (list): The raw data scraped by the spider.
+
+        Returns:
+            list: The processed item.
+        """
+    def add_title_element(self, item) -> None:
+        """Adds the title element to the processor_result dictionary.
+
+        Args:
+            item (dict): The raw data scraped by the spider.
+        """
+    def extract_data(self, current: dict, data: NestedElement):
+        """Extracts data from the raw data.
+
+        This method traverses the raw data and extracts the necessary information.
+
+        Args:
+            current (dict): The current node in the raw data.
+            data (NestedElement): The dictionary where the extracted data is stored.
+        """
+    def handle_table_data(self, current, data: NestedElement):
+        """Handles table content.
+
+        Args:
+            current (dict): The current node in the raw data. It should contain the table content and metadata.
+            data (dict): The dictionary where the extracted data is stored.
+        """
+    def handle_media_data(self, current, data: NestedElement):
+        """Handles media content.
+
+        Args:
+            current (dict): The current node in the raw data.
+            data (dict): The dictionary where the extracted data is stored.
+        """
+    def handle_string_content(self, current, data: NestedElement):
+        """Handles string content.
+
+        Args:
+            current (dict): The current node in the raw data.
+            data (dict): The dictionary where the extracted data is stored.
+        """
+    def handle_other_cases(self, current, data: NestedElement):
+        """Handles other cases.
+
+        Args:
+            current (dict): The current node in the raw data.
+            data (dict): The dictionary where the extracted data is stored.
+        """
+    def handle_current_tag(self, current, data: NestedElement) -> tuple[NestedElement, dict]:
+        """Handles the current tag. This method checks the current tag and updates the data accordingly.
+
+        Args:
+            current (dict): The current node in the raw data.
+            data (dict): The dictionary where the extracted data is stored.
+
+        Returns:
+            NestedElement: The updated NestedElement object.
+            dict: A dictionary containing additional arguments.
+        """
+    def handle_content(self, current, data: NestedElement, args: dict):
+        """Handles content. This method iterates over the content and extracts the necessary information.
+
+        Args:
+            current (dict): The current node in the raw data.
+            data (NestedElement): The dictionary where the extracted data is stored.
+            args (dict): The dictionary containing the arguments for the method.
+        """
+    def add_result(self, data: NestedElement):
+        """Adds the processed data to the processor_result dictionary.
+
+        Args:
+            data (dict): The processed data.
+        """
+    def add_link(self, data: NestedElement) -> NestedElement:
+        """Adds a link to the processed data content.
+
+        Args:
+            data (dict): The processed data.
+
+        Returns:
+            dict: The processed data.
+        """
+    def add_index(self, data: NestedElement) -> NestedElement:
+        """Adds a index to the processed data content.
+
+        Args:
+            data (dict): The processed data.
+
+        Returns:
+            dict: The processed data.
+        """
+    def handle_media(self, current, data: NestedElement) -> NestedElement:
+        """Handles media content.
+
+        Args:
+            current (dict): The current node in the raw data.
+            data (dict): The dictionary where the extracted data is stored.
+
+        Returns:
+            dict: The processed data.
+        """
+    def handle_table(self, current, data: NestedElement) -> list:
+        """Handle Table.
+
+        This method processes table content by iterating over its metadata, handling each row based on its type,
+        and appending the result to the table data.
+
+        Args:
+            current (dict): The current node in the raw data. It should contain the table content and metadata.
+            data (dict): The dictionary where the extracted data is stored. This method adds a 'structure' key with the
+                         value 'table', and appends the extracted table data to this dictionary.
+
+        Returns:
+            list: A list of dictionaries containing the extracted table data.
+        """
+    def print_row(self, row, col_size: Incomplete | None = None):
+        """Formats a table row.
+
+        Args:
+            row (list): The row to be formatted.
+            col_size (list | None, optional): The size of the columns. Defaults to None.
+
+        Returns:
+            str: The formatted row.
+        """
+    def print_table_separator(self, row):
+        """Formats a table separator.
+
+        Returns:
+            str: The formatted table separator.
+        """

gllm_docproc/parser/html/nested/html_nested_parser.pyi

@@ -0,0 +1,24 @@
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.parser.base_parser import BaseParser as BaseParser
+from gllm_docproc.parser.html.nested.html_json_processor import HTMLJsonProcessor as HTMLJsonProcessor
+from typing import Any
+
+class HTMLNestedParser(BaseParser):
+    """A parser class for processing JSON elements into a parsed elements.
+
+    This class inherits from the BaseParser class and implements the parse method
+    to convert loaded HTML elements into a processed JSON format.
+
+    Attributes:
+        None
+    """
+    def parse(self, loaded_elements: list[dict[str, Any]], **kwargs: dict[str, Any]) -> list[dict[str, Any]]:
+        """Processes loaded HTML elements into a JSON format.
+
+        Args:
+            loaded_elements (dict): The loaded HTML elements to be processed.
+            **kwargs (dict[str, Any]): Additional keyword arguments.
+
+        Returns:
+            dict: The processed JSON representation of the HTML elements.
+        """

gllm_docproc/parser/html/nested/nested_element.pyi

@@ -0,0 +1,31 @@
+from gllm_docproc.model.element import Element as Element
+
+class NestedElement(Element):
+    """A specialized class extending Element to represent nested elements.
+
+    This class includes additional functionality specific to nested elements, such as generating
+    a unique element_id and providing methods to convert the instance to a dictionary or Element.
+
+    Attributes:
+        element_id (int): A unique identifier for the nested element.
+
+    Methods:
+        to_dict(): Convert the NestedElement instance to a dictionary.
+        to_element(): Convert the NestedElement instance to an Element.
+    """
+    element_id: int
+    def to_dict(self):
+        """Convert the NestedElement instance to a dictionary.
+
+        Returns:
+            dict: A dictionary representation of the NestedElement instance.
+        """
+    def to_element(self) -> Element:
+        """Convert the NestedElement instance to an Element.
+
+        This method creates an Element instance from the current NestedElement. It deep copies the metadata,
+        assigns the element_id, and constructs an Element with the associated text, metadata, and structure.
+
+        Returns:
+            Element: The Element instance created from the NestedElement.
+        """

gllm_docproc/parser/pipeline_parser.pyi

@@ -0,0 +1,33 @@
+from gllm_docproc.parser.base_parser import BaseParser as BaseParser
+from typing import Any
+
+class PipelineParser:
+    """Pipeline parser for parsing documents.
+
+    This class serves as the pipeline parser for parsing documents. It defines the structure for
+    parsing documents with several parsers using pipeline.
+
+    Methods:
+        add_parser(parser): Add parser to the pipeline parser.
+        parse(elements, **kwargs): Parse the elements using parsers.
+    """
+    parsers: list[BaseParser]
+    def __init__(self) -> None:
+        """Initialize the pipeline parser."""
+    def add_parser(self, parser: BaseParser):
+        """Add parser to the pipeline parser.
+
+        This method defines the process of adding parser to the pipeline parser.
+
+        Args:
+            parser (BaseParser): The parser to be added.
+        """
+    def parse(self, elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        """Parse the elements using pipeline parser.
+
+        This method defines the process of parsing the elements using parsers.
+
+        Args:
+            elements (list[dict[str, Any]]): A list of dictionaries containing elements.
+            **kwargs (Any): Additional keyword arguments.
+        """

gllm_docproc/parser/table/__init__.pyi

@@ -0,0 +1,3 @@
+from .table_caption_parser import TableCaptionParser as TableCaptionParser
+
+__all__ = ['TableCaptionParser']

gllm_docproc/parser/table/table_caption_parser.pyi

@@ -0,0 +1,66 @@
+from _typeshed import Incomplete
+from gllm_docproc.model.element import Element as Element, HEADING as HEADING, PARAGRAPH as PARAGRAPH, TABLE as TABLE, TITLE as TITLE, UNCATEGORIZED_TEXT as UNCATEGORIZED_TEXT
+from gllm_docproc.parser.base_parser import BaseParser as BaseParser
+from typing import Any
+
+TABLE_AND_CAPTION_STRUCTURE: Incomplete
+UPPER_ELEMENT_IS_CAPTION: str
+LOWER_ELEMENT_IS_CAPTION: str
+MAX_CAPTION_LENGTH: str
+REMOVE_CAPTION_FROM_ELEMENT: str
+MAX_CAPTION_ELEMENTS: str
+UPPER_CAPTION_EXTRACTOR: str
+LOWER_CAPTION_EXTRACTOR: str
+
+def curry_upper_caption_extractor(remove_caption_from_element: bool):
+    """Curry Upper Caption Extractor.
+
+    This function curries the extract_upper_caption function with the remove_caption_from_element parameters.
+
+    Why we need to use currying?
+    1. so user can customize the upper_caption_extractor function
+    2. the customize upper_caption_extractor may not require the remove_caption_from_element parameter
+
+    Args:
+        remove_caption_from_element (bool): A boolean value to remove the caption from the element.
+
+    Returns:
+        function: The function to extract the upper caption.
+    """
+def curry_lower_caption_extractor(remove_caption_from_element: bool):
+    """Curry Lower Caption Extractor.
+
+    This function curries the extract_lower_caption function with the remove_caption_from_element parameters.
+
+    Why we need to use currying?
+    1. so user can customize the lower_caption_extractor function
+    2. the customize lower_caption_extractor may not require the remove_caption_from_element parameter
+
+    Args:
+        remove_caption_from_element (bool): A boolean value to remove the caption from the element.
+
+    Returns:
+        function: The function to extract the lower caption.
+    """
+
+class TableCaptionParser(BaseParser):
+    """TableCaptionParser class.
+
+    A class to extract table captions from a document and add them to the metadata of the table element.
+
+    Methods:
+        parse(loaded_elements, **kwargs): Extract table captions from a document and add them to
+            the metadata of the table element.
+    """
+    def parse(self, loaded_elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        """Parses the elements to extract table captions.
+
+        This method extracts table captions from the elements and adds them to the metadata of the table element.
+
+        Args:
+            loaded_elements (list[dict[str, Any]]): The elements to extract table captions from.
+            **kwargs (Any): Additional keyword arguments for customization.
+
+        Returns:
+            list[dict[str, Any]]: The elements with the table captions added to the metadata.
+        """

gllm_docproc/request_handler/__init__.pyi

@@ -0,0 +1,3 @@
+from .base_request_handler import BaseRequestHandler as BaseRequestHandler
+
+__all__ = ['BaseRequestHandler']

gllm_docproc/request_handler/base_request_handler.pyi

@@ -0,0 +1,17 @@
+import abc
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseRequestHandler(ABC, metaclass=abc.ABCMeta):
+    """Base class for request handler."""
+    @abstractmethod
+    def handle_request(self, **kwargs: Any) -> None:
+        """Handles a request.
+
+        Args:
+            **kwargs (Any): Arbitrary keyword arguments.
+                            The implementing class is responsible to define the arguments
+
+        Returns:
+            None
+        """

gllm_docproc/response_handler/__init__.pyi

@@ -0,0 +1,3 @@
+from .base_response_handler import BaseResponseHandler as BaseResponseHandler
+
+__all__ = ['BaseResponseHandler']

gllm_docproc/response_handler/base_response_handler.pyi

@@ -0,0 +1,39 @@
+import abc
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseResponseHandler(ABC, metaclass=abc.ABCMeta):
+    """Base class for document converter."""
+    @abstractmethod
+    def handle_success_response(self, **kwargs: Any) -> None:
+        """Handles a success response (successfully indexed).
+
+        Args:
+            **kwargs (Any): Arbitrary keyword arguments.
+                                       The implementing class is responsible to define the arguments
+
+        Returns:
+            None
+        """
+    @abstractmethod
+    def handle_deleted_response(self, **kwargs: Any) -> None:
+        """Handles a deleted response (successfully deleted).
+
+        Args:
+            **kwargs (Any): Arbitrary keyword arguments.
+                                       The implementing class is responsible to define the arguments
+
+        Returns:
+            None
+        """
+    @abstractmethod
+    def handle_failed_response(self, **kwargs: Any) -> None:
+        """Handles a failed response (either failed to index or failed to delete).
+
+        Args:
+            **kwargs (Any): Arbitrary keyword arguments.
+                                       The implementing class is responsible to define the arguments
+
+        Returns:
+            None
+        """