gllm-docproc-binary 0.7.26__cp311-cp311-macosx_13_0_arm64.whl

gllm_docproc/parser/image/image_plain_small_filter_parser.pyi

@@ -0,0 +1,45 @@
+from _typeshed import Incomplete
+from gllm_docproc.model.element import Element as Element, IMAGE as IMAGE
+from gllm_docproc.parser.base_parser import BaseParser as BaseParser
+from typing import Any
+
+logger: Incomplete
+
+class ImagePlainSmallFilterParser(BaseParser):
+    """ImagePlainSmallFilterParser class.
+
+    A class to filter image elements from a document based on size requirements
+    and meaningful content analysis. This parser focuses on filtering operations
+    and does not perform coordinate-based transformations.
+
+    The parser filters images by:
+    1. Checking minimum dimension requirements
+    2. Validating meaningful content using contrast analysis
+
+    Methods:
+        parse(loaded_elements, **kwargs): Filter image elements in the document.
+    """
+    min_width: Incomplete
+    min_height: Incomplete
+    contrast_threshold: Incomplete
+    def __init__(self, min_width: int = 4, min_height: int = 4, contrast_threshold: float = 0.05) -> None:
+        """Initialize the ImageFilterParser.
+
+        Args:
+            min_width (int, optional): Minimum required width. Defaults to 4.
+            min_height (int, optional): Minimum required height. Defaults to 4.
+            contrast_threshold (float, optional): Contrast quality threshold. Defaults to 0.05.
+        """
+    def parse(self, loaded_elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        """Filter image elements in the document.
+
+        This method processes image elements by validating dimensions and content quality.
+        Images that don't meet the criteria are removed from the document.
+
+        Args:
+            loaded_elements (list[dict[str, Any]]): The elements to process.
+            **kwargs (Any): Additional keyword arguments for customization.
+
+        Returns:
+            list[dict[str, Any]]: The filtered elements with valid images only.
+        """

gllm_docproc/parser/pipeline_parser.pyi

@@ -0,0 +1,33 @@
+from gllm_docproc.parser.base_parser import BaseParser as BaseParser
+from typing import Any
+
+class PipelineParser:
+    """Pipeline parser for parsing documents.
+
+    This class serves as the pipeline parser for parsing documents. It defines the structure for
+    parsing documents with several parsers using pipeline.
+
+    Methods:
+        add_parser(parser): Add parser to the pipeline parser.
+        parse(elements, **kwargs): Parse the elements using parsers.
+    """
+    parsers: list[BaseParser]
+    def __init__(self) -> None:
+        """Initialize the pipeline parser."""
+    def add_parser(self, parser: BaseParser):
+        """Add parser to the pipeline parser.
+
+        This method defines the process of adding parser to the pipeline parser.
+
+        Args:
+            parser (BaseParser): The parser to be added.
+        """
+    def parse(self, elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        """Parse the elements using pipeline parser.
+
+        This method defines the process of parsing the elements using parsers.
+
+        Args:
+            elements (list[dict[str, Any]]): A list of dictionaries containing elements.
+            **kwargs (Any): Additional keyword arguments.
+        """

gllm_docproc/parser/table/__init__.pyi

@@ -0,0 +1,3 @@
+from .table_caption_parser import TableCaptionParser as TableCaptionParser
+
+__all__ = ['TableCaptionParser']

gllm_docproc/parser/table/table_caption_parser.pyi

@@ -0,0 +1,66 @@
+from _typeshed import Incomplete
+from gllm_docproc.model.element import Element as Element, HEADING as HEADING, PARAGRAPH as PARAGRAPH, TABLE as TABLE, TITLE as TITLE, UNCATEGORIZED_TEXT as UNCATEGORIZED_TEXT
+from gllm_docproc.parser.base_parser import BaseParser as BaseParser
+from typing import Any
+
+TABLE_AND_CAPTION_STRUCTURE: Incomplete
+UPPER_ELEMENT_IS_CAPTION: str
+LOWER_ELEMENT_IS_CAPTION: str
+MAX_CAPTION_LENGTH: str
+REMOVE_CAPTION_FROM_ELEMENT: str
+MAX_CAPTION_ELEMENTS: str
+UPPER_CAPTION_EXTRACTOR: str
+LOWER_CAPTION_EXTRACTOR: str
+
+def curry_upper_caption_extractor(remove_caption_from_element: bool):
+    """Curry Upper Caption Extractor.
+
+    This function curries the extract_upper_caption function with the remove_caption_from_element parameters.
+
+    Why we need to use currying?
+    1. so user can customize the upper_caption_extractor function
+    2. the customize upper_caption_extractor may not require the remove_caption_from_element parameter
+
+    Args:
+        remove_caption_from_element (bool): A boolean value to remove the caption from the element.
+
+    Returns:
+        function: The function to extract the upper caption.
+    """
+def curry_lower_caption_extractor(remove_caption_from_element: bool):
+    """Curry Lower Caption Extractor.
+
+    This function curries the extract_lower_caption function with the remove_caption_from_element parameters.
+
+    Why we need to use currying?
+    1. so user can customize the lower_caption_extractor function
+    2. the customize lower_caption_extractor may not require the remove_caption_from_element parameter
+
+    Args:
+        remove_caption_from_element (bool): A boolean value to remove the caption from the element.
+
+    Returns:
+        function: The function to extract the lower caption.
+    """
+
+class TableCaptionParser(BaseParser):
+    """TableCaptionParser class.
+
+    A class to extract table captions from a document and add them to the metadata of the table element.
+
+    Methods:
+        parse(loaded_elements, **kwargs): Extract table captions from a document and add them to
+            the metadata of the table element.
+    """
+    def parse(self, loaded_elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        """Parses the elements to extract table captions.
+
+        This method extracts table captions from the elements and adds them to the metadata of the table element.
+
+        Args:
+            loaded_elements (list[dict[str, Any]]): The elements to extract table captions from.
+            **kwargs (Any): Additional keyword arguments for customization.
+
+        Returns:
+            list[dict[str, Any]]: The elements with the table captions added to the metadata.
+        """

gllm_docproc/request_handler/__init__.pyi

@@ -0,0 +1,3 @@
+from .base_request_handler import BaseRequestHandler as BaseRequestHandler
+
+__all__ = ['BaseRequestHandler']

gllm_docproc/request_handler/base_request_handler.pyi

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseRequestHandler(ABC):
+    """Base class for request handler."""
+    @abstractmethod
+    def handle_request(self, **kwargs: Any) -> None:
+        """Handles a request.
+
+        Args:
+            **kwargs (Any): Arbitrary keyword arguments.
+                            The implementing class is responsible to define the arguments
+
+        Returns:
+            None
+        """

gllm_docproc/response_handler/__init__.pyi

@@ -0,0 +1,3 @@
+from .base_response_handler import BaseResponseHandler as BaseResponseHandler
+
+__all__ = ['BaseResponseHandler']

gllm_docproc/response_handler/base_response_handler.pyi

@@ -0,0 +1,38 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseResponseHandler(ABC):
+    """Base class for document converter."""
+    @abstractmethod
+    def handle_success_response(self, **kwargs: Any) -> None:
+        """Handles a success response (successfully indexed).
+
+        Args:
+            **kwargs (Any): Arbitrary keyword arguments.
+                                       The implementing class is responsible to define the arguments
+
+        Returns:
+            None
+        """
+    @abstractmethod
+    def handle_deleted_response(self, **kwargs: Any) -> None:
+        """Handles a deleted response (successfully deleted).
+
+        Args:
+            **kwargs (Any): Arbitrary keyword arguments.
+                                       The implementing class is responsible to define the arguments
+
+        Returns:
+            None
+        """
+    @abstractmethod
+    def handle_failed_response(self, **kwargs: Any) -> None:
+        """Handles a failed response (either failed to index or failed to delete).
+
+        Args:
+            **kwargs (Any): Arbitrary keyword arguments.
+                                       The implementing class is responsible to define the arguments
+
+        Returns:
+            None
+        """

gllm_docproc/utils/__init__.pyi

@@ -0,0 +1,3 @@
+from .async_utils import run_async_in_sync as run_async_in_sync
+
+__all__ = ['run_async_in_sync']

gllm_docproc/utils/async_utils.pyi

@@ -0,0 +1,22 @@
+from typing import Awaitable, TypeVar
+
+T = TypeVar('T')
+
+def run_async_in_sync(coro: Awaitable[T]) -> T:
+    '''Run an async coroutine from synchronous code safely.
+
+    This function handles the common scenario where you need to call an async function
+    from synchronous code, but you\'re not sure if there\'s already an event loop running.
+
+    Args:
+        coro (Awaitable[T]): The coroutine to run.
+
+    Returns:
+        T: The result of the coroutine.
+
+    Example:
+        >>> async def fetch_data():
+        ...     return "data"
+        >>> result = run_async_in_sync(fetch_data())
+        >>> print(result)  # "data"
+    '''

gllm_docproc/utils/file_utils.pyi

@@ -0,0 +1,76 @@
+from typing import Any
+
+def create_folder(folder_path: str) -> None:
+    """Create a folder.
+
+    This function check if the folder path exists. If the folder path does not
+    exist, the function creates a folder in the specified folder path.
+
+    Args:
+        folder_path (str): The folder path to create.
+    """
+def create_full_path(dir_path: str, filename: str, file_extension: str) -> str:
+    """Create a full path for a file.
+
+    This function creates a full path for a file by combining the directory
+    path, the filename, and the file extension.
+
+    Args:
+        dir_path (str): The directory path.
+        filename (str): The filename.
+        file_extension (str): The file extension.
+
+    Returns:
+        str: The full path for the file.
+    """
+def save_to_json(elements: list[dict[str, Any]] | dict[str, Any], folder_path: str, file_name: str) -> str:
+    """Save a list of elements to a JSON file.
+
+    This function saves a list of elements to a JSON file. The function takes
+    the list of elements, the folder path, and the file name as input and saves
+    the elements to a JSON file in the specified folder.
+
+    Args:
+        elements (list[dict[str, Any]] | dict[str, Any]): The list of elements to save.
+        folder_path (str): The folder path to save the JSON file.
+        file_name (str): The file name of the JSON file.
+
+    Returns:
+        str: The full filepath of the created JSON file.
+    """
+def save_to_csv(elements: list[dict[str, Any]], folder_path: str, file_name: str) -> None:
+    """Save a list of elements to a CSV file.
+
+    This function saves a list of elements to a CSV file. The function takes
+    the list of elements, the folder path, and the file name as input and saves
+    the elements to a CSV file in the specified folder.
+
+    Args:
+        elements (list[dict[str, Any]]): The list of elements to save.
+        folder_path (str): The folder path to save the CSV file.
+        file_name (str): The file name of the CSV file.
+
+    Returns:
+        None
+    """
+def save_file(content: str, filename: str):
+    """Save the content to a file.
+
+    Args:
+        content (str): The content to save.
+        filename (str): The filename to save the content to.
+
+    Returns:
+        None
+    """
+def read_json_file(file_path: str) -> list[dict[str, Any]] | dict[str, Any]:
+    """Read a JSON file.
+
+    This function reads a JSON file and returns the content of the JSON file.
+
+    Args:
+        file_path (str): The path of the JSON file to read.
+
+    Returns:
+        list[dict[str, Any]] | dict[str, Any]: The content of the JSON file.
+    """

gllm_docproc/utils/html_constants.pyi

@@ -0,0 +1,122 @@
+from _typeshed import Incomplete
+from gllm_docproc.model.element import AUDIO as AUDIO, FOOTER as FOOTER, HEADER as HEADER, HEADING as HEADING, IMAGE as IMAGE, TABLE as TABLE, TITLE as TITLE, VIDEO as VIDEO
+
+FORMATTING_TAGS: Incomplete
+SPACING: str
+
+class MetaDataKeys:
+    """Represents keys commonly used in metadata for web content."""
+    CHARSET: str
+    PROPERTY: str
+    CONTENT: str
+    NAME: str
+    HTTP_EQUIV: str
+    URL: str
+    TITLE: str
+    METADATA: str
+    SOURCE: str
+    SOURCE_TYPE: str
+    LOADED_DATETIME: str
+
+class ContentDataKeys:
+    """Represents keys commonly used in web content data."""
+    TAG: str
+    CONTENT: str
+    SOURCE: str
+    TYPE: str
+    SRC: str
+    PLACEHOLDER: str
+    TABLE: str
+    HREF: str
+    ALT: str
+    CLASS: str
+    VALUE: str
+
+class ItemDataKeys:
+    """Represents keys used for handling item data."""
+    ELEMENTS: str
+    TEXT: str
+    STRUCTURE: str
+    ELEMENT_ID: str
+    INDEX: str
+    LINK: str
+    FORMATS: str
+    COMBINE_PREV: str
+    LIST_TYPE: str
+    IS_LIST_FIRST_ITEM: str
+    METADATA: str
+    URL: str
+    GROUP_ID: str
+    PARENT_ID: str
+    LINE_BREAK: str
+    HTML_TAGS: str
+    ROW_ITEM: str
+    COLSPAN: str
+    ROWSPAN: str
+
+class HTMLTags:
+    """Represents commonly used HTML tags as constants."""
+    IMG: str
+    INPUT: str
+    SVG: str
+    SOURCE: str
+    TABLE: str
+    A: str
+    VIDEO: str
+    AUDIO: str
+    IFRAME: str
+    EMBED: str
+    TEXT: str
+    UL: str
+    OL: str
+    LI: str
+    P: str
+    BR: str
+    H: Incomplete
+    HEADER: str
+    TITLE: str
+    FOOTER: str
+    MEDIA_TAGS: Incomplete
+    TR: str
+    TD: str
+    TH: str
+    TBODY: str
+    TFOOT: str
+    THEAD: str
+    IMAGE_TAGS: Incomplete
+
+class ErrorMessage:
+    """Represents predefined error messages used in the application."""
+    ERROR_FAILED_SAVE_JSON: str
+    ERROR_FAILED_SAVE_CSV: str
+    ERROR_FAILED_EXTRACT_DATA: str
+    ERROR_MISSING_KEY: str
+    ERROR_FAILED_TO_PROCESS_ITEM: str
+    ERROR_FAILED_TO_OPEN_SPIDER: str
+    ERROR_UNKNOWN_SOURCE: str
+
+class Structure:
+    """Represents the structure of the content."""
+    @classmethod
+    def get_structure(cls, tag: str):
+        """Get the structure associated with the given HTML tag.
+
+        This class method maps HTML tags to their corresponding structure types and returns the
+        structure associated with the provided HTML tag.
+
+        Args:
+            tag (str): The HTML tag for which to retrieve the structure.
+
+        Returns:
+            str or None: The structure associated with the HTML tag, or None if the tag is not mapped.
+        """
+
+class TableConstants:
+    """Represents constants used for table extraction."""
+    TABLE_META_KEY: str
+    TABLE_CONTENT_KEY: str
+    TABLE_ROW_TYPE_KEY: str
+    MAX_CHAR_COUNT_PER_COLUMN: str
+    HEADER: str
+    BODY: str
+    FOOTER: str

gllm_docproc/validator/__init__.pyi

@@ -0,0 +1,6 @@
+from gllm_docproc.validator.character_count_validator import CharacterCountValidator as CharacterCountValidator
+from gllm_docproc.validator.file_size_validator import FileSizeValidator as FileSizeValidator
+from gllm_docproc.validator.page_count_validator import PageCountValidator as PageCountValidator
+from gllm_docproc.validator.pipeline_validator import PipelineValidator as PipelineValidator
+
+__all__ = ['PipelineValidator', 'CharacterCountValidator', 'FileSizeValidator', 'PageCountValidator']

gllm_docproc/validator/base_validator.pyi

@@ -0,0 +1,34 @@
+from _typeshed import Incomplete
+from abc import ABC
+from gllm_docproc.validator.model.validator_input import ValidatorInput as ValidatorInput
+from gllm_docproc.validator.model.validator_result import ValidatorResult as ValidatorResult
+
+class BaseValidator(ABC):
+    """Abstract base class for file validators.
+
+    This class defines the interface that all file validators must implement.
+    Each validator should validate a specific aspect of a file and return
+    a ValidatorResult indicating success/failure and an appropriate message.
+    """
+    stop_on_failure: Incomplete
+    applicable_extensions: Incomplete
+    logger: Incomplete
+    def __init__(self, stop_on_failure: bool = False, applicable_extensions: list[str] | None = None) -> None:
+        """Initialize the BaseValidator.
+
+        Args:
+            stop_on_failure (bool, optional): Whether to terminate the validation process if this validator fails.
+                Default is False.
+            applicable_extensions (list[str] | None, optional): The list of file extensions that this validator is
+                applicable to. Default is None which means all extensions are applicable.
+        """
+    def validate(self, file_validation_input: ValidatorInput) -> ValidatorResult:
+        """Validate the file against the validator's criteria.
+
+        Args:
+            file_validation_input (ValidatorInput): The ValidatorInput object to validate.
+
+        Returns:
+            ValidatorResult: A ValidatorResult object indicating success or failure
+                with an appropriate message.
+        """

gllm_docproc/validator/character_count_validator.pyi

@@ -0,0 +1,26 @@
+from _typeshed import Incomplete
+from gllm_docproc.loader.csv.pandas_loader import CSV_VARIANTS as CSV_VARIANTS
+from gllm_docproc.validator.base_validator import BaseValidator as BaseValidator
+from gllm_docproc.validator.model.validator_input import ValidatorInput as ValidatorInput
+from gllm_docproc.validator.model.validator_result import ValidatorResult as ValidatorResult
+
+class CharacterCountValidator(BaseValidator):
+    """Validator for checking if the total character length of file content does not exceed a maximum limit.
+
+    Character length counting is currently supported for:
+        - CSV files (csv, tsv, psv, ssv)
+        - TXT files
+    """
+    CHUNK_SIZE_BYTES: Incomplete
+    max_character_length: Incomplete
+    def __init__(self, max_character_length: int = 500000, stop_on_failure: bool = False, applicable_extensions: list[str] | None = None) -> None:
+        """Initialize the CharacterCountValidator.
+
+        Args:
+            max_character_length (int, optional): The maximum allowed character length for the file.
+                Default is 500,000 characters. It should be greater than 0.
+            stop_on_failure (bool, optional): Whether to stop the validation process if this
+                validator fails. Default is False.
+            applicable_extensions (list[str] | None, optional): The list of file extensions that this validator
+                is applicable to. Default is None which means all extensions are applicable.
+        """

gllm_docproc/validator/file_size_validator.pyi

@@ -0,0 +1,20 @@
+from _typeshed import Incomplete
+from gllm_docproc.validator.base_validator import BaseValidator as BaseValidator
+from gllm_docproc.validator.model.validator_input import ValidatorInput as ValidatorInput
+from gllm_docproc.validator.model.validator_result import ValidatorResult as ValidatorResult
+
+class FileSizeValidator(BaseValidator):
+    """Validator for checking if file size does not exceed a maximum limit."""
+    max_file_size: Incomplete
+    def __init__(self, max_file_size: int = 10485760, stop_on_failure: bool = True, applicable_extensions: list[str] | None = None) -> None:
+        """Initialize the FileSizeValidator.
+
+        Args:
+            max_file_size (int, optional): The maximum allowed size for the file in bytes.
+                Default is 10,485,760 bytes (10MB). It should be greater than 0.
+            stop_on_failure (bool, optional): Whether to stop the validation process if this
+                validator fails. Default is True.
+            applicable_extensions (list[str] | None, optional): The list of file extensions
+                that this validator is applicable to. Default is None which means
+                all extensions are applicable.
+        """

gllm_docproc/validator/model/__init__.pyi

@@ -0,0 +1,4 @@
+from .validator_input import ValidatorInput as ValidatorInput
+from .validator_result import ValidatorResult as ValidatorResult
+
+__all__ = ['ValidatorInput', 'ValidatorResult']

gllm_docproc/validator/model/validator_input.pyi

@@ -0,0 +1,50 @@
+from _typeshed import Incomplete
+from pydantic import BaseModel
+from types import TracebackType
+from typing import BinaryIO
+
+class ValidatorInput(BaseModel):
+    """File object used for validation.
+
+    Attributes:
+        name (str): File name (basename).
+        extension (str): File extension without leading dot, lowercased (e.g., 'pdf').
+        size (int): File size in bytes.
+        file (BinaryIO): Open binary file handle for content-based validations.
+        content_type (str | None): Optional content type (MIME), if known.
+    """
+    model_config: Incomplete
+    name: str
+    extension: str
+    size: int
+    file: BinaryIO
+    content_type: str | None
+    @classmethod
+    def from_path(cls, path: str) -> ValidatorInput:
+        """Create a ValidatorInput from a local path (opens in rb mode).
+
+        Args:
+            path (str): The file path to create ValidatorInput from.
+
+        Returns:
+            ValidatorInput: A ValidatorInput instance.
+        """
+    def close(self) -> None:
+        """Close the underlying file handle if owned by this object.
+
+        This method is idempotent and will not raise an error if the file is already closed.
+        """
+    def __enter__(self) -> ValidatorInput:
+        """Enter the runtime context related to this object.
+
+        Returns:
+            ValidatorInput: The ValidatorInput instance itself.
+        """
+    def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None) -> None:
+        """Exit the runtime context and close the file handle if owned.
+
+        Args:
+            exc_type (type[BaseException] | None): The exception type, if any.
+            exc_val (BaseException | None): The exception value, if any.
+            exc_tb (TracebackType | None): The traceback, if any.
+        """

gllm_docproc/validator/model/validator_result.pyi

@@ -0,0 +1,19 @@
+from pydantic import BaseModel
+from typing import Any
+
+class ValidatorResult(BaseModel):
+    """Represents the result of a validation operation.
+
+    This class encapsulates the result of a validation operation, including whether
+    the validation passed or failed and any associated message.
+
+    Attributes:
+        is_valid (bool): Whether the validation passed or failed.
+        source_validator (str): Validator class name that produced this result.
+        message (str): The message associated with the validation result.
+        params (dict[str, Any]): The parameters associated with the validation result.
+    """
+    is_valid: bool
+    source_validator: str
+    message: str
+    params: dict[str, Any]

gllm_docproc/validator/page_count_validator.pyi

@@ -0,0 +1,23 @@
+from _typeshed import Incomplete
+from gllm_docproc.validator.base_validator import BaseValidator as BaseValidator
+from gllm_docproc.validator.model.validator_input import ValidatorInput as ValidatorInput
+from gllm_docproc.validator.model.validator_result import ValidatorResult as ValidatorResult
+
+class PageCountValidator(BaseValidator):
+    """Validator for checking if the number of pages in a file does not exceed a maximum limit.
+
+    Page counting is currently supported for PDF files only.
+    """
+    max_pages: Incomplete
+    def __init__(self, max_pages: int = 100, stop_on_failure: bool = False, applicable_extensions: list[str] | None = None) -> None:
+        """Initialize the PageCountValidator.
+
+        Args:
+            max_pages (int, optional): The maximum allowed number of pages. A non-negative
+                value enforces a limit. Default is 100 pages. It should be greater than 0.
+            stop_on_failure (bool, optional): Whether to stop the validation process if this
+                validator fails. Default is False.
+            applicable_extensions (list[str] | None, optional): The list of file extensions
+                that this validator is applicable to. Default is None which means
+                all extensions are applicable.
+        """