PyPI - gllm-docproc-binary - Versions diffs - 0.1.8__cp312-cp312-macosx_14_0_arm64.whl - Mend

gllm-docproc-binary 0.1.8__cp312-cp312-macosx_14_0_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of gllm-docproc-binary might be problematic. Click here for more details.

Files changed (123) hide show

gllm_docproc/__init__.pyi +0 -0
gllm_docproc/chunker/__init__.pyi +3 -0
gllm_docproc/chunker/base_chunker.pyi +29 -0
gllm_docproc/chunker/structured_element/__init__.pyi +3 -0
gllm_docproc/chunker/structured_element/chunk_enricher.pyi +43 -0
gllm_docproc/chunker/structured_element/structured_element_chunker.pyi +80 -0
gllm_docproc/chunker/table/__init__.pyi +3 -0
gllm_docproc/chunker/table/table_chunker.pyi +45 -0
gllm_docproc/converter/__init__.pyi +3 -0
gllm_docproc/converter/base_converter.pyi +16 -0
gllm_docproc/data_generator/__init__.pyi +3 -0
gllm_docproc/data_generator/base_data_generator.pyi +19 -0
gllm_docproc/downloader/__init__.pyi +3 -0
gllm_docproc/downloader/base_downloader.pyi +16 -0
gllm_docproc/downloader/html/__init__.pyi +4 -0
gllm_docproc/downloader/html/exception/__init__.pyi +4 -0
gllm_docproc/downloader/html/exception/item_scrape_failed_exception.pyi +16 -0
gllm_docproc/downloader/html/exception/zyte_api_key_not_provided_exception.pyi +15 -0
gllm_docproc/downloader/html/html_downloader.pyi +91 -0
gllm_docproc/downloader/html/scraper/__init__.pyi +0 -0
gllm_docproc/downloader/html/scraper/scraper/__init__.pyi +0 -0
gllm_docproc/downloader/html/scraper/scraper/spiders/__init__.pyi +9 -0
gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_pdf_spider.pyi +27 -0
gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_sitemap_link_spider.pyi +29 -0
gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_sitemap_spider.pyi +61 -0
gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_spider.pyi +50 -0
gllm_docproc/downloader/html/scraper/scraper/spiders/playwright_scrape_spider.pyi +22 -0
gllm_docproc/downloader/html/scraper/scraper/spiders/scrape_spider.pyi +57 -0
gllm_docproc/downloader/html/scraper/scraper/spiders/zyte_scrape_spider.pyi +51 -0
gllm_docproc/downloader/html/scraper/web_scraper_executor.pyi +45 -0
gllm_docproc/downloader/html/utils/__init__.pyi +3 -0
gllm_docproc/downloader/html/utils/web_utils.pyi +21 -0
gllm_docproc/dpo_router/__init__.pyi +3 -0
gllm_docproc/dpo_router/base_dpo_router.pyi +17 -0
gllm_docproc/housekeeping/__init__.pyi +3 -0
gllm_docproc/housekeeping/base_housekeeping.pyi +15 -0
gllm_docproc/indexer/__init__.pyi +3 -0
gllm_docproc/indexer/base_indexer.pyi +31 -0
gllm_docproc/indexer/graph/__init__.pyi +3 -0
gllm_docproc/indexer/knowledge_graph/__init__.pyi +4 -0
gllm_docproc/loader/__init__.pyi +4 -0
gllm_docproc/loader/audio/__init__.pyi +3 -0
gllm_docproc/loader/base_loader.pyi +31 -0
gllm_docproc/loader/docx/__init__.pyi +5 -0
gllm_docproc/loader/docx/docx2python_loader.pyi +46 -0
gllm_docproc/loader/docx/python_docx_loader.pyi +35 -0
gllm_docproc/loader/docx/python_docx_table_loader.pyi +35 -0
gllm_docproc/loader/exception/__init__.pyi +3 -0
gllm_docproc/loader/exception/unsupported_file_extension_error.pyi +7 -0
gllm_docproc/loader/html/__init__.pyi +5 -0
gllm_docproc/loader/html/exception/__init__.pyi +3 -0
gllm_docproc/loader/html/exception/html_load_exception.pyi +7 -0
gllm_docproc/loader/html/flat/__init__.pyi +3 -0
gllm_docproc/loader/html/flat/html_flat_base_handler.pyi +52 -0
gllm_docproc/loader/html/flat/html_flat_loader.pyi +30 -0
gllm_docproc/loader/html/flat/html_flat_merger.pyi +22 -0
gllm_docproc/loader/html/html_base_loader.pyi +25 -0
gllm_docproc/loader/html/nested/__init__.pyi +3 -0
gllm_docproc/loader/html/nested/dictionary_utils.pyi +40 -0
gllm_docproc/loader/html/nested/html_nested_base_handler.pyi +128 -0
gllm_docproc/loader/html/nested/html_nested_element_handler.pyi +24 -0
gllm_docproc/loader/html/nested/html_nested_loader.pyi +15 -0
gllm_docproc/loader/html/utils/__init__.pyi +0 -0
gllm_docproc/loader/html/utils/flat_table_utils.pyi +29 -0
gllm_docproc/loader/html/utils/html_utils.pyi +41 -0
gllm_docproc/loader/html/utils/removed_components.pyi +53 -0
gllm_docproc/loader/html/utils/string_utils.pyi +33 -0
gllm_docproc/loader/html/utils/table_utils.pyi +78 -0
gllm_docproc/loader/json/__init__.pyi +3 -0
gllm_docproc/loader/json/json_elements_loader.pyi +33 -0
gllm_docproc/loader/loader_utils.pyi +42 -0
gllm_docproc/loader/pdf/__init__.pyi +13 -0
gllm_docproc/loader/pdf/adobe_pdf_extract_loader.pyi +40 -0
gllm_docproc/loader/pdf/azure_ai_document_intelligence_loader.pyi +45 -0
gllm_docproc/loader/pdf/azure_ai_document_intelligence_raw_loader.pyi +50 -0
gllm_docproc/loader/pdf/glair_vision_ocr_loader.pyi +38 -0
gllm_docproc/loader/pdf/pdf_loader_utils.pyi +59 -0
gllm_docproc/loader/pdf/pdf_miner_loader.pyi +38 -0
gllm_docproc/loader/pdf/pdf_miner_word_loader.pyi +33 -0
gllm_docproc/loader/pdf/pdf_plumber_loader.pyi +38 -0
gllm_docproc/loader/pdf/pymupdf_loader.pyi +43 -0
gllm_docproc/loader/pdf/pymupdf_span_loader.pyi +44 -0
gllm_docproc/loader/pdf/pymupdf_utils.pyi +34 -0
gllm_docproc/loader/pdf/tabula_loader.pyi +32 -0
gllm_docproc/loader/pdf/text_inject_pdf_plumber_loader.pyi +37 -0
gllm_docproc/loader/pipeline_loader.pyi +48 -0
gllm_docproc/loader/txt/__init__.pyi +3 -0
gllm_docproc/loader/txt/txt_loader.pyi +26 -0
gllm_docproc/loader/xlsx/__init__.pyi +3 -0
gllm_docproc/loader/xlsx/openpyxl_loader.pyi +37 -0
gllm_docproc/model/__init__.pyi +4 -0
gllm_docproc/model/element.pyi +37 -0
gllm_docproc/model/element_metadata.pyi +35 -0
gllm_docproc/parser/__init__.pyi +4 -0
gllm_docproc/parser/base_parser.pyi +29 -0
gllm_docproc/parser/document/__init__.pyi +6 -0
gllm_docproc/parser/document/docx_parser.pyi +27 -0
gllm_docproc/parser/document/pdf_parser.pyi +35 -0
gllm_docproc/parser/document/txt_parser.pyi +22 -0
gllm_docproc/parser/document/xlsx_parser.pyi +26 -0
gllm_docproc/parser/html/__init__.pyi +4 -0
gllm_docproc/parser/html/flat/__init__.pyi +0 -0
gllm_docproc/parser/html/flat/html_flat_parser.pyi +27 -0
gllm_docproc/parser/html/nested/__init__.pyi +0 -0
gllm_docproc/parser/html/nested/html_json_processor.pyi +158 -0
gllm_docproc/parser/html/nested/html_nested_parser.pyi +24 -0
gllm_docproc/parser/html/nested/nested_element.pyi +31 -0
gllm_docproc/parser/pipeline_parser.pyi +33 -0
gllm_docproc/parser/table/__init__.pyi +3 -0
gllm_docproc/parser/table/table_caption_parser.pyi +66 -0
gllm_docproc/request_handler/__init__.pyi +3 -0
gllm_docproc/request_handler/base_request_handler.pyi +17 -0
gllm_docproc/response_handler/__init__.pyi +3 -0
gllm_docproc/response_handler/base_response_handler.pyi +39 -0
gllm_docproc/utils/__init__.pyi +0 -0
gllm_docproc/utils/file_utils.pyi +76 -0
gllm_docproc/utils/html_constants.pyi +121 -0
gllm_docproc.build/.gitignore +1 -0
gllm_docproc.cpython-312-darwin.so +0 -0
gllm_docproc.pyi +149 -0
gllm_docproc_binary-0.1.8.dist-info/METADATA +110 -0
gllm_docproc_binary-0.1.8.dist-info/RECORD +123 -0
gllm_docproc_binary-0.1.8.dist-info/WHEEL +4 -0

gllm_docproc/loader/html/nested/dictionary_utils.pyi ADDED Viewed

@@ -0,0 +1,40 @@
+class DictionaryUtils:
+    """A utility class providing methods to manipulate dictionaries."""
+    @staticmethod
+    def add_or_skip_value(dictionary, key, value):
+        """Adds a value to a dictionary if the value is not None for a given key.
+        Args:
+            dictionary (dict): The dictionary to be modified.
+            key (hashable): The key where the value needs to be added.
+            value (any): The value to be added.
+        Returns:
+            dict: The modified dictionary.
+        """
+    @staticmethod
+    def append_value(dictionary, key, value):
+        """Appends a value to a list under a specific key in a dictionary.
+        If the key already exists in the dictionary, the value is appended to the list under that key.
+        If the key does not exist, a new list is created with the value as its first element.
+        Args:
+            dictionary (dict): The dictionary to be modified.
+            key (hashable): The key under which the value needs to be added.
+            value (any): The value to be appended to the list under the key.
+        Returns:
+            dict: The modified dictionary with the value appended to the list under the key.
+        """
+    @staticmethod
+    def put_key_to_bottom(dictionary, key):
+        """Rearange key in dictionary.
+        Args:
+            dictionary (dict): The dictionary to be modified.
+            key (hashable): The key.
+        Returns:
+            dict: The modified dictionary.
+        """

gllm_docproc/loader/html/nested/html_nested_base_handler.pyi ADDED Viewed

@@ -0,0 +1,128 @@
+from .html_nested_element_handler import get_element as get_element
+from gllm_docproc.loader.html.exception import HtmlLoadException as HtmlLoadException
+from gllm_docproc.loader.html.nested.dictionary_utils import DictionaryUtils as DictionaryUtils
+from gllm_docproc.loader.html.utils.removed_components import RemovedComponents as RemovedComponents
+from gllm_docproc.loader.html.utils.string_utils import StringUtils as StringUtils
+from gllm_docproc.loader.html.utils.table_utils import TableUtils as TableUtils
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, HTMLTags as HTMLTags
+def get_element_content(content_selector, removed_components: RemovedComponents) -> list[dict]:
+    '''Traverses each element to get the content.
+    This function extract html body recursively
+    Input example:
+    .. code-block:: html
+        <html>
+        <head>
+        <title>Title</title>
+        </head>
+        <body>
+            <div class="container">
+                <h1>Welcome to My Website</h1>
+                <div>
+                    Hello World
+                </div>
+            </div>
+            <p>This is another paragraph.</p>
+        </body>
+        </html>
+    Output:
+    .. code-block:: groovy
+        [
+            {
+                \'tag\': \'body\',
+                \'class\': None,
+                \'content\': [
+                    {
+                        \'tag\': \'div\',
+                        \'class\': \'container\',
+                        \'content\': [
+                            {
+                                \'tag\': \'h1\',
+                                \'class\': None, \'content\': [
+                                    {
+                                        \'tag\': \'text\',
+                                        \'content\': \'Welcome to My Website\'
+                                    }
+                                ]
+                            },
+                            {
+                                \'tag\': \'div\',
+                                \'class\': None,
+                                \'content\': [
+                                    {\'tag\': \'text\', \'content\': \'Hello World\'}
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        \'tag\': \'p\',
+                        \'class\': None,
+                        \'content\': [
+                            {\'tag\': \'text\', \'content\': \'This is another paragraph.\'}
+                        ]
+                    }
+                ]
+            }
+        ]
+    Args:
+        content_selector: The content to be traversed.
+        removed_components: Removed class or tags.
+    Returns:
+        The List of extracted contents.
+    '''
+def is_base_element(content_selector, removed_components: RemovedComponents) -> bool:
+    """Check if the given content selector represents a base element.
+    See html_flat_base_handler.py for more information.
+    Args:
+        content_selector: The content selector to check.
+        removed_components (RemovedComponents): An instance of RemovedComponents class.
+    Returns:
+        bool: True if the content_selector represents a base element; False otherwise.
+    """
+def handle_base_element(content_selector, removed_components: RemovedComponents) -> list[dict]:
+    """Handle the processing of a base HTML element.
+    Args:
+        content_selector: The content selector representing the HTML element.
+        removed_components (RemovedComponents): An object containing information about components to be removed.
+    Returns:
+        List of dict : A List of dictionaries containing information about the HTML element, or None
+                      if the element should be skipped.
+                      - tag: HTML tag name.
+                      - class: CSS class of the element (if available).
+                      Additional keys may be added based on the specific element handler.
+    """
+def get_handler(tag: str):
+    """Get the element information from the specified content selector.
+    Args:
+        tag (str): The HTML tag to get the handler for.
+    Returns:
+        dict: A dictionary containing information about the HTML element.
+              - tag: HTML tag name.
+              - class: CSS class of the element (if available).
+              Additional keys may be added based on the specific element handler
+    """
+def create_text_dict(message):
+    """Creates a dictionary with 'text' tag and specified content.
+    Args:
+        message: The content to be added to the dictionary.
+    Returns:
+        A dictionary with 'text' tag and specified content.
+    """

gllm_docproc/loader/html/nested/html_nested_element_handler.pyi ADDED Viewed

@@ -0,0 +1,24 @@
+from gllm_docproc.loader.html.nested.dictionary_utils import DictionaryUtils as DictionaryUtils
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, HTMLTags as HTMLTags
+def get_element(content_selector) -> dict:
+    """Get the element information from the specified content selector.
+    Args:
+        content_selector: The content selector representing the HTML element.
+    Returns:
+        dict: A dictionary containing information about the HTML element.
+              - tag: HTML tag name.
+              - class: CSS class of the element (if available).
+              Additional keys may be added based on the specific element handler
+    """
+def get_handler(tag: str):
+    """Gets the handler for the specified HTML tag.
+    Args:
+        tag (str): The HTML tag to get the handler for.
+    Returns:
+        Callable: The handler function for the specified HTML tag.
+    """

gllm_docproc/loader/html/nested/html_nested_loader.pyi ADDED Viewed

@@ -0,0 +1,15 @@
+from .html_nested_base_handler import get_element_content as get_element_content
+from gllm_docproc.loader.html.exception import HtmlLoadException as HtmlLoadException
+from gllm_docproc.loader.html.html_base_loader import HTMLBaseLoader as HTMLBaseLoader
+from gllm_docproc.loader.html.utils.html_utils import extract_html_head as extract_html_head
+from gllm_docproc.loader.html.utils.removed_components import RemovedComponents as RemovedComponents
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, MetaDataKeys as MetaDataKeys
+class HTMLNestedLoader(HTMLBaseLoader):
+    """A loader class for loading web content and extracting information.
+    This class inherits from the BaseLoader class and provides methods to load web content,
+    extract information, and scrape data using Scrapy spiders.
+    """
+    def __init__(self) -> None:
+        """Initialize the HTMLNestedLoader."""

gllm_docproc/loader/html/utils/__init__.pyi ADDED Viewed

File without changes

gllm_docproc/loader/html/utils/flat_table_utils.pyi ADDED Viewed

@@ -0,0 +1,29 @@
+from _typeshed import Incomplete
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.utils.html_constants import HTMLTags as HTMLTags, ItemDataKeys as ItemDataKeys, TableConstants as TableConstants
+class FlatTableUtils:
+    """A utility class providing methods for extracting data from HTML tables."""
+    colcount: int
+    rowspans: Incomplete
+    def __init__(self) -> None:
+        """Initialize the FlatTableUtils."""
+    def generate_tables(self, content: list[Element]) -> list[list[str]]:
+        """Generate tables from HTML content.
+        Args:
+            content (List[Element]): The list of Element instances representing the HTML content.
+        Returns:
+            List[List[str]]: A list containing the generated tables.
+        """
+    def filter_table(self, table_content: list[Element]) -> tuple[list[Element], list[Element]]:
+        """Filter the HTML table content.
+        Args:
+            table_content (List[Element]): The list of Element instances representing the HTML table.
+        Returns:
+            tuple[List[Element], List[Element]]: A tuple containing the filtered table content
+                and the removed elements.
+        """

gllm_docproc/loader/html/utils/html_utils.pyi ADDED Viewed

@@ -0,0 +1,41 @@
+from gllm_docproc.loader.html.exception import HtmlLoadException as HtmlLoadException
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata, HTML as HTML
+from gllm_docproc.utils.html_constants import HTMLTags as HTMLTags, MetaDataKeys as MetaDataKeys
+from scrapy.http import HtmlResponse as HtmlResponse
+from typing import Any
+def is_html_content(content: str) -> bool:
+    '''Check if the provided content appears to be HTML.
+    This function performs a case-insensitive check to determine if the content contains HTML tags,
+    specifically by searching for the opening and closing HTML tags ("<html" and "</html>").
+    Args:
+        content (str): The content to check.
+    Returns:
+        bool: True if the content is identified as HTML; False otherwise.
+    '''
+def extract_html_head(response: HtmlResponse, element_metadata: dict[str, Any] | None) -> ElementMetadata:
+    """Extracts metadata from an HTML response.
+    Args:
+        response (HtmlResponse): The HTML response.
+        element_metadata (dict[str, Any] | None): The element metadata.
+    Returns:
+        ElementMetadata: A class containing element metadata.
+    Raises:
+        HtmlLoadException: If an error occurs during the extraction process.
+    """
+def extract_html_title_tag(metadata: ElementMetadata) -> list[Element]:
+    """Gets the title element as a Element.
+    Args:
+        metadata (ElementMetadata): A class containing element metadata.
+    Returns:
+        List[Element]: List containing a single Element instance representing the title element.
+    """

gllm_docproc/loader/html/utils/removed_components.pyi ADDED Viewed

@@ -0,0 +1,53 @@
+class RemovedComponents:
+    """Class representing removed components from a document.
+    This class defines three methods for retrieving partial class, full class, and HTML tags
+    associated with removed components.
+    """
+    def get_partial_class(self) -> list[str]:
+        """Get partial class.
+        Method to get the partial class of the removed component. Partial class consists of
+        classes that will be filtered.
+        Returns:
+            str: The partial class name associated with the removed component.
+        """
+    def get_full_class(self) -> list[str]:
+        """Get full class.
+        Method to get the full class of the removed component. Full class consists of
+        exact match of classes that will be filtered.
+        Returns:
+            str: The full class name associated with the removed component.
+        """
+    def get_html_tags(self) -> list[str]:
+        """Method to get the HTML tags associated with the removed component.
+        Returns:
+            list: A list of HTML tags associated with the removed component.
+        """
+    @staticmethod
+    def is_removed_component(tag: str | None, class_: str | None, removed_components: RemovedComponents | None) -> bool:
+        """Checks if a component should be removed based on its tag and class.
+        Args:
+            tag (str): The tag of the component.
+            class_ (str): The class of the component.
+            removed_components (RemovedComponents): The components to be removed, including HTML tags and classes.
+        Returns:
+            True if the component should be removed, False otherwise.
+        """
+    @staticmethod
+    def check_list_in_substring(message: str, check_list: list[str]) -> bool:
+        """Checks if any substring from the check_list exists in the message string.
+        Args:
+            message (str): The string to search for substrings.
+            check_list (list): A list of substrings to be checked.
+        Returns:
+        - bool: True if any substring from check_list is found in message, otherwise False.
+        """

gllm_docproc/loader/html/utils/string_utils.pyi ADDED Viewed

@@ -0,0 +1,33 @@
+class StringUtils:
+    """A utility class providing methods for text cleaning."""
+    @staticmethod
+    def clean_text(text: str | None) -> str:
+        """Clean the input text by removing extra whitespace, newlines, and tabs.
+        Args:
+            text (str): The text to be cleaned.
+        Returns:
+            str: The cleaned text.
+        """
+    @staticmethod
+    def remove_extension(file_name: str) -> str:
+        """Removes the file extension from a given file name.
+        Args:
+            file_name (str): The name of the file from which the extension will be removed.
+        Returns:
+            str: File name without the extension.
+        """
+    @staticmethod
+    def append_character(text: str, new_char: str) -> str:
+        """Appends a character to the end of a string, handling newline endings.
+        Args:
+            text (str): The input text string to which the character will be appended.
+            new_char (str): The character to append to the text.
+        Returns:
+            str: The modified string with the appended character.
+        """

gllm_docproc/loader/html/utils/table_utils.pyi ADDED Viewed

@@ -0,0 +1,78 @@
+from _typeshed import Incomplete
+from gllm_docproc.loader.html.utils.string_utils import StringUtils as StringUtils
+from gllm_docproc.utils.html_constants import TableConstants as TableConstants
+from typing import Any
+class TableUtils:
+    """A utility class providing methods for extracting data from HTML tables."""
+    colcount: int
+    table_selector: Incomplete
+    rowspans: Incomplete
+    def __init__(self, table_selector) -> None:
+        """Initialize TableUtils with the given table selector.
+        Args:
+            table_selector: Selector for the HTML table.
+        """
+    def get_table(self) -> dict[str, Any]:
+        """Extract data from the HTML table and return it as a dictionary."""
+    def extract_table(self):
+        """Extract data from the HTML table and return it as a list of lists representing the table structure."""
+    def extract_table_row_type(self):
+        """Extract metadata from the HTML table and return it as a list of strings representing the row types."""
+    def update_col_count(self, row_cells, prev_rowspans) -> None:
+        """Update the number of columns in the table.
+        Args:
+            row_cells: List of HTML cells in a row.
+            prev_rowspans: List of previous rowspans.
+        """
+    def get_row_type(self, row):
+        """Get the type of the row.
+        Args:
+            row: HTML row.
+        """
+    def extract_max_char_count(self, table):
+        """Extract maximum character count.
+        Extract metadata from the HTML table and return it as a list of integers representing
+        the maximum number of characters in each column.
+        Args:
+            table: List of lists representing the table structure.
+        Returns:
+            list: A list of integers representing the maximum number of characters in each column.
+        """
+    @staticmethod
+    def convert_to_texts(table) -> list[str]:
+        """Convert to texts.
+        This method processes table content by iterating over its metadata, handling each row based
+        on its type, and appending the result to the table data.
+        Args:
+            table: table which will be converted to text
+        Returns:
+            list: A list of dictionaries containing the extracted table data.
+        """
+    @staticmethod
+    def print_row(row, col_size: Incomplete | None = None):
+        """Formats a table row.
+        Args:
+            row (list): The row to be formatted.
+            col_size (list | None): List of max characters size in each column
+        Returns:
+            str: The formatted row.
+        """
+    @staticmethod
+    def print_table_separator(row):
+        """Formats a table separator.
+        Returns:
+            str: The formatted table separator.
+        """

gllm_docproc/loader/json/__init__.pyi ADDED Viewed

@@ -0,0 +1,3 @@
+from .json_elements_loader import JSONElementsLoader as JSONElementsLoader
+__all__ = ['JSONElementsLoader']

gllm_docproc/loader/json/json_elements_loader.pyi ADDED Viewed

@@ -0,0 +1,33 @@
+from gllm_docproc.loader.base_loader import BaseLoader as BaseLoader
+from gllm_docproc.loader.loader_utils import validate_file_extension as validate_file_extension
+from gllm_docproc.model.element import Element as Element
+from typing import Any
+class JSONElementsLoader(BaseLoader):
+    """JSON Elements Loader class.
+    This class provides a loader for extracting information from JSON files.
+    The JSON file must be in the format of list of dictionaries. where each dictionary
+    must be following the structure of Element class.
+    Methods:
+        load(source, element_metadata, **kwargs): Load and process a document.
+    """
+    def load(self, source: str, loaded_elements: Any = None, **kwargs: Any) -> list[dict[str, Any]]:
+        """Load and process a document.
+        This method loads the JSON file and returns the list of elements. If file id is provided,
+        the file id will be added to the element metadata and the chunk id and chunk relation metadata
+        that contains file id as prefix will be updated.
+        Args:
+            source (str): The file path of the JSON file.
+            loaded_elements (Any, optional): The loaded elements. JSON Loader ignore this parameter.
+            **kwargs (Any): The keyword arguments.
+        Kwargs:
+            file_id (str, optional): The file id of for the elements. Defaults to None.
+        Returns:
+            list[dict[str, Any]]: The loaded elements.
+        """

gllm_docproc/loader/loader_utils.pyi ADDED Viewed

@@ -0,0 +1,42 @@
+from gllm_docproc.loader.exception import UnsupportedFileExtensionError as UnsupportedFileExtensionError
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata
+def validate_file_extension(expected_extension: str, loader_name: str):
+    """Decorator to validate the file extension of the input file.
+    Args:
+        expected_extension (str): The expected file extension.
+        loader_name (str): The name of the loader.
+    Returns:
+        Callable[[Callable[..., Any]], Callable[..., Any]]: A decorator that wraps the original
+            function to validate the file extension.
+    Raises:
+        UnsupportedFileExtensionError: If the file extension does not match the expected extension.
+    """
+def create_base_element_metadata(source: str, source_type: str) -> ElementMetadata:
+    """Create the base element metadata.
+    This function creates the base element metadata for the loaded element. Base element metadata
+    includes the source, source type, and loaded datetime.
+    Args:
+        source (str): The source of the element.
+        source_type (str): The source type.
+    Returns:
+        ElementMetadata: The base element metadata.
+    """
+def trim_table_empty_cells(table: list[list[str]]) -> list[list[str]]:
+    """Trim the empty cells in the table.
+    This function trims the empty cells in the table by removing the empty cells at the end of each
+    row. The function also ensures that all rows have the same number of columns.
+    Args:
+        table (List[List[str]]): A list of lists containing the table content.
+    Returns:
+        List[List[str]]: A list of lists containing the trimmed table content.
+    """

gllm_docproc/loader/pdf/__init__.pyi ADDED Viewed

@@ -0,0 +1,13 @@
+from .adobe_pdf_extract_loader import AdobePDFExtractLoader as AdobePDFExtractLoader
+from .azure_ai_document_intelligence_loader import AzureAIDocumentIntelligenceLoader as AzureAIDocumentIntelligenceLoader
+from .azure_ai_document_intelligence_raw_loader import AzureAIDocumentIntelligenceRawLoader as AzureAIDocumentIntelligenceRawLoader
+from .glair_vision_ocr_loader import GLAIRVisionOCRLoader as GLAIRVisionOCRLoader
+from .pdf_miner_loader import PDFMinerLoader as PDFMinerLoader
+from .pdf_miner_word_loader import PDFMinerWordLoader as PDFMinerWordLoader
+from .pdf_plumber_loader import PDFPlumberLoader as PDFPlumberLoader
+from .pymupdf_loader import PyMuPDFLoader as PyMuPDFLoader
+from .pymupdf_span_loader import PyMuPDFSpanLoader as PyMuPDFSpanLoader
+from .tabula_loader import TabulaLoader as TabulaLoader
+from .text_inject_pdf_plumber_loader import TextInjectPDFPlumberLoader as TextInjectPDFPlumberLoader
+__all__ = ['AdobePDFExtractLoader', 'AzureAIDocumentIntelligenceLoader', 'AzureAIDocumentIntelligenceRawLoader', 'GLAIRVisionOCRLoader', 'PDFMinerLoader', 'PDFMinerWordLoader', 'PDFPlumberLoader', 'PyMuPDFLoader', 'PyMuPDFSpanLoader', 'TabulaLoader', 'TextInjectPDFPlumberLoader']

gllm_docproc/loader/pdf/adobe_pdf_extract_loader.pyi ADDED Viewed

@@ -0,0 +1,40 @@
+from _typeshed import Incomplete
+from adobe.pdfservices.operation.io.cloud_asset import CloudAsset as CloudAsset
+from adobe.pdfservices.operation.io.stream_asset import StreamAsset as StreamAsset
+from adobe.pdfservices.operation.pdf_services_response import PDFServicesResponse as PDFServicesResponse
+from gllm_docproc.loader.base_loader import BaseLoader as BaseLoader
+from gllm_docproc.loader.loader_utils import validate_file_extension as validate_file_extension
+from typing import Any
+class AdobePDFExtractLoader(BaseLoader):
+    """Adobe PDF Extract Loader class.
+    This class provides a loader for extracting information from PDF files using Adobe PDF Extract.
+    It implements the 'load' method to load PDF files and extract information.
+    Methods:
+        load(source, loaded_elements, **kwargs): Loads a PDF file and extracts information.
+    """
+    credentials: Incomplete
+    def __init__(self, client_id: str, client_secret: str) -> None:
+        """Initializes the Adobe PDF Extract Loader.
+        Args:
+            client_id (str): The client ID for the Adobe PDF Extract API.
+            client_secret (str): The client secret for the Adobe PDF Extract
+        """
+    def load(self, source: str, loaded_elements: Any = None, **kwargs: Any) -> dict[str, Any]:
+        """Loads a PDF file and extracts information using Adobe PDF Extract.
+        This method loads a PDF file and extracts information from the file using Adobe PDF Extract.
+        The extracted information is returned as a dictionary. The extracted information includes text,
+        tables, and other elements from the PDF file.
+        Args:
+            source (str): The source PDF file to load and extract information from.
+            loaded_elements (Any): A list of loaded elements to be processed.
+            **kwargs (Any): Additional keyword arguments.
+        Returns:
+            dict[str, Any]: The extracted information as a dictionary
+        """

gllm_docproc/loader/pdf/azure_ai_document_intelligence_loader.pyi ADDED Viewed

@@ -0,0 +1,45 @@
+from _typeshed import Incomplete
+from gllm_docproc.loader.base_loader import BaseLoader as BaseLoader
+from gllm_docproc.loader.loader_utils import create_base_element_metadata as create_base_element_metadata, validate_file_extension as validate_file_extension
+from gllm_docproc.loader.pdf.azure_ai_document_intelligence_raw_loader import AzureAIDocumentIntelligenceRawLoader as AzureAIDocumentIntelligenceRawLoader
+from gllm_docproc.loader.pdf.pdf_loader_utils import merge_loaded_elements_by_coordinates as merge_loaded_elements_by_coordinates
+from gllm_docproc.model.element import Element as Element, TABLE as TABLE, UNCATEGORIZED_TEXT as UNCATEGORIZED_TEXT
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata, PDF as PDF
+from typing import Any
+class AzureAIDocumentIntelligenceLoader(BaseLoader):
+    """Azure AI Document Intelligence Loader class.
+    This class provides a loader for extracting text, tables, and images from PDF files
+    using the Azure AI Document Intelligence API. It implements the 'load' method to handle document
+    loading from a given source.
+    Methods:
+        load(source, loaded_elements, **kwargs): Load and process a document.
+    """
+    endpoint: Incomplete
+    key: Incomplete
+    logger: Incomplete
+    def __init__(self, endpoint: str, key: str) -> None:
+        """Initializes the Azure AI Document Intelligence Loader class.
+        Args:
+            endpoint (str): The endpoint for the Azure AI Document Intelligence API.
+            key (str): The key for the Azure AI Document Intelligence API.
+        """
+    def load(self, source: str, loaded_elements: list[dict[str, Any]] | None = None, **kwargs: Any) -> list[dict[str, Any]]:
+        """Load and process a document using the Azure AI Document Intelligence API.
+        Args:
+            source (str): The source of the document to be processed.
+            loaded_elements (list[dict[str, Any]], optional): A list of dictionaries containing loaded content and
+                metadata. Defaults to None.
+            **kwargs (Any): Additional keyword arguments for the loader.
+        Kwargs:
+            raw_output (dict[str, Any], optional): The raw output from the Azure AI Document Intelligence Raw Loader.
+            original_source (str, optional): The original source of the document.
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries containing loaded content and metadata.
+        """