gllm-docproc-binary 0.1.8__cp312-cp312-manylinux_2_31_x86_64.whl

gllm_docproc/downloader/html/scraper/scraper/spiders/scrape_spider.pyi

@@ -0,0 +1,57 @@
+import scrapy
+from _typeshed import Incomplete
+from scrapy.crawler import Crawler as Crawler
+from scrapy.http import Response as Response
+from typing import Any
+
+class ScrapeSpider(scrapy.Spider):
+    """A Scrapy spider designed to scrape content from website.
+
+    Attributes:
+        name (str): The name of the spider - 'scrape_spider'.
+        start_urls (list): The list of URLs to start the spider.
+        allowed_domains (list): The list of allowed domains for crawling.
+        extracted_html (str): The HTML content extracted during crawling.
+        custom_settings (dict): Custom settings for the spider, including the log level and log file.
+    """
+    name: str
+    custom_settings: dict[str, Any]
+    start_urls: Incomplete
+    allowed_domains: Incomplete
+    callback: Incomplete
+    removed_components: Incomplete
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initializes the ScrapeSpider.
+
+        Args:
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+        """
+    def parse(self, response: Response, **kwargs: Any) -> None:
+        """Parses the response obtained from the website, distinguishing between HTML content and other file types.
+
+        Args:
+            response (scrapy.http.Response): The response obtained from the website.
+            **kwargs (dict[str, Any]): Additional keyword arguments.
+        """
+    def get_content_type(self, response: Response) -> str:
+        """Gets the content type from the response headers.
+
+        Args:
+            response (scrapy.http.Response): The response object.
+
+        Returns:
+            str: The content type.
+        """
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+        """Creates a new ScrapeSpider instance from the crawler.
+
+        Args:
+            crawler (scrapy.crawler.Crawler): The crawler object.
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+
+        Returns:
+            ScrapeSpider: The ScrapeSpider instance.
+        """

gllm_docproc/downloader/html/scraper/scraper/spiders/zyte_scrape_spider.pyi

@@ -0,0 +1,51 @@
+from .scrape_spider import ScrapeSpider as ScrapeSpider
+from _typeshed import Incomplete
+from collections.abc import Generator
+from scrapy.crawler import Crawler as Crawler
+from scrapy.http import Response as Response
+from typing import Any
+
+class ZyteScrapeSpider(ScrapeSpider):
+    """A Scrapy spider designed to scrape content from a website using the Zyte API.
+
+    This spider is specifically tailored for scraping content from a website using the Zyte API to handle block and
+    render Javascript loaded page.
+
+    Attributes:
+        name (str): The name of the spider - 'zyte_scrape_spider'.
+        allowed_domains (list): The list of allowed domains for crawling.
+        start_urls (list): The starting URLs for the spider to initiate crawling
+        custom_settings (dict): Custom settings for the spider, including item pipelines configuration.
+    """
+    name: str
+    custom_settings: dict[str, Any]
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+        """Create an instance of the spider from a Scrapy crawler.
+
+        This method is a class method that is called by Scrapy to create an instance of the spider
+        based on the provided Scrapy crawler and additional arguments.
+
+        Args:
+            crawler: The Scrapy crawler object.
+            *args: Variable length argument list.
+            **kwargs: Variable length keyword argument list.
+
+        Returns:
+            WebLoaderSpider: An instance of the spider.
+        """
+    def start_requests(self) -> Generator[Incomplete]:
+        """Start Request.
+
+        Initiates requests for the specified start URLs using Scrapy requests with additional
+        meta information for zyte usage.
+
+        This method iterates over the start_urls list and creates a Scrapy Request for each URL.
+        The Request includes meta information to enable the browserHtml feature of the Zyte Automatic Extraction API.
+        """
+    def parse(self, response: Response) -> None:
+        """Parses the HTML response obtained from the website.
+
+        Args:
+            response (scrapy.http.Response): The response object to parse.
+        """

gllm_docproc/downloader/html/scraper/web_scraper_executor.pyi

@@ -0,0 +1,45 @@
+from _typeshed import Incomplete
+from gllm_docproc.downloader.html.exception import ZyteApiKeyNotProvidedException as ZyteApiKeyNotProvidedException
+from gllm_docproc.downloader.html.scraper.scraper.spiders import ScrapeSpider as ScrapeSpider, ZyteScrapeSpider as ZyteScrapeSpider
+from scrapy import Spider as Spider
+from typing import Any
+
+class WebScraperExecutor:
+    '''A utility class for initiating and running web scraping processes using Scrapy spiders.
+
+    This class supports multiple spider types such as PlaywrightScrapeSpider, ZyteScrapeSpider, CrawlBaseSpider,
+    and CrawlSitemapSpider. It utilizes multiprocessing to run the scraping process concurrently.
+
+    Methods:
+        __is_connected_to_internet: Checks if the system is connected to the internet.
+        get_spider_class: Gets the appropriate Scrapy spider class based on the provided spider type.
+        __init__: Initializes the WebScraperExecutor instance.
+        _crawler_results: Appends the provided item to the list of items.
+        _create_crawl_process: Creates and runs a Scrapy crawl process for a specific spider.
+        get_html_strings: Initiates the Scrapy spider and starts the scraping process using multiprocessing.
+
+    Raises:
+        ZyteApiKeyNotProvidedException: If the spider is "zyte" but the Zyte API key is not provided.
+    '''
+    URL_INDEX: int
+    HMLT_: int
+    results: Incomplete
+    items: list[tuple[str, bytes | Exception]]
+    kwargs: Incomplete
+    spider: Incomplete
+    def __init__(self, urls: list[str] | str, **kwargs: Any) -> None:
+        """Initializes the WebScraperExecutor instance.
+
+        Args:
+            urls (List[str] | str): The URLs to be scraped.
+            **kwargs (Dict[str, Any]): Additional keyword arguments.
+        """
+    def get_url_content_pairs(self) -> list[tuple[str, bytes | Exception]]:
+        '''Initiates the Scrapy spider and starts the scraping process using multiprocessing.
+
+        Returns:
+            List: A list of scraped url and html content.
+
+        Raises:
+            ZyteApiKeyNotProvidedException: If the spider is "zyte" but the Zyte API key is not provided.
+        '''

gllm_docproc/downloader/html/utils/__init__.pyi

@@ -0,0 +1,3 @@
+from .web_utils import clean_url as clean_url, is_valid_url as is_valid_url
+
+__all__ = ['is_valid_url', 'clean_url']

gllm_docproc/downloader/html/utils/web_utils.pyi

@@ -0,0 +1,21 @@
+def is_valid_url(url: str) -> bool:
+    """Checks if the provided URL is valid.
+
+    Args:
+        url (str): The URL to be validated.
+
+    Returns:
+        bool: True if the URL is valid; False otherwise.
+    """
+def clean_url(url: str) -> str:
+    """Clean the URL so it's suitable for filename.
+
+    Sample input: https://www.bca.co.id/promo-bca
+    Sample output: https___www.bca.co.id_promo-bca
+
+    Args:
+    url (str): The URL to clean
+
+    Returns:
+    str: The cleaned URL
+    """

gllm_docproc/dpo_router/__init__.pyi

@@ -0,0 +1,3 @@
+from .base_dpo_router import BaseDPORouter as BaseDPORouter
+
+__all__ = ['BaseDPORouter']

gllm_docproc/dpo_router/base_dpo_router.pyi

@@ -0,0 +1,17 @@
+import abc
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseDPORouter(ABC, metaclass=abc.ABCMeta):
+    """Base class for routing in document processing."""
+    @abstractmethod
+    def route(self, *args: Any, **kwargs: Any) -> Any:
+        """Routes the input into different processing pipelines based on certain criteria.
+
+        Args:
+            *args (Any): Variable length argument list for routing parameters.
+            **kwargs (Any): Arbitrary keyword arguments for additional routing configuration.
+
+        Returns:
+            Any: The result of the routing process.
+        """

gllm_docproc/housekeeping/__init__.pyi

@@ -0,0 +1,3 @@
+from .base_housekeeping import BaseHouseKeeping as BaseHouseKeeping
+
+__all__ = ['BaseHouseKeeping']

gllm_docproc/housekeeping/base_housekeeping.pyi

@@ -0,0 +1,15 @@
+import abc
+from abc import ABC, abstractmethod
+
+class BaseHouseKeeping(ABC, metaclass=abc.ABCMeta):
+    """Base class for document converter."""
+    @abstractmethod
+    def housekeeping(self, folder_path: str) -> None:
+        """Placeholder method for performing housekeeping tasks on a specified folder.
+
+        Args:
+            folder_path (str): The path to the folder to perform housekeeping on.
+
+        Returns:
+            None
+        """

gllm_docproc/indexer/__init__.pyi

@@ -0,0 +1,3 @@
+from gllm_docproc.indexer.base_indexer import BaseIndexer as BaseIndexer
+
+__all__ = ['BaseIndexer']

gllm_docproc/indexer/base_indexer.pyi

@@ -0,0 +1,31 @@
+import abc
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseIndexer(ABC, metaclass=abc.ABCMeta):
+    """Base class for document converter."""
+    @abstractmethod
+    def index(self, elements: Any, **kwargs: Any) -> Any:
+        """Index data from a source file into Elasticsearch.
+
+        Args:
+            elements (Any): The information to be indexed. Ideally formatted as List[Dict] and
+                each Dict following the structure of model 'Element'.
+            **kwargs (Any): Additional keyword arguments for customization.
+
+        Returns:
+            Any: The response from the indexing process.
+        """
+    @abstractmethod
+    def delete(self, **kwargs: Any) -> Any:
+        """Delete document from a vector DB.
+
+        The arguments are not defined yet, it depends on the implementation.
+        Some vector database will require: db_url, index_name, document_id.
+
+        Args:
+            **kwargs (Any): Additional keyword arguments for customization.
+
+        Returns:
+            Any: The response from the deletion process.
+        """

gllm_docproc/indexer/graph/__init__.pyi

@@ -0,0 +1,3 @@
+from .llama_index_graph_rag_indexer import LlamaIndexGraphRAGIndexer as LlamaIndexGraphRAGIndexer
+
+__all__ = ['LlamaIndexGraphRAGIndexer']

gllm_docproc/indexer/knowledge_graph/__init__.pyi

@@ -0,0 +1,4 @@
+from gllm_docproc.indexer.knowledge_graph.base_kg_indexer import BaseKGIndexer as BaseKGIndexer
+from gllm_docproc.indexer.knowledge_graph.llama_index_kg_indexer import LlamaIndexKGIndexer as LlamaIndexKGIndexer
+
+__all__ = ['BaseKGIndexer', 'LlamaIndexKGIndexer']

gllm_docproc/loader/__init__.pyi

@@ -0,0 +1,4 @@
+from .base_loader import BaseLoader as BaseLoader
+from .pipeline_loader import PipelineLoader as PipelineLoader
+
+__all__ = ['BaseLoader', 'PipelineLoader']

gllm_docproc/loader/audio/__init__.pyi

@@ -0,0 +1,3 @@
+from .audio_loader import AudioLoader as AudioLoader
+
+__all__ = ['AudioLoader']

gllm_docproc/loader/base_loader.pyi

@@ -0,0 +1,31 @@
+import abc
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseLoader(ABC, metaclass=abc.ABCMeta):
+    """An abstract base class for document loaders.
+
+    This class defines the structure for loading and processing documents to retrieve
+    required values. Subclasses are expected to implement the 'load' method
+    to handle document loading from a given source.
+
+    Methods:
+        load(source, loaded_elements, **kwargs): Abstract method to load a document.
+    """
+    @abstractmethod
+    def load(self, source: str, loaded_elements: Any = None, **kwargs: Any) -> Any:
+        """Load and process a document.
+
+        This method is abstract and must be implemented in subclasses.
+        It defines the process of loading a document using its source.
+
+        Args:
+            source (str): Might be file path, URL, the content itself.
+            loaded_elements (Any): The loaded elements from previous loaders. ideally formatted as List[Dict].
+            **kwargs (Any): Additional keyword arguments for customization.
+
+        Returns:
+            Any: The loaded document, ideally formatted as List[Dict]. Each dictionary within
+                the list are recommended to follows the structure of model 'Element',
+                to ensure consistency and ease of use across Document Processing Orchestrator.
+        """

gllm_docproc/loader/docx/__init__.pyi

@@ -0,0 +1,5 @@
+from .docx2python_loader import DOCX2PythonLoader as DOCX2PythonLoader
+from .python_docx_loader import PythonDOCXLoader as PythonDOCXLoader
+from .python_docx_table_loader import PythonDOCXTableLoader as PythonDOCXTableLoader
+
+__all__ = ['PythonDOCXLoader', 'PythonDOCXTableLoader', 'DOCX2PythonLoader']

gllm_docproc/loader/docx/docx2python_loader.pyi

@@ -0,0 +1,46 @@
+from _typeshed import Incomplete
+from docx2python.docx_output import DocxContent as DocxContent
+from gllm_docproc.loader.base_loader import BaseLoader as BaseLoader
+from gllm_docproc.loader.loader_utils import create_base_element_metadata as create_base_element_metadata, trim_table_empty_cells as trim_table_empty_cells, validate_file_extension as validate_file_extension
+from gllm_docproc.model.element import Element as Element, FOOTER as FOOTER, HEADER as HEADER, IMAGE as IMAGE, TABLE as TABLE, UNCATEGORIZED_TEXT as UNCATEGORIZED_TEXT
+from gllm_docproc.model.element_metadata import DOCX as DOCX, ElementMetadata as ElementMetadata
+from typing import Any
+
+class DOCX2PythonLoader(BaseLoader):
+    """A class for loading and processing DOCX document using docx2python library.
+
+    This class defines the structure for loading and processing DOCX document to retrieve required values
+    (text, table, image, header, footer footnote, endnote). It implements the 'load' method to handle DOCX loading
+    from a given file path.
+
+    DOCX2PythonLoader is used to extract the text, table, image, header, footer, footnote, endnote
+    from the DOCX document.
+
+    Methods:
+        load(source, loaded_elements, **kwargs): Load a DOCX document.
+    """
+    duplicate_merged_cells: Incomplete
+    def __init__(self, duplicate_merged_cells: bool = True) -> None:
+        """Initialize the DOCX2PythonLoader.
+
+        Args:
+            duplicate_merged_cells (bool): A boolean value indicating whether to duplicate merged cells.
+        """
+    def load(self, source: str, loaded_elements: list[dict[str, str]] | None = None, **kwargs: Any) -> list[dict[str, str]]:
+        """Load and process a DOCX document specified by the file path and name (source).
+
+        This method defines the process of loading a DOCX document using its file path.
+        It extracts the text, table, image, header, footer, footnote, endnote from the DOCX document.
+
+        Args:
+            source (str): The file path of the DOCX document.
+            loaded_elements (list[dict[str, str]] | None): A list of loaded elements containing text content
+                and metadata.
+            **kwargs (Any): Additional keyword arguments for loading the DOCX document.
+
+        Kwargs:
+            original_source (str, optional): The original source of the document.
+
+        Returns:
+            list[dict[str, str]]: A list of dictionaries containing loaded content and metadata.
+        """

gllm_docproc/loader/docx/python_docx_loader.pyi

@@ -0,0 +1,35 @@
+from gllm_docproc.loader.base_loader import BaseLoader as BaseLoader
+from gllm_docproc.loader.loader_utils import create_base_element_metadata as create_base_element_metadata, trim_table_empty_cells as trim_table_empty_cells, validate_file_extension as validate_file_extension
+from gllm_docproc.model.element import Element as Element, FOOTER as FOOTER, HEADER as HEADER, TABLE as TABLE, UNCATEGORIZED_TEXT as UNCATEGORIZED_TEXT
+from gllm_docproc.model.element_metadata import DOCX as DOCX, ElementMetadata as ElementMetadata
+from typing import Any
+
+class PythonDOCXLoader(BaseLoader):
+    """A class for loading and processing DOCX document using PythonDOCXLoader.
+
+    This class defines the structure for loading and processing DOCX document to retrieve required values
+    (Header, Body (Text and Table), Footer). It implements the 'load' method to handle DOCX loading
+    from a given file path.
+
+    PythonDOCXLoader is used to extract the Header, Body (Text and Table), Footer and metadata from the DOCX document.
+
+    Methods:
+        load(source, loaded_elements, **kwargs): Load a DOCX document.
+    """
+    def load(self, source: str, loaded_elements: list[dict[str, Any]] | None = None, **kwargs: Any) -> list[dict[str, Any]]:
+        """Load and process a DOCX document specified by the file path and name (source).
+
+        This method defines the process of loading a DOCX document using its file path.
+        It uses PythonDOCX to extract element text (with text structure) and table from the DOCX document.
+
+        Args:
+            source (str): The path to the DOCX document file.
+            loaded_elements (list[dict[str, Any]]): A list of dictionaries containing loaded content and metadata.
+            **kwargs (Any): Additional keyword arguments for the loader.
+
+        Kwargs:
+            original_source (str, optional): The original source of the document.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries containing loaded content and metadata.
+        """

gllm_docproc/loader/docx/python_docx_table_loader.pyi

@@ -0,0 +1,35 @@
+from gllm_docproc.loader.base_loader import BaseLoader as BaseLoader
+from gllm_docproc.loader.docx.python_docx_loader import PythonDOCXLoader as PythonDOCXLoader
+from gllm_docproc.model.element import Element as Element, TABLE as TABLE
+from typing import Any
+
+class PythonDOCXTableLoader(BaseLoader):
+    """Python DOCX Table Loader class to load tables from DOCX document.
+
+    This class is used to load tables from DOCX document using python-docx library.
+    Then it combined the existing loaded elements with the loaded tables.
+
+    Methods:
+        load: Load the tables from the DOCX document and combine it with the existing loaded elements.
+        _filter_table_elements: Filter the table elements from the loaded elements.
+        _get_table_content_count: Get the table content count.
+        _is_table_match: Is the table match with the merged table.
+        _find_matching_merged_table: Find the matching merged table.
+    """
+    def load(self, source: str, loaded_elements: list[dict[str, Any]] | None = None, **kwargs: Any) -> list[dict[str, Any]]:
+        """Load the tables from the DOCX document and combine it with the existing loaded elements.
+
+        This function loads the tables from the DOCX document using python-docx library.
+        Then it combined the existing loaded elements with the loaded tables.
+
+        Args:
+            source (str): The source file path.
+            loaded_elements (list[dict[str, Any]] | None): The existing loaded elements.
+            **kwargs (Any): The keyword arguments.
+
+        Kwargs:
+            original_source (str, optional): The original source of the document.
+
+        Returns:
+            list[dict[str, Any]]: The loaded elements.
+        """

gllm_docproc/loader/exception/__init__.pyi

@@ -0,0 +1,3 @@
+from .unsupported_file_extension_error import UnsupportedFileExtensionError as UnsupportedFileExtensionError
+
+__all__ = ['UnsupportedFileExtensionError']

gllm_docproc/loader/exception/unsupported_file_extension_error.pyi

@@ -0,0 +1,7 @@
+from _typeshed import Incomplete
+
+class UnsupportedFileExtensionError(Exception):
+    """An exception for unsupported file extension."""
+    message: Incomplete
+    def __init__(self, ext: str, loader_name: str) -> None:
+        """Initialize the exception."""

gllm_docproc/loader/html/__init__.pyi

@@ -0,0 +1,5 @@
+from .flat import HTMLFlatLoader as HTMLFlatLoader
+from .html_base_loader import HTMLBaseLoader as HTMLBaseLoader
+from .nested import HTMLNestedLoader as HTMLNestedLoader
+
+__all__ = ['HTMLBaseLoader', 'HTMLFlatLoader', 'HTMLNestedLoader']

gllm_docproc/loader/html/exception/__init__.pyi

@@ -0,0 +1,3 @@
+from .html_load_exception import HtmlLoadException as HtmlLoadException
+
+__all__ = ['HtmlLoadException']

gllm_docproc/loader/html/exception/html_load_exception.pyi

@@ -0,0 +1,7 @@
+from _typeshed import Incomplete
+
+class HtmlLoadException(Exception):
+    """Custom exception for handling HtmlLoadException errors."""
+    message: Incomplete
+    def __init__(self, message: str) -> None:
+        """Initialize the HtmlLoadException."""

gllm_docproc/loader/html/flat/__init__.pyi

@@ -0,0 +1,3 @@
+from .html_flat_loader import HTMLFlatLoader as HTMLFlatLoader
+
+__all__ = ['HTMLFlatLoader']

gllm_docproc/loader/html/flat/html_flat_base_handler.pyi

@@ -0,0 +1,52 @@
+from gllm_docproc.loader.html.utils.removed_components import RemovedComponents as RemovedComponents
+from gllm_docproc.loader.html.utils.string_utils import StringUtils as StringUtils
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, HTMLTags as HTMLTags
+from parsel import Selector as Selector, SelectorList as SelectorList
+from typing import Callable
+
+def is_base_element(content_selector: Selector | SelectorList[Selector] | None, removed_components: RemovedComponents) -> bool:
+    """Check if the given content selector represents a base element.
+
+    A base element is determined by the type of element in the HTML document.
+    Supported base elements include:
+    1. Unsupported result (if content_selector is None)
+    2. String text
+    3. Removed Components (by class or tag) defined in RemovedComponents
+    4. <input>
+    5. <svg> image
+    6. <img>
+    7. <audio>, <video> (if multiple sources are given, select only the first one)
+    8. <iframe> (cannot get the content of the iframe)
+    9. <embed> (cannot get the content of the embed)
+    10. <br>
+
+    Args:
+        content_selector (Selector | SelectorList[Selector] | None): The selector representing the HTML content.
+        removed_components (RemovedComponents): Components to be removed from processing.
+
+    Returns:
+        bool: True if the content_selector represents a base element; False otherwise.
+    """
+def handle_base_element(content_selector: Selector | SelectorList[Selector] | None, html_head: ElementMetadata, removed_components: RemovedComponents) -> list[Element]:
+    """Handle the base HTML element and generate Element instances.
+
+    Args:
+        content_selector (Selector | SelectorList[Selector] | None): The selector representing the HTML content.
+        html_head (ElementMetadata): The metadata extracted from the HTML head.
+        removed_components (RemovedComponents): Components to be removed from processing.
+
+    Returns:
+        list[Element]: A list of Element instances generated from the HTML content.
+    """
+def get_handler(tag: str) -> Callable[[Selector | SelectorList[Selector], ElementMetadata], list[Element]] | None:
+    """Get the handler function corresponding to the given HTML tag.
+
+    Args:
+        tag (str): The HTML tag for which the handler function is requested.
+
+    Returns:
+        Callable[[Selector | SelectorList[Selector], ElementMetadata], list[Element]] | None: The handler
+        function corresponding to the given HTML tag.
+    """

gllm_docproc/loader/html/flat/html_flat_loader.pyi

@@ -0,0 +1,30 @@
+from gllm_docproc.loader.html.exception import HtmlLoadException as HtmlLoadException
+from gllm_docproc.loader.html.flat.html_flat_base_handler import handle_base_element as handle_base_element, is_base_element as is_base_element
+from gllm_docproc.loader.html.flat.html_flat_merger import merge_html_elements as merge_html_elements
+from gllm_docproc.loader.html.html_base_loader import HTMLBaseLoader as HTMLBaseLoader
+from gllm_docproc.loader.html.utils.html_utils import extract_html_head as extract_html_head, extract_html_title_tag as extract_html_title_tag
+from gllm_docproc.loader.html.utils.removed_components import RemovedComponents as RemovedComponents
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata
+from parsel import Selector as Selector, SelectorList as SelectorList
+
+class HTMLFlatLoader(HTMLBaseLoader):
+    """A loader class for loading web content and extracting information.
+
+    This class inherits from the BaseLoader class and provides methods to load web content,
+    extract information, and scrape data using Scrapy spiders.
+    """
+    def __init__(self) -> None:
+        """Initialize the HTMLFlatLoader."""
+    @classmethod
+    def extract_html_element(cls, content_selector: SelectorList[Selector] | Selector, html_head: ElementMetadata, removed_components: RemovedComponents) -> list[Element]:
+        """Recursively extract the content of an HTML element.
+
+        Args:
+            content_selector (SelectorList[Selector] | Selector): The content selector.
+            html_head (ElementMetadata): The HTML head metadata.
+            removed_components (RemovedComponents): The removed components.
+
+        Returns:
+                list[Element]: A list of web elements.
+        """

gllm_docproc/loader/html/flat/html_flat_merger.pyi

@@ -0,0 +1,22 @@
+from gllm_docproc.loader.html.utils.flat_table_utils import FlatTableUtils as FlatTableUtils
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, HTMLTags as HTMLTags, ItemDataKeys as ItemDataKeys
+from parsel import Selector as Selector, SelectorList as SelectorList
+
+def merge_html_elements(content_selector: Selector | SelectorList[Selector], contents: list[Element], html_head: ElementMetadata) -> list[Element]:
+    """For non-base element, add metadata and merge children into one with parent element.
+
+    1. Add its HTML tag into its metadata
+    2. For some HTML tags, combine its children into a single element into the parent, for example:
+        1. Combine <ul> / <ol> children into a single element
+        2. Combine <a> children to become [text](https://link.com)
+
+    Args:
+        content_selector(Selector | SelectorList[Selector]): The content selector representing the HTML element.
+        contents (list[Element]): list of Element instances representing the contents of the HTML element.
+        html_head (ElementMetadata): The metadata extracted from the HTML head.
+
+    Returns:
+        list[Element]: list of Element instances after handling the contents based on the parent tag.
+    """

gllm_docproc/loader/html/html_base_loader.pyi

@@ -0,0 +1,25 @@
+from gllm_docproc.loader.base_loader import BaseLoader as BaseLoader
+from gllm_docproc.loader.html.utils.html_utils import is_html_content as is_html_content
+from typing import Any
+
+class HTMLBaseLoader(BaseLoader):
+    """A loader class for loading web content and extracting information.
+
+    This class inherits from the BaseLoader class and provides methods to load web content,
+    extract information, and scrape data using Scrapy spiders.
+    """
+    URL_INDEX: int
+    CONTENT_INDEX: int
+    def __init__(self, load_from_html_string: Any) -> None:
+        """Initialize the HTMLBaseLoader."""
+    def load(self, source: str, loaded_elements: list[dict[str, Any]] | None = None, **kwargs: Any) -> list[dict[str, Any]]:
+        """Loads web content and returns the extracted information in JSON format.
+
+        Args:
+            source (str): The source of the web content, either a URL or a file path.
+            loaded_elements (list[dict]): A list of loaded elements to be processed.
+            **kwargs (dict[str, Any]): Additional keyword arguments.
+
+        Returns:
+            list[dict]: The extracted information in JSON format.
+        """

gllm_docproc/loader/html/nested/__init__.pyi

@@ -0,0 +1,3 @@
+from .html_nested_loader import HTMLNestedLoader as HTMLNestedLoader
+
+__all__ = ['HTMLNestedLoader']