gllm-docproc-binary 0.7.21__cp311-cp311-macosx_13_0_arm64.whl

gllm_docproc/downloader/html/html_downloader.pyi

@@ -0,0 +1,114 @@
+from _typeshed import Incomplete
+from gllm_docproc.downloader.base_downloader import BaseDownloader as BaseDownloader
+from gllm_docproc.downloader.html.exception import ItemScrapeFailedException as ItemScrapeFailedException
+from gllm_docproc.downloader.html.scraper.scraper.spiders import CrawlBaseSpider as CrawlBaseSpider, CrawlSitemapLinkSpider as CrawlSitemapLinkSpider, CrawlSitemapSpider as CrawlSitemapSpider
+from gllm_docproc.downloader.html.scraper.web_scraper_executor import WebScraperExecutor as WebScraperExecutor
+from gllm_docproc.downloader.html.utils import generate_filename_from_url as generate_filename_from_url, is_valid_url as is_valid_url
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata, HTML as HTML
+from gllm_docproc.utils.file_utils import create_full_path as create_full_path, save_file as save_file, save_to_json as save_to_json
+from scrapy import Spider as Spider
+from typing import Any
+
+class HTMLDownloader(BaseDownloader):
+    """A downloader class for downloading web content.
+
+    This class inherits from the BaseDownloader class and provides methods to download web content.
+
+    Args:
+        **kwargs (Any): Additional keyword arguments.
+    """
+    URL_INDEX: int
+    CONTENT_INDEX: int
+    kwargs: Incomplete
+    logger: Incomplete
+    def __init__(self, **kwargs: Any) -> None:
+        """Initializes the WebDownloader class.
+
+        Args:
+            **kwargs (Any): Additional keyword arguments.
+        """
+    def download(self, source: str, output: str, **kwargs: Any) -> list[str]:
+        """Downloads and saves web content from a single URL.
+
+        This method uses a web scraper to fetch the HTML content from the specified URL
+        and saves it as a JSON file in the given output directory.
+
+        Args:
+            source (str): The URL of the web content to download.
+            output (str): The directory path where the downloaded content (in JSON format) will be saved.
+            **kwargs (Any): Additional keyword arguments.
+
+        Kwargs:
+            include_markdown (bool, optional): Whether to include markdown conversion in the output. Defaults to False.
+
+        Returns:
+            list[str]: The list of full filepath of the created JSON files.
+
+        Raises:
+            ItemScrapeFailedException: If the provided source is not a valid URL, or if the web content
+                cannot be successfully scraped or downloaded from the URL.
+        """
+    def download_from_multiple_urls(self, urls: list[str], output: str = '.', **kwargs: Any) -> list[str]:
+        """Downloads web content from multiple URLs.
+
+        Args:
+            urls (list[str]): The URLs to download.
+            output (str): The output where we put the downloaded content (usually a folder path).
+            **kwargs (Any): Additional keyword arguments.
+
+        Kwargs:
+            include_markdown (bool, optional): Whether to include markdown conversion in the output. Defaults to False.
+
+        Returns:
+            list[str]: The list of full filepath of the created JSON files.
+        """
+    def download_crawl(self, urls: list[str] | str, output: str = '.', spider_type: type[Spider] | None = None, **kwargs: Any) -> list[str]:
+        """Downloads web content from the provided URLs.
+
+        This method uses a web scraper to crawl the provided URLs and saves the downloaded content to a file.
+
+        Args:
+            output (str): The output where we put the downloaded content (usually a folder path).
+            urls (list[str] | str): The URLs to crawl. Can be a single URL (str) or a list of URLs (list[str]).
+            spider_type (type[Spider] | None): The type of spider to use for downloading.
+                Defaults to None, which will use CrawlBaseSpider.
+            **kwargs (Any): Additional keyword arguments.
+
+        Kwargs:
+            include_markdown (bool, optional): Whether to include markdown conversion in the output. Defaults to False.
+
+        Returns:
+            list[str]: The list of full filepath of the created JSON files.
+        """
+    def download_sitemap(self, urls: list[str] | str, output: str = '.', spider_type: type[Spider] | None = None, **kwargs: Any) -> list[str]:
+        """Downloads web content from the sitemap of the provided URLs.
+
+        This method uses a web scraper to scrape the sitemap of each URL and saves the downloaded content to a file.
+
+        Args:
+            urls (list[str] | str): The URLs to scrape. Can be a single URL (str) or a list of URLs (list[str]).
+            output (str): The output where we put the downloaded content (usually a folder path).
+            spider_type (type[Spider] | None): The type of spider to use for downloading.
+                Defaults to None, which will use CrawlSitemapSpider.
+            **kwargs (Any): Additional keyword arguments.
+
+        Kwargs:
+            include_markdown (bool, optional): Whether to include markdown conversion in the output. Defaults to False.
+
+        Returns:
+            list[str]: The list of full filepath of the created JSON files.
+        """
+    def download_sitemap_links(self, urls: list[str] | str, output: str = '.', spider_type: type[Spider] | None = None) -> None:
+        """Retrieves all links from the sitemap of the provided URLs.
+
+        This method uses a web scraper to scrape the sitemap of each URL and returns a list of all found links.
+
+        Args:
+            urls (list[str] | str): The URLs to scrape. Can be a single URL (str) or a list of URLs (list[str]).
+            output (str): The output where we put the downloaded content (usually a folder path).
+            spider_type (type[Spider] | None)): The type of spider to use for downloading.
+                Defaults to None, which will use CrawlSitemapLinkSpider.
+
+        Returns:
+            None
+        """

gllm_docproc/downloader/html/requests_downloader.pyi

@@ -0,0 +1,46 @@
+from _typeshed import Incomplete
+from gllm_docproc.downloader import BaseDownloader as BaseDownloader
+from gllm_docproc.downloader.html.utils import generate_filename_from_url as generate_filename_from_url
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata, HTML as HTML
+from gllm_docproc.utils.file_utils import save_to_json as save_to_json
+from typing import Any
+
+class RequestsDownloader(BaseDownloader):
+    """A class for downloading HTML content from web pages and returning JSON format.
+
+    This class downloads HTML content from a given URL and returns it in a specific
+    dictionary format containing ElementMetadata and the decoded HTML content.
+    """
+    max_retries: Incomplete
+    timeout: Incomplete
+    proxies: Incomplete
+    session: Incomplete
+    logger: Incomplete
+    def __init__(self, max_retries: int = 3, timeout: float | None = None, proxies: dict[str, str] | None = None) -> None:
+        '''Initialize the RequestsDownloader.
+
+        Args:
+            max_retries (int, optional): The maximum number of retries for failed downloads. Defaults to 3.
+            timeout (float | None, optional): The timeout for the download request in seconds. Defaults to None.
+            proxies (dict[str, str] | None, optional): Dictionary of proxy servers to use. Defaults to None.
+                Example: {"http": "http://proxy.example.com:8080", "https": "https://proxy.example.com:8080"}
+        '''
+    def download(self, source: str, output: str, **kwargs: Any) -> list[str]:
+        """Download HTML content from the source URL and save as JSON file.
+
+        Args:
+            source (str): The URL to download HTML content from.
+            output (str): The output directory where the downloaded JSON file will be saved.
+            **kwargs (Any): Additional keyword arguments.
+
+        kwargs:
+            timeout (float | None, optional): Override the timeout for this specific request.
+            max_retries (int, optional): Override the max_retries for this specific request.
+            proxies (dict[str, str] | None, optional): Override the proxies for this specific request.
+
+        Returns:
+            list[str]: A list containing the file path of the saved JSON file.
+
+        Raises:
+            requests.RequestException: If the download fails after all retries.
+        """

gllm_docproc/downloader/html/scraper/scraper/spiders/__init__.pyi

@@ -0,0 +1,9 @@
+from .crawl_pdf_spider import CrawlPDFSpider as CrawlPDFSpider
+from .crawl_sitemap_link_spider import CrawlSitemapLinkSpider as CrawlSitemapLinkSpider
+from .crawl_sitemap_spider import CrawlSitemapSpider as CrawlSitemapSpider
+from .crawl_spider import CrawlBaseSpider as CrawlBaseSpider
+from .playwright_scrape_spider import PlaywrightScrapeSpider as PlaywrightScrapeSpider
+from .scrape_spider import ScrapeSpider as ScrapeSpider
+from .zyte_scrape_spider import ZyteScrapeSpider as ZyteScrapeSpider
+
+__all__ = ['ScrapeSpider', 'PlaywrightScrapeSpider', 'ZyteScrapeSpider', 'CrawlBaseSpider', 'CrawlSitemapSpider', 'CrawlSitemapLinkSpider', 'CrawlPDFSpider']

gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_pdf_spider.pyi

@@ -0,0 +1,27 @@
+from .crawl_spider import CrawlBaseSpider as CrawlBaseSpider
+from _typeshed import Incomplete
+from collections.abc import Generator
+from gllm_docproc.downloader.html.utils import clean_url as clean_url
+from scrapy.http import Request, Response
+
+class CrawlPDFSpider(CrawlBaseSpider):
+    """Scrapy CrawlSpider to crawl websites and save responses as PDFs using Playwright.
+
+    Attributes:
+        name (str): The name of the spider - 'crawl_pdf_spider'.
+        allowed_domains (list): The allowed domains for spider to crawl.
+        start_urls (list): The starting URLs for the spider to initiate crawling.
+        custom_settings (dict): Custom settings for the spider, including item pipelines configuration.
+        rules (tuple): The rules to be followed during the crawling process.
+    """
+    name: str
+    def add_playwright(self, request: Request, _: Response):
+        """Adds playwright meta information to the request."""
+    async def start(self) -> Generator[Incomplete]:
+        """Start Request.
+
+        Initiates requests for the specified start URLs using Scrapy requests with additional
+        meta information for playwright usage.
+        """
+    async def parse_web(self, response: Response):
+        """Parses the response obtained from the website, distinguishing between HTML content and other file types."""

gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_sitemap_link_spider.pyi

@@ -0,0 +1,28 @@
+from .crawl_sitemap_spider import CrawlSitemapSpider as CrawlSitemapSpider
+from scrapy.crawler import Crawler
+from typing import Any
+
+class CrawlSitemapLinkSpider(CrawlSitemapSpider):
+    """A Scrapy spider designed to scrape links from the sitemaps.
+
+    This spider uses the CrawlSitemapSpider base class to follow the sitemap links provided in the
+    robots.txt file of the website. It parses each page and extracts the URLs of the pages. If
+
+    Attributes:
+        name (str): The name of the spider - 'crawl_sitemap_link_spider'.
+        custom_settings (dict): Custom settings for the spider, including the log level and log file.
+    """
+    name: str
+    custom_settings: dict[str, Any]
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+        """Creates a new instance of the spider.
+
+        Args:
+            crawler (scrapy.crawler.Crawler): The Scrapy crawler object.
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+
+        Returns:
+            CrawlSitemapLinkSpider: A new instance of the spider.
+        """

gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_sitemap_spider.pyi

@@ -0,0 +1,61 @@
+from _typeshed import Incomplete
+from scrapy.crawler import Crawler
+from scrapy.http import Response
+from scrapy.spiders import SitemapSpider
+from typing import Any
+
+class CrawlSitemapSpider(SitemapSpider):
+    """A Scrapy spider designed to scrape content from the sitemaps.
+
+    This spider uses the SitemapSpider base class to follow the sitemap links provided in the
+    robots.txt file of the website. It parses each page and extracts the URLs of the pages. If
+    an error occurs during parsing, it logs the error.
+
+    Attributes:
+        name (str): The name of the spider - 'crawl_sitemap_spider'.
+        sitemap_urls (list): The URLs of the sitemaps to start crawling from.
+        allowed_domains (list): The domains that this spider is allowed to crawl.
+        custom_settings (dict): Custom settings for the spider, including the log level and log file.
+    """
+    name: str
+    custom_settings: dict[str, Any]
+    sitemap_urls: Incomplete
+    allowed_domains: Incomplete
+    callback: Incomplete
+    removed_components: Incomplete
+    is_follow_page: Incomplete
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initializes the CrawlSitemapSpider instance.
+
+        The method initializes the CrawlSitemapSpider instance and sets the sitemap_urls, allowed_domains,
+        and sitemap_from_robots attributes based on the provided arguments.
+
+        Args:
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+        """
+    def parse(self, response: Response):
+        """Parse the response.
+
+        This method parses the response obtained from the website, extracts the URLs of the pages,
+        and follows the links to the next pages.
+
+        This method attempts to yield a dictionary containing the URL of the response. If an error occurs,
+        it yields the URL and an error message.
+        It also extracts the URLs of the next pages from the response and follows them.
+
+        Args:
+            response (scrapy.http.Response): The response object to parse.
+        """
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> CrawlSitemapSpider:
+        """Creates a new CrawlSitemapSpider instance and sets the custom settings.
+
+        Args:
+            crawler (scrapy.crawler.Crawler): The crawler object.
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+
+        Returns:
+            CrawlSitemapSpider: The CrawlSitemapSpider instance.
+        """

gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_spider.pyi

@@ -0,0 +1,66 @@
+from _typeshed import Incomplete
+from collections.abc import Generator
+from scrapy.crawler import Crawler
+from scrapy.http import Request, Response
+from scrapy.spiders import CrawlSpider
+from typing import Any
+
+class CrawlBaseSpider(CrawlSpider):
+    """A Scrapy CrawlSpider designed to crawl and extract content from website.
+
+    Attributes:
+        name (str): The name of the spider - 'crawl_spider'.
+        allowed_domains (list): The allowed domains for spider to crawl.
+        start_urls (list): The starting URLs for the spider to initiate crawling.
+        custom_settings (dict): Custom settings for the spider, including item pipelines configuration.
+        rules (tuple): The rules to be followed during the crawling process.
+    """
+    name: str
+    rules: Incomplete
+    custom_settings: dict[str, Any]
+    start_urls: Incomplete
+    allowed_domains: Incomplete
+    callback: Incomplete
+    removed_components: Incomplete
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initialize the CrawlBaseSpider."""
+    def add_playwright(self, request: Request, _: Response):
+        """Adds playwright meta information to the request."""
+    async def start(self) -> Generator[Incomplete]:
+        """Start Request.
+
+        Initiates requests for the specified start URLs using Scrapy requests with additional
+        meta information for playwright usage.
+        """
+    def parse_web(self, response: Response):
+        """Parses the response obtained from the website, distinguishing between HTML content and other file types.
+
+        Args:
+            response (Response): The response object containing the page content.
+
+        Returns:
+            None
+        """
+    def follow_selected_urls(self, response: Response):
+        """Follows selected URLs from the response.
+
+        Args:
+            response (Response): The response object containing the page content.
+
+        Returns:
+            None
+        """
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+        """Creates a new instance of the CrawlBaseSpider with custom settings.
+
+        Args:
+            crawler (Crawler): The crawler object.
+            *args (Any): The arguments to be passed to the spider.
+            **kwargs (Any): The keyword arguments to be passed to the spider.
+
+        Returns:
+            CrawlBaseSpider: The CrawlBaseSpider instance.
+        """
+    async def errback(self, failure: Any):
+        """Handles errors encountered during the crawling process and closes playwright pages."""

gllm_docproc/downloader/html/scraper/scraper/spiders/playwright_scrape_spider.pyi

@@ -0,0 +1,22 @@
+from .scrape_spider import ScrapeSpider as ScrapeSpider
+from _typeshed import Incomplete
+from collections.abc import Generator
+from typing import Any
+
+class PlaywrightScrapeSpider(ScrapeSpider):
+    """A Scrapy spider designed to scrape content from website using playwright to render Javascript loaded page.
+
+    Attributes:
+        name (str): The name of the spider - 'playwright_scrape_spider'.
+        allowed_domains (list): The allowed domains for spider to crawl.
+        start_urls (list): The starting URLs for the spider to initiate crawling
+        custom_settings (dict): Custom settings for the spider, including item pipelines configuration.
+    """
+    name: str
+    custom_settings: dict[str, Any]
+    async def start(self) -> Generator[Incomplete]:
+        """Start Request.
+
+        Initiates requests for the specified start URLs using Scrapy requests with additional
+        meta information for playwright usage.
+        """

gllm_docproc/downloader/html/scraper/scraper/spiders/scrape_spider.pyi

@@ -0,0 +1,57 @@
+import scrapy
+from _typeshed import Incomplete
+from scrapy.crawler import Crawler
+from scrapy.http import Response
+from typing import Any
+
+class ScrapeSpider(scrapy.Spider):
+    """A Scrapy spider designed to scrape content from website.
+
+    Attributes:
+        name (str): The name of the spider - 'scrape_spider'.
+        start_urls (list): The list of URLs to start the spider.
+        allowed_domains (list): The list of allowed domains for crawling.
+        extracted_html (str): The HTML content extracted during crawling.
+        custom_settings (dict): Custom settings for the spider, including the log level and log file.
+    """
+    name: str
+    custom_settings: dict[str, Any]
+    start_urls: Incomplete
+    allowed_domains: Incomplete
+    callback: Incomplete
+    removed_components: Incomplete
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initializes the ScrapeSpider.
+
+        Args:
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+        """
+    def parse(self, response: Response, **kwargs: Any) -> None:
+        """Parses the response obtained from the website, distinguishing between HTML content and other file types.
+
+        Args:
+            response (scrapy.http.Response): The response obtained from the website.
+            **kwargs (dict[str, Any]): Additional keyword arguments.
+        """
+    def get_content_type(self, response: Response) -> str:
+        """Gets the content type from the response headers.
+
+        Args:
+            response (scrapy.http.Response): The response object.
+
+        Returns:
+            str: The content type.
+        """
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+        """Creates a new ScrapeSpider instance from the crawler.
+
+        Args:
+            crawler (scrapy.crawler.Crawler): The crawler object.
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+
+        Returns:
+            ScrapeSpider: The ScrapeSpider instance.
+        """

gllm_docproc/downloader/html/scraper/scraper/spiders/zyte_scrape_spider.pyi

@@ -0,0 +1,51 @@
+from .scrape_spider import ScrapeSpider as ScrapeSpider
+from _typeshed import Incomplete
+from collections.abc import Generator
+from scrapy.crawler import Crawler
+from scrapy.http import Response
+from typing import Any
+
+class ZyteScrapeSpider(ScrapeSpider):
+    """A Scrapy spider designed to scrape content from a website using the Zyte API.
+
+    This spider is specifically tailored for scraping content from a website using the Zyte API to handle block and
+    render Javascript loaded page.
+
+    Attributes:
+        name (str): The name of the spider - 'zyte_scrape_spider'.
+        allowed_domains (list): The list of allowed domains for crawling.
+        start_urls (list): The starting URLs for the spider to initiate crawling
+        custom_settings (dict): Custom settings for the spider, including item pipelines configuration.
+    """
+    name: str
+    custom_settings: dict[str, Any]
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+        """Create an instance of the spider from a Scrapy crawler.
+
+        This method is a class method that is called by Scrapy to create an instance of the spider
+        based on the provided Scrapy crawler and additional arguments.
+
+        Args:
+            crawler: The Scrapy crawler object.
+            *args: Variable length argument list.
+            **kwargs: Variable length keyword argument list.
+
+        Returns:
+            WebLoaderSpider: An instance of the spider.
+        """
+    async def start(self) -> Generator[Incomplete]:
+        """Start Request.
+
+        Initiates requests for the specified start URLs using Scrapy requests with additional
+        meta information for zyte usage.
+
+        This method iterates over the start_urls list and creates a Scrapy Request for each URL.
+        The Request includes meta information to enable the browserHtml feature of the Zyte Automatic Extraction API.
+        """
+    def parse(self, response: Response) -> None:
+        """Parses the HTML response obtained from the website.
+
+        Args:
+            response (scrapy.http.Response): The response object to parse.
+        """

gllm_docproc/downloader/html/scraper/web_scraper_executor.pyi

@@ -0,0 +1,43 @@
+from _typeshed import Incomplete
+from gllm_docproc.downloader.html.exception import ZyteApiKeyNotProvidedException as ZyteApiKeyNotProvidedException
+from gllm_docproc.downloader.html.scraper.scraper.spiders import ScrapeSpider as ScrapeSpider, ZyteScrapeSpider as ZyteScrapeSpider
+from scrapy import Spider as Spider
+from typing import Any
+
+class WebScraperExecutor:
+    '''A utility class for initiating and running web scraping processes using Scrapy spiders.
+
+    This class supports multiple spider types such as PlaywrightScrapeSpider, ZyteScrapeSpider, CrawlBaseSpider,
+    and CrawlSitemapSpider. It utilizes multiprocessing to run the scraping process concurrently.
+
+    Methods:
+        __init__: Initializes the WebScraperExecutor instance.
+        get_html_strings: Initiates the Scrapy spider and starts the scraping process using multiprocessing.
+        get_spider_class: Gets the appropriate Scrapy spider class based on the provided spider type.
+        _crawler_results: Appends the provided item to the list of items.
+        _create_crawl_process: Creates and runs a Scrapy crawl process for a specific spider.
+        _is_connected_to_internet: Checks if the system is connected to the internet.
+
+    Raises:
+        ZyteApiKeyNotProvidedException: If the spider is "zyte" but the Zyte API key is not provided.
+    '''
+    results: Incomplete
+    items: dict[str, bytes | Exception]
+    kwargs: Incomplete
+    spider: Incomplete
+    def __init__(self, urls: list[str] | str, **kwargs: Any) -> None:
+        """Initializes the WebScraperExecutor instance.
+
+        Args:
+            urls (List[str] | str): The URLs to be scraped.
+            **kwargs (Dict[str, Any]): Additional keyword arguments.
+        """
+    def get_url_content_pairs(self) -> list[tuple[str, bytes | Exception]]:
+        '''Initiates the Scrapy spider and starts the scraping process using multiprocessing.
+
+        Returns:
+            List: A list of scraped url and html content.
+
+        Raises:
+            ZyteApiKeyNotProvidedException: If the spider is "zyte" but the Zyte API key is not provided.
+        '''

gllm_docproc/downloader/html/utils/__init__.pyi

@@ -0,0 +1,3 @@
+from .web_utils import clean_url as clean_url, generate_filename_from_url as generate_filename_from_url, is_valid_url as is_valid_url
+
+__all__ = ['is_valid_url', 'clean_url', 'generate_filename_from_url']

gllm_docproc/downloader/html/utils/web_utils.pyi

@@ -0,0 +1,39 @@
+def is_valid_url(url: str) -> bool:
+    """Checks if the provided URL is valid.
+
+    Args:
+        url (str): The URL to be validated.
+
+    Returns:
+        bool: True if the URL is valid; False otherwise.
+    """
+def clean_url(url: str) -> str:
+    '''Return a deterministic filename stem by sanitizing only disallowed characters.
+
+    Only these characters are replaced with underscores: < > : " / \\ | ? *
+
+    Example:
+        https://www.bca.co.id/promo-bca -> https_www.bca.co.id_promo-bca
+
+    Args:
+        url (str): The URL to clean.
+
+    Returns:
+        str: A sanitized filename stem with only the listed characters replaced.
+    '''
+def generate_filename_from_url(url: str, max_filename_len: int = 128) -> str:
+    """Generate a sanitized, unique, and length-safe filename stem from a URL.
+
+    The returned value is a filename stem (no extension). It is composed of a sanitized
+    version of the URL plus a uniqueness suffix consisting of a short random token.
+    The function also trims the base so that the final filename stem will not exceed
+    the specified maximum length.
+
+    Args:
+        url (str): The URL to derive the filename from.
+        max_filename_len (int, optional): Maximum total filename length for the stem.
+            Defaults to 128.
+
+    Returns:
+        str: A safe filename stem (no extension) that is unique and within length constraints.
+    """

gllm_docproc/dpo_router/__init__.pyi

@@ -0,0 +1,5 @@
+from .base_dpo_router import BaseDPORouter as BaseDPORouter
+from .loader_router import LoaderRouter as LoaderRouter
+from .parser_router import ParserRouter as ParserRouter
+
+__all__ = ['BaseDPORouter', 'LoaderRouter', 'ParserRouter']

gllm_docproc/dpo_router/base_dpo_router.pyi

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseDPORouter(ABC):
+    """Base class for routing in document processing."""
+    @abstractmethod
+    def route(self, *args: Any, **kwargs: Any) -> Any:
+        """Routes the input into different processing pipelines based on certain criteria.
+
+        Args:
+            *args (Any): Variable length argument list for routing parameters.
+            **kwargs (Any): Arbitrary keyword arguments for additional routing configuration.
+
+        Returns:
+            Any: The result of the routing process.
+        """

gllm_docproc/dpo_router/loader_router.pyi

@@ -0,0 +1,52 @@
+from _typeshed import Incomplete
+from gllm_docproc.dpo_router.base_dpo_router import BaseDPORouter as BaseDPORouter
+from gllm_docproc.loader.csv.pandas_loader import CSV_VARIANTS as CSV_VARIANTS
+from gllm_docproc.loader.image import ImageLoader as ImageLoader
+from gllm_docproc.loader.json.json_elements_loader import JSON as JSON
+from gllm_docproc.loader.txt import TXTLoader as TXTLoader
+from gllm_docproc.loader.video.video_loader_utils import is_supported_video_file as is_supported_video_file
+from gllm_docproc.model import Element as Element, LoaderType as LoaderType
+from gllm_docproc.model.element_metadata import DOCX as DOCX, HTML as HTML, PDF as PDF, PPTX as PPTX, XLSX as XLSX
+from typing import Any
+
+class LoaderRouter(BaseDPORouter):
+    """Loader Router class.
+
+    This router determines the appropriate loader type based on the input source.
+    Returns a dict with the loader type information.
+    """
+    logger: Incomplete
+    txt_loader: Incomplete
+    image_loader: Incomplete
+    def __init__(self) -> None:
+        """Initialize the LoaderRouter.
+
+        This method initializes the LoaderRouter.
+        """
+    def route(self, source: str, *args: Any, **kwargs: Any) -> dict[str, Any]:
+        """Route the input source to the appropriate loader type.
+
+        This method determines the appropriate loader type based on the input source.
+        It checks if the source is a file or a YouTube URL.
+        1. If it is a file, it checks the file extension or content to determine the loader type.
+        2. If it is a YouTube URL, it returns the audio loader type.
+        3. If it is not a file or a YouTube URL, it returns the uncategorized loader type.
+
+        Args:
+            source (str): The input source, either a file path or a YouTube URL.
+            *args (Any): Additional arguments.
+            **kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            dict[str, Any]: A dictionary containing the loader type information.
+                Example: {LoaderType.KEY: LoaderType.PDF_LOADER}
+        """
+    def is_html_from_json(self, source: str) -> bool:
+        """Check if the source file contains valid HTML metadata.
+
+        Args:
+            source (str): The file path to check.
+
+        Returns:
+            bool: True if the file is a valid HTML metadata file, False otherwise.
+        """