gllm-docproc-binary 0.1.8__cp311-cp311-macosx_14_0_arm64.whl

gllm_docproc/chunker/__init__.pyi

@@ -0,0 +1,3 @@
+from .base_chunker import BaseChunker as BaseChunker
+
+__all__ = ['BaseChunker']

gllm_docproc/chunker/base_chunker.pyi

@@ -0,0 +1,29 @@
+import abc
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseChunker(ABC, metaclass=abc.ABCMeta):
+    """An abstract base class for chunker.
+
+    This class segmenting or chunking elements based on contextual information.
+    Subclasses are expected to implement the 'chunk' method to handle chunking elements.
+
+    Methods:
+        chunk(elements, **kwargs): Abstract method to chunk a document.
+    """
+    @abstractmethod
+    def chunk(self, elements: Any, **kwargs: Any) -> Any:
+        """Chunk a document.
+
+        This method is abstract and must be implemented in subclasses.
+        It defines the process of chunking information from elements.
+
+        Args:
+            elements (Any): The information to be chunked. ideally formatted as List[Dict].
+            **kwargs (Any): Additional keyword arguments for customization.
+
+        Returns:
+            Any: The chunked information, ideally formatted as List[Dict]. Each dictionary within
+                the list are recommended to follows the structure of model 'Element',
+                to ensure consistency and ease of use across Document Processing Orchestrator.
+        """

gllm_docproc/chunker/structured_element/__init__.pyi

@@ -0,0 +1,3 @@
+from .structured_element_chunker import StructuredElementChunker as StructuredElementChunker
+
+__all__ = ['StructuredElementChunker']

gllm_docproc/chunker/structured_element/chunk_enricher.pyi

@@ -0,0 +1,43 @@
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.model.element_metadata import AUDIO as AUDIO, PDF as PDF
+
+def enrich_chunk(chunk: Element, elements: list[Element]) -> Element:
+    """Enrich the chunk with information from the original elements.
+
+    This is the default enrichment function for structured element chunker.
+    The function enrich the chunk with information from the original elements.
+    Based on the source type, the information that we want to keep are different.
+
+    Args:
+        chunk (Element): The chunk to be enriched.
+        elements (list[Element]): The original elements that form the chunk.
+
+    Returns:
+        Element: The enriched chunk.
+    """
+def enrich_pdf_chunk(chunk: Element, elements: list[Element]) -> Element:
+    """The default function for enriching the PDF chunk.
+
+    The function enriches the PDF chunk with the coordinates and page_number information
+    of the original elements.
+
+    Args:
+        chunk (Element): The PDF chunk to be enriched.
+        elements (list[Element]): The original elements that form the chunk.
+
+    Returns:
+        Element: The enriched PDF chunk.
+    """
+def enrich_audio_chunk(chunk: Element, elements: list[Element]) -> Element:
+    """The default function for enriching the audio chunk.
+
+    The function enriches the audio chunk by replacing the double newlines with a single newline.
+    Then, it adds the start_time, end_time, and lang_id information of the original elements.
+
+    Args:
+        chunk (Element): The audio chunk to be enriched.
+        elements (list[Element]): The original elements that form the chunk.
+
+    Returns:
+        Element: The enriched audio chunk.
+    """

gllm_docproc/chunker/structured_element/structured_element_chunker.pyi

@@ -0,0 +1,80 @@
+from _typeshed import Incomplete
+from gllm_docproc.chunker.base_chunker import BaseChunker as BaseChunker
+from gllm_docproc.chunker.structured_element.chunk_enricher import enrich_chunk as enrich_chunk
+from gllm_docproc.chunker.table import TableChunker as TableChunker
+from gllm_docproc.model.element import AUDIO as AUDIO, Element as Element, FOOTER as FOOTER, FOOTNOTE as FOOTNOTE, HEADER as HEADER, HEADING as HEADING, IMAGE as IMAGE, TABLE as TABLE, TITLE as TITLE, UNCATEGORIZED_TEXT as UNCATEGORIZED_TEXT, VIDEO as VIDEO
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from typing import Any
+
+NON_TEXT_STRUCTURE: Incomplete
+
+def default_text_splitter() -> RecursiveCharacterTextSplitter:
+    '''Define the default text splitter for structured text chunking.
+
+    This function defines the default text splitter for structured text chunking.
+    The text splitter is defined with the following separators:
+
+        1. "\\n#" : Split by Title or Heading
+        2. "\\n\\n" : Split between Paragraph Elements
+        3. "\\n" : Split between Title/Heading and Paragraph Elements
+        4. ". " | "! " | "? " : Split by Sentence
+        5. ", " : Split by Word
+        6. " " : Split by Word
+        7. "" : Split by Character
+
+    Returns:
+        RecursiveCharacterTextSplitter: A RecursiveCharacterTextSplitter object for structured text chunking.
+    '''
+
+class StructuredElementChunker(BaseChunker):
+    """A class for structured text chunker.
+
+    This class defines the structure for chunking structured text into smaller chunks. It implements
+    the 'chunk' method to handle structured text chunking.
+
+    Methods:
+        chunk(elements, **kwargs): Chunk the structured text into smaller chunks.
+    """
+    default_text_splitter: Incomplete
+    default_table_chunker: Incomplete
+    text_splitter: Incomplete
+    table_chunker: Incomplete
+    is_parent_structure_info_included: Incomplete
+    def __init__(self, text_splitter: RecursiveCharacterTextSplitter = ..., table_chunker: BaseChunker = ..., is_parent_structure_info_included: bool = True) -> None:
+        """Initialize the structured text chunker.
+
+        Args:
+            text_splitter (RecursiveCharacterTextSplitter): A RecursiveCharacterTextSplitter object
+                for structured text chunking.
+            table_chunker (BaseChunker): A BaseChunker object for table chunking.
+            is_parent_structure_info_included (bool): A boolean value to include parent structure
+                information in the chunk.
+        """
+    def chunk(self, elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        '''Chunk the structured text into smaller chunks.
+
+        This method defines the process of chunking structured text into smaller chunks. It uses the
+        RecursiveCharacterTextSplitter to split the text into chunks based on the defined separators.
+
+        The method will split the text recursively based on the defined separators, or by default:
+            1. "\\n#" : Split by Title or Heading
+            2. "\\n\\n" : Split between Paragraph Elements
+            3. "\\n" : Split between Title/Heading and Paragraph Elements
+            4. ". " | "! " | "? " : Split by Sentence
+            5. ", " : Split by Word
+            6. " " : Split by Word
+            7. "" : Split by Character
+
+        Kwargs:
+            excluded_structures (list[str]): A list of structures to be excluded from the chunking process.
+            enrich_chunk (Callable[[Element, list[Element]], Element]): A function to enrich the chunked element.
+            file_id (str | None): The file id of the chunked elements. Defaults to None.
+
+        Args:
+            elements (list[dict[str, any]]): A list of dictionaries containing text and structure.
+            **kwargs (Any): Additional keyword arguments for the chunker.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries containing chunked text and metadata.
+        '''

gllm_docproc/chunker/table/__init__.pyi

@@ -0,0 +1,3 @@
+from .table_chunker import CSV as CSV, HTML as HTML, MARKDOWN as MARKDOWN, TableChunker as TableChunker
+
+__all__ = ['CSV', 'HTML', 'MARKDOWN', 'TableChunker']

gllm_docproc/chunker/table/table_chunker.pyi

@@ -0,0 +1,45 @@
+from _typeshed import Incomplete
+from gllm_docproc.chunker.base_chunker import BaseChunker as BaseChunker
+from gllm_docproc.model.element import Element as Element, TABLE as TABLE
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata
+from typing import Any
+
+MARKDOWN: str
+CSV: str
+HTML: str
+
+class TableChunker(BaseChunker):
+    """Table Chunker class.
+
+    This class is used to chunk a table element into smaller chunks. It implements the 'chunk' method
+    to handle chunking the table element based on the chunk size and overlap. The table is converted
+    into the expected format (markdown, csv, or html).
+
+    Methods:
+        chunk(elements, **kwargs): Chunk a table element into smaller chunks.
+    """
+    chunk_size: Incomplete
+    chunk_overlap: Incomplete
+    table_format: Incomplete
+    table_splitter: Incomplete
+    def __init__(self, chunk_size: int = 4000, chunk_overlap: int = 0, table_format: str = ...) -> None:
+        """Initializes the TableChunker class.
+
+        Args:
+            chunk_size (int): The size of each chunk.
+            chunk_overlap (int): The overlap between each chunk.
+            table_format (str): The format of the table (markdown, csv, or html).
+        """
+    def chunk(self, elements: list[dict[str, Any]], **kwargs: Any) -> list[dict[str, Any]]:
+        """Chunk a table element into smaller chunks.
+
+        This method chunks a table element into smaller chunks based on the chunk size and overlap.
+        It converts the table into the expected format (markdown, csv, or html) and then chunks the table.
+
+        Args:
+            elements (list[dict[str, Any]]): The table element to be chunked.
+            **kwargs (Any): Additional keyword arguments for customization.
+
+        Returns:
+            list[dict[str, Any]]: The list of smaller chunks.
+        """

gllm_docproc/converter/__init__.pyi

@@ -0,0 +1,3 @@
+from .base_converter import BaseConverter as BaseConverter
+
+__all__ = ['BaseConverter']

gllm_docproc/converter/base_converter.pyi

@@ -0,0 +1,16 @@
+import abc
+from abc import ABC, abstractmethod
+
+class BaseConverter(ABC, metaclass=abc.ABCMeta):
+    """Base class for document converter."""
+    @abstractmethod
+    def convert(self, path_input: str, path_output: str) -> None:
+        """Converts a document.
+
+        Args:
+            path_input (str): The path of the document to be converted.
+            path_output (str): The path of the converted document.
+
+        Returns:
+            None
+        """

gllm_docproc/data_generator/__init__.pyi

@@ -0,0 +1,3 @@
+from gllm_docproc.data_generator.base_data_generator import BaseDataGenerator as BaseDataGenerator
+
+__all__ = ['BaseDataGenerator']

gllm_docproc/data_generator/base_data_generator.pyi

@@ -0,0 +1,19 @@
+import abc
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseDataGenerator(ABC, metaclass=abc.ABCMeta):
+    """Base class for data generator."""
+    @abstractmethod
+    def generate(self, elements: Any, **kwargs: Any) -> Any:
+        """Generates data for a list of chunks.
+
+        Args:
+            elements (Any): The elements to be used for generating data / metadata. ideally formatted as List[Dict].
+            **kwargs (Any): Additional keyword arguments for customization.
+
+        Returns:
+            Any: The generated data, ideally formatted as List[Dict]. Each dictionary within
+                the list are recommended to follows the structure of model 'Element',
+                to ensure consistency and ease of use across Document Processing Orchestrator.
+        """

gllm_docproc/downloader/__init__.pyi

@@ -0,0 +1,3 @@
+from .base_downloader import BaseDownloader as BaseDownloader
+
+__all__ = ['BaseDownloader']

gllm_docproc/downloader/base_downloader.pyi

@@ -0,0 +1,16 @@
+import abc
+from abc import ABC, abstractmethod
+
+class BaseDownloader(ABC, metaclass=abc.ABCMeta):
+    """Base class for document converter."""
+    @abstractmethod
+    def download(self, source: str, output: str) -> None:
+        """Converts a document.
+
+        Args:
+            source (str): The source of the document (could be JSON-formatted or URL)
+            output (str): The output where we put the downloaded content (usually a folder path).
+
+        Returns:
+            None
+        """

gllm_docproc/downloader/html/__init__.pyi

@@ -0,0 +1,4 @@
+from . import utils as utils
+from .html_downloader import HTMLDownloader as HTMLDownloader
+
+__all__ = ['HTMLDownloader', 'utils']

gllm_docproc/downloader/html/exception/__init__.pyi

@@ -0,0 +1,4 @@
+from .item_scrape_failed_exception import ItemScrapeFailedException as ItemScrapeFailedException
+from .zyte_api_key_not_provided_exception import ZyteApiKeyNotProvidedException as ZyteApiKeyNotProvidedException
+
+__all__ = ['ItemScrapeFailedException', 'ZyteApiKeyNotProvidedException']

gllm_docproc/downloader/html/exception/item_scrape_failed_exception.pyi

@@ -0,0 +1,16 @@
+from _typeshed import Incomplete
+
+class ItemScrapeFailedException(Exception):
+    """Exception raised when an item fails to be scraped.
+
+    Attributes:
+        message (str): Optional. The error message indicating the reason for the item scrape failure.
+    """
+    message: Incomplete
+    def __init__(self, message: str = 'Item failed to be scraped.') -> None:
+        '''Initialize the ItemScrapeFailedException.
+
+        Args:
+            message (str): Optional. The error message indicating the reason for the item scrape failure.
+                Defaults to "Item failed to be scraped."
+        '''

gllm_docproc/downloader/html/exception/zyte_api_key_not_provided_exception.pyi

@@ -0,0 +1,15 @@
+from _typeshed import Incomplete
+
+class ZyteApiKeyNotProvidedException(Exception):
+    """Custom exception raised when the Zyte API key is not provided.
+
+    Attributes:
+        message (str): Optional. The error message associated with the exception.
+    """
+    message: Incomplete
+    def __init__(self, message: str = 'Zyte API Key not provided.') -> None:
+        '''Initialize the ZyteApiKeyNotProvidedException.
+
+        Args:
+            message (str, optional): The error message associated with the exception.                 Defaults to "Zyte API Key not provided."
+        '''

gllm_docproc/downloader/html/html_downloader.pyi

@@ -0,0 +1,91 @@
+from _typeshed import Incomplete
+from gllm_docproc.downloader.base_downloader import BaseDownloader as BaseDownloader
+from gllm_docproc.downloader.html.exception import ItemScrapeFailedException as ItemScrapeFailedException
+from gllm_docproc.downloader.html.scraper.scraper.spiders import CrawlBaseSpider as CrawlBaseSpider, CrawlSitemapLinkSpider as CrawlSitemapLinkSpider, CrawlSitemapSpider as CrawlSitemapSpider
+from gllm_docproc.downloader.html.scraper.web_scraper_executor import WebScraperExecutor as WebScraperExecutor
+from gllm_docproc.downloader.html.utils import clean_url as clean_url, is_valid_url as is_valid_url
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata, HTML as HTML
+from gllm_docproc.utils.file_utils import create_full_path as create_full_path, save_file as save_file, save_to_json as save_to_json
+from scrapy import Spider as Spider
+from typing import Any
+
+class HTMLDownloader(BaseDownloader):
+    """A downloader class for downloading web content.
+
+    This class inherits from the BaseDownloader class and provides methods to download web content.
+
+    Args:
+        **kwargs (Any): Additional keyword arguments.
+    """
+    URL_INDEX: int
+    CONTENT_INDEX: int
+    kwargs: Incomplete
+    def __init__(self, **kwargs: Any) -> None:
+        """Initializes the WebDownloader class.
+
+        Args:
+            **kwargs (Any): Additional keyword arguments.
+        """
+    def download(self, source: str, output: str) -> None:
+        """Downloads web content.
+
+        Args:
+            source (str): The source of the web content URL.
+            output (str): The output where we put the downloaded content (usually a folder path).
+
+        Returns:
+            None
+        """
+    def download_from_multiple_urls(self, urls: list[str], output: str = '.', **kwargs: Any) -> None:
+        """Downloads web content from multiple URLs.
+
+        Args:
+            urls (list[str]): The URLs to download.
+            output (str): The output where we put the downloaded content (usually a folder path).
+            **kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            None
+        """
+    def download_crawl(self, urls: list[str] | str, output: str = '.', spider_type: type[Spider] | None = None) -> None:
+        """Downloads web content from the provided URLs.
+
+        This method uses a web scraper to crawl the provided URLs and saves the downloaded content to a file.
+
+        Args:
+            output (str): The output where we put the downloaded content (usually a folder path).
+            urls (list[str] | str): The URLs to crawl. Can be a single URL (str) or a list of URLs (list[str]).
+            spider_type (type[Spider] | None): The type of spider to use for downloading.
+                Defaults to None, which will use CrawlBaseSpider.
+
+        Returns:
+            None
+        """
+    def download_sitemap(self, urls: list[str] | str, output: str = '.', spider_type: type[Spider] | None = None) -> None:
+        """Downloads web content from the sitemap of the provided URLs.
+
+        This method uses a web scraper to scrape the sitemap of each URL and saves the downloaded content to a file..
+
+        Args:
+            urls (list[str] | str): The URLs to scrape. Can be a single URL (str) or a list of URLs (list[str]).
+            output (str): The output where we put the downloaded content (usually a folder path).
+            spider_type (type[Spider] | None): The type of spider to use for downloading.
+                Defaults to None, which will use CrawlSitemapSpider.
+
+        Returns:
+            None
+        """
+    def download_sitemap_links(self, urls: list[str] | str, output: str = '.', spider_type: type[Spider] | None = None) -> None:
+        """Retrieves all links from the sitemap of the provided URLs.
+
+        This method uses a web scraper to scrape the sitemap of each URL and returns a list of all found links.
+
+        Args:
+            urls (list[str] | str): The URLs to scrape. Can be a single URL (str) or a list of URLs (list[str]).
+            output (str): The output where we put the downloaded content (usually a folder path).
+            spider_type (type[Spider] | None)): The type of spider to use for downloading.
+                Defaults to None, which will use CrawlSitemapLinkSpider.
+
+        Returns:
+            None
+        """

gllm_docproc/downloader/html/scraper/scraper/spiders/__init__.pyi

@@ -0,0 +1,9 @@
+from .crawl_pdf_spider import CrawlPDFSpider as CrawlPDFSpider
+from .crawl_sitemap_link_spider import CrawlSitemapLinkSpider as CrawlSitemapLinkSpider
+from .crawl_sitemap_spider import CrawlSitemapSpider as CrawlSitemapSpider
+from .crawl_spider import CrawlBaseSpider as CrawlBaseSpider
+from .playwright_scrape_spider import PlaywrightScrapeSpider as PlaywrightScrapeSpider
+from .scrape_spider import ScrapeSpider as ScrapeSpider
+from .zyte_scrape_spider import ZyteScrapeSpider as ZyteScrapeSpider
+
+__all__ = ['ScrapeSpider', 'PlaywrightScrapeSpider', 'ZyteScrapeSpider', 'CrawlBaseSpider', 'CrawlSitemapSpider', 'CrawlSitemapLinkSpider', 'CrawlPDFSpider']

gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_pdf_spider.pyi

@@ -0,0 +1,27 @@
+from .crawl_spider import CrawlBaseSpider as CrawlBaseSpider
+from _typeshed import Incomplete
+from collections.abc import Generator
+from gllm_docproc.downloader.html.utils import clean_url as clean_url
+from scrapy.http import Request as Request, Response as Response
+
+class CrawlPDFSpider(CrawlBaseSpider):
+    """Scrapy CrawlSpider to crawl websites and save responses as PDFs using Playwright.
+
+    Attributes:
+        name (str): The name of the spider - 'crawl_pdf_spider'.
+        allowed_domains (list): The allowed domains for spider to crawl.
+        start_urls (list): The starting URLs for the spider to initiate crawling.
+        custom_settings (dict): Custom settings for the spider, including item pipelines configuration.
+        rules (tuple): The rules to be followed during the crawling process.
+    """
+    name: str
+    def add_playwright(self, request: Request, _: Response):
+        """Adds playwright meta information to the request."""
+    def start_requests(self) -> Generator[Incomplete]:
+        """Start Request.
+
+        Initiates requests for the specified start URLs using Scrapy requests with additional
+        meta information for playwright usage.
+        """
+    async def parse_web(self, response: Response):
+        """Parses the response obtained from the website, distinguishing between HTML content and other file types."""

gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_sitemap_link_spider.pyi

@@ -0,0 +1,29 @@
+from .crawl_sitemap_spider import CrawlSitemapSpider as CrawlSitemapSpider
+from scrapy.crawler import Crawler as Crawler
+from scrapy.http import Response as Response
+from typing import Any
+
+class CrawlSitemapLinkSpider(CrawlSitemapSpider):
+    """A Scrapy spider designed to scrape links from the sitemaps.
+
+    This spider uses the CrawlSitemapSpider base class to follow the sitemap links provided in the
+    robots.txt file of the website. It parses each page and extracts the URLs of the pages. If
+
+    Attributes:
+        name (str): The name of the spider - 'crawl_sitemap_link_spider'.
+        custom_settings (dict): Custom settings for the spider, including the log level and log file.
+    """
+    name: str
+    custom_settings: dict[str, Any]
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+        """Creates a new instance of the spider.
+
+        Args:
+            crawler (scrapy.crawler.Crawler): The Scrapy crawler object.
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+
+        Returns:
+            CrawlSitemapLinkSpider: A new instance of the spider.
+        """

gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_sitemap_spider.pyi

@@ -0,0 +1,61 @@
+from _typeshed import Incomplete
+from scrapy.crawler import Crawler as Crawler
+from scrapy.http import Response as Response
+from scrapy.spiders import SitemapSpider
+from typing import Any
+
+class CrawlSitemapSpider(SitemapSpider):
+    """A Scrapy spider designed to scrape content from the sitemaps.
+
+    This spider uses the SitemapSpider base class to follow the sitemap links provided in the
+    robots.txt file of the website. It parses each page and extracts the URLs of the pages. If
+    an error occurs during parsing, it logs the error.
+
+    Attributes:
+        name (str): The name of the spider - 'crawl_sitemap_spider'.
+        sitemap_urls (list): The URLs of the sitemaps to start crawling from.
+        allowed_domains (list): The domains that this spider is allowed to crawl.
+        custom_settings (dict): Custom settings for the spider, including the log level and log file.
+    """
+    name: str
+    custom_settings: dict[str, Any]
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+        """Creates a new CrawlSitemapSpider instance and sets the custom settings.
+
+        Args:
+            crawler (scrapy.crawler.Crawler): The crawler object.
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+
+        Returns:
+            CrawlSitemapSpider: The CrawlSitemapSpider instance.
+        """
+    sitemap_urls: Incomplete
+    allowed_domains: Incomplete
+    callback: Incomplete
+    removed_components: Incomplete
+    is_follow_page: Incomplete
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initializes the CrawlSitemapSpider instance.
+
+        The method initializes the CrawlSitemapSpider instance and sets the sitemap_urls, allowed_domains,
+        and sitemap_from_robots attributes based on the provided arguments.
+
+        Args:
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+        """
+    def parse(self, response: Response):
+        """Parse the response.
+
+        This method parses the response obtained from the website, extracts the URLs of the pages,
+        and follows the links to the next pages.
+
+        This method attempts to yield a dictionary containing the URL of the response. If an error occurs,
+        it yields the URL and an error message.
+        It also extracts the URLs of the next pages from the response and follows them.
+
+        Args:
+            response (scrapy.http.Response): The response object to parse.
+        """

gllm_docproc/downloader/html/scraper/scraper/spiders/crawl_spider.pyi

@@ -0,0 +1,50 @@
+from _typeshed import Incomplete
+from collections.abc import Generator
+from scrapy.crawler import Crawler as Crawler
+from scrapy.http import Request as Request, Response as Response
+from scrapy.spiders import CrawlSpider
+from typing import Any
+
+class CrawlBaseSpider(CrawlSpider):
+    """A Scrapy CrawlSpider designed to crawl and extract content from website.
+
+    Attributes:
+        name (str): The name of the spider - 'crawl_spider'.
+        allowed_domains (list): The allowed domains for spider to crawl.
+        start_urls (list): The starting URLs for the spider to initiate crawling.
+        custom_settings (dict): Custom settings for the spider, including item pipelines configuration.
+        rules (tuple): The rules to be followed during the crawling process.
+    """
+    name: str
+    rules: Incomplete
+    custom_settings: dict[str, Any]
+    def add_playwright(self, request: Request, _: Response):
+        """Adds playwright meta information to the request."""
+    def start_requests(self) -> Generator[Incomplete]:
+        """Start Request.
+
+        Initiates requests for the specified start URLs using Scrapy requests with additional
+        meta information for playwright usage.
+        """
+    start_urls: Incomplete
+    allowed_domains: Incomplete
+    callback: Incomplete
+    removed_components: Incomplete
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initialize the CrawlBaseSpider."""
+    async def parse_web(self, response: Response):
+        """Parses the response obtained from the website, distinguishing between HTML content and other file types."""
+    @classmethod
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+        """Creates a new instance of the CrawlBaseSpider with custom settings.
+
+        Args:
+            crawler (Crawler): The crawler object.
+            *args (Any): The arguments to be passed to the spider.
+            **kwargs (Any): The keyword arguments to be passed to the spider.
+
+        Returns:
+            CrawlBaseSpider: The CrawlBaseSpider instance.
+        """
+    async def errback(self, failure: Any):
+        """Handles errors encountered during the crawling process and closes playwright pages."""

gllm_docproc/downloader/html/scraper/scraper/spiders/playwright_scrape_spider.pyi

@@ -0,0 +1,22 @@
+from .scrape_spider import ScrapeSpider as ScrapeSpider
+from _typeshed import Incomplete
+from collections.abc import Generator
+from typing import Any
+
+class PlaywrightScrapeSpider(ScrapeSpider):
+    """A Scrapy spider designed to scrape content from website using playwright to render Javascript loaded page.
+
+    Attributes:
+        name (str): The name of the spider - 'playwright_scrape_spider'.
+        allowed_domains (list): The allowed domains for spider to crawl.
+        start_urls (list): The starting URLs for the spider to initiate crawling
+        custom_settings (dict): Custom settings for the spider, including item pipelines configuration.
+    """
+    name: str
+    custom_settings: dict[str, Any]
+    def start_requests(self) -> Generator[Incomplete]:
+        """Start Request.
+
+        Initiates requests for the specified start URLs using Scrapy requests with additional
+        meta information for playwright usage.
+        """