gllm-docproc-binary 0.7.26__cp311-cp311-macosx_13_0_arm64.whl

gllm_docproc/loader/docx/python_docx_loader.pyi

@@ -0,0 +1,35 @@
+from gllm_docproc.loader.base_loader import BaseLoader as BaseLoader
+from gllm_docproc.loader.loader_utils import create_base_element_metadata as create_base_element_metadata, trim_table_empty_cells as trim_table_empty_cells, validate_file_extension as validate_file_extension
+from gllm_docproc.model.element import Element as Element, FOOTER as FOOTER, HEADER as HEADER, TABLE as TABLE, UNCATEGORIZED_TEXT as UNCATEGORIZED_TEXT
+from gllm_docproc.model.element_metadata import DOCX as DOCX, ElementMetadata as ElementMetadata
+from typing import Any
+
+class PythonDOCXLoader(BaseLoader):
+    """A class for loading and processing DOCX document using PythonDOCXLoader.
+
+    This class defines the structure for loading and processing DOCX document to retrieve required values
+    (Header, Body (Text and Table), Footer). It implements the 'load' method to handle DOCX loading
+    from a given file path.
+
+    PythonDOCXLoader is used to extract the Header, Body (Text and Table), Footer and metadata from the DOCX document.
+
+    Methods:
+        load(source, loaded_elements, **kwargs): Load a DOCX document.
+    """
+    def load(self, source: str, loaded_elements: list[dict[str, Any]] | None = None, **kwargs: Any) -> list[dict[str, Any]]:
+        """Load and process a DOCX document specified by the file path and name (source).
+
+        This method defines the process of loading a DOCX document using its file path.
+        It uses PythonDOCX to extract element text (with text structure) and table from the DOCX document.
+
+        Args:
+            source (str): The path to the DOCX document file.
+            loaded_elements (list[dict[str, Any]]): A list of dictionaries containing loaded content and metadata.
+            **kwargs (Any): Additional keyword arguments for the loader.
+
+        Kwargs:
+            original_source (str, optional): The original source of the document.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries containing loaded content and metadata.
+        """

gllm_docproc/loader/docx/python_docx_table_loader.pyi

@@ -0,0 +1,35 @@
+from gllm_docproc.loader.base_loader import BaseLoader as BaseLoader
+from gllm_docproc.loader.docx.python_docx_loader import PythonDOCXLoader as PythonDOCXLoader
+from gllm_docproc.model.element import Element as Element, TABLE as TABLE
+from typing import Any
+
+class PythonDOCXTableLoader(BaseLoader):
+    """Python DOCX Table Loader class to load tables from DOCX document.
+
+    This class is used to load tables from DOCX document using python-docx library.
+    Then it combined the existing loaded elements with the loaded tables.
+
+    Methods:
+        load: Load the tables from the DOCX document and combine it with the existing loaded elements.
+        _filter_table_elements: Filter the table elements from the loaded elements.
+        _get_table_content_count: Get the table content count.
+        _is_table_match: Is the table match with the merged table.
+        _find_matching_merged_table: Find the matching merged table.
+    """
+    def load(self, source: str, loaded_elements: list[dict[str, Any]] | None = None, **kwargs: Any) -> list[dict[str, Any]]:
+        """Load the tables from the DOCX document and combine it with the existing loaded elements.
+
+        This function loads the tables from the DOCX document using python-docx library.
+        Then it combined the existing loaded elements with the loaded tables.
+
+        Args:
+            source (str): The source file path.
+            loaded_elements (list[dict[str, Any]] | None): The existing loaded elements.
+            **kwargs (Any): The keyword arguments.
+
+        Kwargs:
+            original_source (str, optional): The original source of the document.
+
+        Returns:
+            list[dict[str, Any]]: The loaded elements.
+        """

gllm_docproc/loader/exception/__init__.pyi

@@ -0,0 +1,4 @@
+from .unsupported_file_extension_error import UnsupportedFileExtensionError as UnsupportedFileExtensionError
+from .video_conversion_error import VideoConversionError as VideoConversionError
+
+__all__ = ['UnsupportedFileExtensionError', 'VideoConversionError']

gllm_docproc/loader/exception/unsupported_file_extension_error.pyi

@@ -0,0 +1,7 @@
+from _typeshed import Incomplete
+
+class UnsupportedFileExtensionError(Exception):
+    """An exception for unsupported file extension."""
+    message: Incomplete
+    def __init__(self, ext: str, loader_name: str) -> None:
+        """Initialize the exception."""

gllm_docproc/loader/exception/video_conversion_error.pyi

@@ -0,0 +1,12 @@
+from _typeshed import Incomplete
+
+class VideoConversionError(Exception):
+    """An exception for video conversion failures."""
+    message: Incomplete
+    def __init__(self, video_path: str, cause: str) -> None:
+        """Initialize the exception.
+
+        Args:
+            video_path (str): Path to the video file that failed to convert.
+            cause (str): Description of the underlying cause of the failure.
+        """

gllm_docproc/loader/html/__init__.pyi

@@ -0,0 +1,5 @@
+from .flat import HTMLFlatLoader as HTMLFlatLoader
+from .html_base_loader import HTMLBaseLoader as HTMLBaseLoader
+from .nested import HTMLNestedLoader as HTMLNestedLoader
+
+__all__ = ['HTMLBaseLoader', 'HTMLFlatLoader', 'HTMLNestedLoader']

gllm_docproc/loader/html/exception/__init__.pyi

@@ -0,0 +1,3 @@
+from .html_load_exception import HtmlLoadException as HtmlLoadException
+
+__all__ = ['HtmlLoadException']

gllm_docproc/loader/html/exception/html_load_exception.pyi

@@ -0,0 +1,7 @@
+from _typeshed import Incomplete
+
+class HtmlLoadException(Exception):
+    """Custom exception for handling HtmlLoadException errors."""
+    message: Incomplete
+    def __init__(self, message: str) -> None:
+        """Initialize the HtmlLoadException."""

gllm_docproc/loader/html/flat/__init__.pyi

@@ -0,0 +1,3 @@
+from .html_flat_loader import HTMLFlatLoader as HTMLFlatLoader
+
+__all__ = ['HTMLFlatLoader']

gllm_docproc/loader/html/flat/html_flat_base_handler.pyi

@@ -0,0 +1,66 @@
+from _typeshed import Incomplete
+from gllm_docproc.loader.html.utils.html_utils import resolve_relative_url as resolve_relative_url
+from gllm_docproc.loader.html.utils.removed_components import RemovedComponents as RemovedComponents
+from gllm_docproc.loader.html.utils.string_utils import StringUtils as StringUtils
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata
+from gllm_docproc.model.media import Media as Media, MediaSourceType as MediaSourceType, MediaType as MediaType
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, HTMLTags as HTMLTags
+from parsel import Selector, SelectorList
+from typing import Callable
+
+def setup_mingw_path_for_cairosvg() -> None:
+    """On Windows, add mingw bin to PATH so CairoSVG can find required DLLs.
+
+    CairoSVG relies on DLLs (e.g., for font rendering and image processing) that may not be available
+    in the default PATH. Adding the mingw bin directory ensures these dependencies are found,
+    preventing import or runtime errors.
+    """
+
+CAIROSVG_AVAILABLE: bool
+logger: Incomplete
+
+def is_base_element(content_selector: Selector | SelectorList[Selector] | None, removed_components: RemovedComponents) -> bool:
+    """Check if the given content selector represents a base element.
+
+    A base element is determined by the type of element in the HTML document.
+    Supported base elements include:
+    1. Unsupported result (if content_selector is None)
+    2. String text
+    3. Removed Components (by class or tag) defined in RemovedComponents
+    4. <input>
+    5. <svg> image
+    6. <img>
+    7. <audio>, <video> (if multiple sources are given, select only the first one)
+    8. <iframe> (cannot get the content of the iframe)
+    9. <embed> (cannot get the content of the embed)
+    10. <br>
+
+    Args:
+        content_selector (Selector | SelectorList[Selector] | None): The selector representing the HTML content.
+        removed_components (RemovedComponents): Components to be removed from processing.
+
+    Returns:
+        bool: True if the content_selector represents a base element; False otherwise.
+    """
+def handle_base_element(content_selector: Selector | SelectorList[Selector] | None, html_head: ElementMetadata, removed_components: RemovedComponents) -> list[Element]:
+    """Handle the base HTML element and generate Element instances.
+
+    Args:
+        content_selector (Selector | SelectorList[Selector] | None): The selector representing the HTML content.
+        html_head (ElementMetadata): The metadata extracted from the HTML head.
+        removed_components (RemovedComponents): Components to be removed from processing.
+
+    Returns:
+        list[Element]: A list of Element instances generated from the HTML content.
+    """
+def get_handler(tag: str) -> Callable[[Selector | SelectorList[Selector], ElementMetadata], list[Element]] | None:
+    """Get the handler function corresponding to the given HTML tag.
+
+    Args:
+        tag (str): The HTML tag for which the handler function is requested.
+
+    Returns:
+        Callable[[Selector | SelectorList[Selector], ElementMetadata], list[Element]] | None: The handler
+        function corresponding to the given HTML tag.
+    """

gllm_docproc/loader/html/flat/html_flat_loader.pyi

@@ -0,0 +1,30 @@
+from gllm_docproc.loader.html.exception import HtmlLoadException as HtmlLoadException
+from gllm_docproc.loader.html.flat.html_flat_base_handler import handle_base_element as handle_base_element, is_base_element as is_base_element
+from gllm_docproc.loader.html.flat.html_flat_merger import merge_html_elements as merge_html_elements
+from gllm_docproc.loader.html.html_base_loader import HTMLBaseLoader as HTMLBaseLoader
+from gllm_docproc.loader.html.utils.html_utils import extract_html_head as extract_html_head, extract_html_title_tag as extract_html_title_tag
+from gllm_docproc.loader.html.utils.removed_components import RemovedComponents as RemovedComponents
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata
+from parsel import Selector, SelectorList
+
+class HTMLFlatLoader(HTMLBaseLoader):
+    """A loader class for loading web content and extracting information.
+
+    This class inherits from the BaseLoader class and provides methods to load web content,
+    extract information, and scrape data using Scrapy spiders.
+    """
+    def __init__(self) -> None:
+        """Initialize the HTMLFlatLoader."""
+    @classmethod
+    def extract_html_element(cls, content_selector: SelectorList[Selector] | Selector, html_head: ElementMetadata, removed_components: RemovedComponents) -> list[Element]:
+        """Recursively extract the content of an HTML element.
+
+        Args:
+            content_selector (SelectorList[Selector] | Selector): The content selector.
+            html_head (ElementMetadata): The HTML head metadata.
+            removed_components (RemovedComponents): The removed components.
+
+        Returns:
+                list[Element]: A list of web elements.
+        """

gllm_docproc/loader/html/flat/html_flat_merger.pyi

@@ -0,0 +1,23 @@
+from gllm_docproc.loader.html.utils.flat_table_utils import FlatTableUtils as FlatTableUtils
+from gllm_docproc.loader.html.utils.html_utils import resolve_relative_url as resolve_relative_url
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, HTMLTags as HTMLTags, ItemDataKeys as ItemDataKeys
+from parsel import Selector, SelectorList
+
+def merge_html_elements(content_selector: Selector | SelectorList[Selector], contents: list[Element], html_head: ElementMetadata) -> list[Element]:
+    """For non-base element, add metadata and merge children into one with parent element.
+
+    1. Add its HTML tag into its metadata
+    2. For some HTML tags, combine its children into a single element into the parent, for example:
+        1. Combine <ul> / <ol> children into a single element
+        2. Combine <a> children to become [text](https://link.com)
+
+    Args:
+        content_selector(Selector | SelectorList[Selector]): The content selector representing the HTML element.
+        contents (list[Element]): list of Element instances representing the contents of the HTML element.
+        html_head (ElementMetadata): The metadata extracted from the HTML head.
+
+    Returns:
+        list[Element]: list of Element instances after handling the contents based on the parent tag.
+    """

gllm_docproc/loader/html/html_base_loader.pyi

@@ -0,0 +1,25 @@
+from gllm_docproc.loader.base_loader import BaseLoader as BaseLoader
+from gllm_docproc.loader.html.utils.html_utils import is_html_content as is_html_content
+from typing import Any
+
+class HTMLBaseLoader(BaseLoader):
+    """A loader class for loading web content and extracting information.
+
+    This class inherits from the BaseLoader class and provides methods to load web content,
+    extract information, and scrape data using Scrapy spiders.
+    """
+    URL_INDEX: int
+    CONTENT_INDEX: int
+    def __init__(self, load_from_html_string: Any) -> None:
+        """Initialize the HTMLBaseLoader."""
+    def load(self, source: str, loaded_elements: list[dict[str, Any]] | None = None, **kwargs: Any) -> list[dict[str, Any]]:
+        """Loads web content and returns the extracted information in JSON format.
+
+        Args:
+            source (str): The source of the web content, either a URL or a file path.
+            loaded_elements (list[dict]): A list of loaded elements to be processed.
+            **kwargs (dict[str, Any]): Additional keyword arguments.
+
+        Returns:
+            list[dict]: The extracted information in JSON format.
+        """

gllm_docproc/loader/html/nested/__init__.pyi

@@ -0,0 +1,3 @@
+from .html_nested_loader import HTMLNestedLoader as HTMLNestedLoader
+
+__all__ = ['HTMLNestedLoader']

gllm_docproc/loader/html/nested/dictionary_utils.pyi

@@ -0,0 +1,40 @@
+class DictionaryUtils:
+    """A utility class providing methods to manipulate dictionaries."""
+    @staticmethod
+    def add_or_skip_value(dictionary, key, value):
+        """Adds a value to a dictionary if the value is not None for a given key.
+
+        Args:
+            dictionary (dict): The dictionary to be modified.
+            key (hashable): The key where the value needs to be added.
+            value (any): The value to be added.
+
+        Returns:
+            dict: The modified dictionary.
+        """
+    @staticmethod
+    def append_value(dictionary, key, value):
+        """Appends a value to a list under a specific key in a dictionary.
+
+        If the key already exists in the dictionary, the value is appended to the list under that key.
+        If the key does not exist, a new list is created with the value as its first element.
+
+        Args:
+            dictionary (dict): The dictionary to be modified.
+            key (hashable): The key under which the value needs to be added.
+            value (any): The value to be appended to the list under the key.
+
+        Returns:
+            dict: The modified dictionary with the value appended to the list under the key.
+        """
+    @staticmethod
+    def put_key_to_bottom(dictionary, key):
+        """Rearange key in dictionary.
+
+        Args:
+            dictionary (dict): The dictionary to be modified.
+            key (hashable): The key.
+
+        Returns:
+            dict: The modified dictionary.
+        """

gllm_docproc/loader/html/nested/html_nested_base_handler.pyi

@@ -0,0 +1,128 @@
+from .html_nested_element_handler import get_element as get_element
+from gllm_docproc.loader.html.exception import HtmlLoadException as HtmlLoadException
+from gllm_docproc.loader.html.nested.dictionary_utils import DictionaryUtils as DictionaryUtils
+from gllm_docproc.loader.html.utils.removed_components import RemovedComponents as RemovedComponents
+from gllm_docproc.loader.html.utils.string_utils import StringUtils as StringUtils
+from gllm_docproc.loader.html.utils.table_utils import TableUtils as TableUtils
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, HTMLTags as HTMLTags
+
+def get_element_content(content_selector, removed_components: RemovedComponents) -> list[dict]:
+    '''Traverses each element to get the content.
+
+    This function extract html body recursively
+
+    Input example:
+
+    .. code-block:: html
+
+        <html>
+        <head>
+        <title>Title</title>
+        </head>
+        <body>
+            <div class="container">
+                <h1>Welcome to My Website</h1>
+                <div>
+                    Hello World
+                </div>
+            </div>
+            <p>This is another paragraph.</p>
+        </body>
+        </html>
+
+    Output:
+
+    .. code-block:: groovy
+
+        [
+            {
+                \'tag\': \'body\',
+                \'class\': None,
+                \'content\': [
+                    {
+                        \'tag\': \'div\',
+                        \'class\': \'container\',
+                        \'content\': [
+                            {
+                                \'tag\': \'h1\',
+                                \'class\': None, \'content\': [
+                                    {
+                                        \'tag\': \'text\',
+                                        \'content\': \'Welcome to My Website\'
+                                    }
+                                ]
+                            },
+                            {
+                                \'tag\': \'div\',
+                                \'class\': None,
+                                \'content\': [
+                                    {\'tag\': \'text\', \'content\': \'Hello World\'}
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        \'tag\': \'p\',
+                        \'class\': None,
+                        \'content\': [
+                            {\'tag\': \'text\', \'content\': \'This is another paragraph.\'}
+                        ]
+                    }
+                ]
+            }
+        ]
+
+    Args:
+        content_selector: The content to be traversed.
+        removed_components: Removed class or tags.
+
+    Returns:
+        The List of extracted contents.
+    '''
+def is_base_element(content_selector, removed_components: RemovedComponents) -> bool:
+    """Check if the given content selector represents a base element.
+
+    See html_flat_base_handler.py for more information.
+
+    Args:
+        content_selector: The content selector to check.
+        removed_components (RemovedComponents): An instance of RemovedComponents class.
+
+    Returns:
+        bool: True if the content_selector represents a base element; False otherwise.
+    """
+def handle_base_element(content_selector, removed_components: RemovedComponents) -> list[dict]:
+    """Handle the processing of a base HTML element.
+
+    Args:
+        content_selector: The content selector representing the HTML element.
+        removed_components (RemovedComponents): An object containing information about components to be removed.
+
+    Returns:
+        List of dict : A List of dictionaries containing information about the HTML element, or None
+                      if the element should be skipped.
+                      - tag: HTML tag name.
+                      - class: CSS class of the element (if available).
+                      Additional keys may be added based on the specific element handler.
+    """
+def get_handler(tag: str):
+    """Get the element information from the specified content selector.
+
+    Args:
+        tag (str): The HTML tag to get the handler for.
+
+    Returns:
+        dict: A dictionary containing information about the HTML element.
+              - tag: HTML tag name.
+              - class: CSS class of the element (if available).
+              Additional keys may be added based on the specific element handler
+    """
+def create_text_dict(message):
+    """Creates a dictionary with 'text' tag and specified content.
+
+    Args:
+        message: The content to be added to the dictionary.
+
+    Returns:
+        A dictionary with 'text' tag and specified content.
+    """

gllm_docproc/loader/html/nested/html_nested_element_handler.pyi

@@ -0,0 +1,24 @@
+from gllm_docproc.loader.html.nested.dictionary_utils import DictionaryUtils as DictionaryUtils
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, HTMLTags as HTMLTags
+
+def get_element(content_selector) -> dict:
+    """Get the element information from the specified content selector.
+
+    Args:
+        content_selector: The content selector representing the HTML element.
+
+    Returns:
+        dict: A dictionary containing information about the HTML element.
+              - tag: HTML tag name.
+              - class: CSS class of the element (if available).
+              Additional keys may be added based on the specific element handler
+    """
+def get_handler(tag: str):
+    """Gets the handler for the specified HTML tag.
+
+    Args:
+        tag (str): The HTML tag to get the handler for.
+
+    Returns:
+        Callable: The handler function for the specified HTML tag.
+    """

gllm_docproc/loader/html/nested/html_nested_loader.pyi

@@ -0,0 +1,15 @@
+from .html_nested_base_handler import get_element_content as get_element_content
+from gllm_docproc.loader.html.exception import HtmlLoadException as HtmlLoadException
+from gllm_docproc.loader.html.html_base_loader import HTMLBaseLoader as HTMLBaseLoader
+from gllm_docproc.loader.html.utils.html_utils import extract_html_head as extract_html_head
+from gllm_docproc.loader.html.utils.removed_components import RemovedComponents as RemovedComponents
+from gllm_docproc.utils.html_constants import ContentDataKeys as ContentDataKeys, MetaDataKeys as MetaDataKeys
+
+class HTMLNestedLoader(HTMLBaseLoader):
+    """A loader class for loading web content and extracting information.
+
+    This class inherits from the BaseLoader class and provides methods to load web content,
+    extract information, and scrape data using Scrapy spiders.
+    """
+    def __init__(self) -> None:
+        """Initialize the HTMLNestedLoader."""

gllm_docproc/loader/html/utils/flat_table_utils.pyi

@@ -0,0 +1,44 @@
+from _typeshed import Incomplete
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.utils.html_constants import HTMLTags as HTMLTags, ItemDataKeys as ItemDataKeys, TableConstants as TableConstants
+
+class FlatTableUtils:
+    """A utility class providing methods for extracting data from HTML tables."""
+    colcount: int
+    rowspans: Incomplete
+    def __init__(self) -> None:
+        """Initialize the FlatTableUtils."""
+    def generate_tables(self, content: list[Element]) -> list[list[str]]:
+        """Generate tables from HTML content.
+
+        Args:
+            content (List[Element]): The list of Element instances representing the HTML content.
+
+        Returns:
+            List[List[str]]: A list containing the generated tables.
+        """
+    def filter_table(self, table_content: list[Element]) -> tuple[list[Element], list[Element]]:
+        """Filter the HTML table content.
+
+        Args:
+            table_content (List[Element]): The list of Element instances representing the HTML table.
+
+        Returns:
+            tuple[List[Element], List[Element]]: A tuple containing the filtered table content
+                and the removed elements.
+        """
+    def find_and_update_table_media(self, table_element: Element, elements: list[Element]) -> tuple[Element, list[Element]]:
+        """Find images in the table and return updated table element and found images.
+
+        Instead of modifying the input table_element directly, this function creates a copy and returns
+        both the updated table element and the list of images found in the table.
+
+        Args:
+            table_element (Element): The table element.
+            elements (list[Element]): The list of elements.
+
+        Returns:
+            tuple[Element, list[Element]]: A tuple containing:
+                - Updated copy of the table element with media metadata
+                - List of images found in the table
+        """

gllm_docproc/loader/html/utils/html_utils.pyi

@@ -0,0 +1,59 @@
+from gllm_docproc.loader.html.exception import HtmlLoadException as HtmlLoadException
+from gllm_docproc.model.element import Element as Element
+from gllm_docproc.model.element_metadata import ElementMetadata as ElementMetadata, HTML as HTML
+from gllm_docproc.utils.html_constants import HTMLTags as HTMLTags, MetaDataKeys as MetaDataKeys
+from scrapy.http import HtmlResponse
+from typing import Any
+
+def is_html_content(content: str) -> bool:
+    '''Check if the provided content appears to be HTML.
+
+    This function performs a case-insensitive check to determine if the content contains HTML tags,
+    specifically by searching for the opening and closing HTML tags ("<html" and "</html>").
+
+    Args:
+        content (str): The content to check.
+
+    Returns:
+        bool: True if the content is identified as HTML; False otherwise.
+    '''
+def extract_html_head(response: HtmlResponse, element_metadata: dict[str, Any] | None) -> ElementMetadata:
+    """Extracts metadata from an HTML response.
+
+    Args:
+        response (HtmlResponse): The HTML response.
+        element_metadata (dict[str, Any] | None): The element metadata.
+
+    Returns:
+        ElementMetadata: A class containing element metadata.
+
+    Raises:
+        HtmlLoadException: If an error occurs during the extraction process.
+    """
+def extract_html_title_tag(metadata: ElementMetadata) -> list[Element]:
+    """Gets the title element as a Element.
+
+    Args:
+        metadata (ElementMetadata): A class containing element metadata.
+
+    Returns:
+        List[Element]: List containing a single Element instance representing the title element.
+    """
+def resolve_relative_url(url: str, base_url: str) -> str:
+    '''Resolve a relative URL against a base URL.
+
+    This function handles various relative URL patterns:
+    1. Relative paths: "image.png", "folder/image.png"
+    2. Parent directory: "../image.png"
+    3. Absolute paths: "/image.png"
+    4. Already absolute URLs are returned as-is
+
+    Args:
+        url (str): The URL to resolve (may be relative or absolute).
+        base_url (str): The base URL to resolve against (can be a full path like
+            "http://example.com/articles/news/news_id").
+
+    Returns:
+        str: The resolved absolute URL, or the original url if it\'s already absolute
+            or base_url is invalid.
+    '''

gllm_docproc/loader/html/utils/removed_components.pyi

@@ -0,0 +1,53 @@
+class RemovedComponents:
+    """Class representing removed components from a document.
+
+    This class defines three methods for retrieving partial class, full class, and HTML tags
+    associated with removed components.
+    """
+    def get_partial_class(self) -> list[str]:
+        """Get partial class.
+
+        Method to get the partial class of the removed component. Partial class consists of
+        classes that will be filtered.
+
+        Returns:
+            str: The partial class name associated with the removed component.
+        """
+    def get_full_class(self) -> list[str]:
+        """Get full class.
+
+        Method to get the full class of the removed component. Full class consists of
+        exact match of classes that will be filtered.
+
+        Returns:
+            str: The full class name associated with the removed component.
+        """
+    def get_html_tags(self) -> list[str]:
+        """Method to get the HTML tags associated with the removed component.
+
+        Returns:
+            list: A list of HTML tags associated with the removed component.
+        """
+    @staticmethod
+    def is_removed_component(tag: str | None, class_: str | None, removed_components: RemovedComponents | None) -> bool:
+        """Checks if a component should be removed based on its tag and class.
+
+        Args:
+            tag (str): The tag of the component.
+            class_ (str): The class of the component.
+            removed_components (RemovedComponents): The components to be removed, including HTML tags and classes.
+
+        Returns:
+            True if the component should be removed, False otherwise.
+        """
+    @staticmethod
+    def check_list_in_substring(message: str, check_list: list[str]) -> bool:
+        """Checks if any substring from the check_list exists in the message string.
+
+        Args:
+            message (str): The string to search for substrings.
+            check_list (list): A list of substrings to be checked.
+
+        Returns:
+        - bool: True if any substring from check_list is found in message, otherwise False.
+        """