betterhtmlchunking 0.9__py3-none-any.whl

betterhtmlchunking/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python3
+
+from betterhtmlchunking.main import DomRepresentation

betterhtmlchunking/main.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+import attrs
+
+from attrs_strict import type_validator
+
+from betterhtmlchunking.utils import remove_unwanted_tags
+
+from betterhtmlchunking.tree_representation import\
+    DOMTreeRepresentation
+
+from betterhtmlchunking.tree_regions_system import\
+    TreeRegionsSystem
+from betterhtmlchunking.tree_regions_system import\
+    ReprLengthComparisionBy
+
+from betterhtmlchunking.render_system import\
+    RenderSystem
+
+
+tag_list_to_filter_out: list[str] = [
+    "/head",
+    "/select",
+    # "/form",
+    "/footer",
+    "/svg",
+    "/defs",
+    "/g",
+    "/header",
+    "/footer",
+    "/script",
+    "/style"
+]
+
+
+@attrs.define()
+class DomRepresentation:
+    # Input:
+    MAX_NODE_REPR_LENGTH: int = attrs.field(
+        validator=type_validator()
+    )
+    website_code: str = attrs.field(
+        validator=type_validator(),
+        repr=False
+    )
+    repr_length_compared_by: ReprLengthComparisionBy = attrs.field(
+        validator=type_validator()
+    )
+
+    # Optional inputs:
+    tag_list_to_filter_out: list[str] = attrs.field(
+        validator=type_validator(),
+        default=None
+    )
+
+    # Result:
+    tree_representation: DOMTreeRepresentation = attrs.field(
+        validator=type_validator(),
+        init=False,
+        repr=False
+    )
+    tree_regions_system: TreeRegionsSystem = attrs.field(
+        validator=type_validator(),
+        init=False,
+        repr=False
+    )
+    render_system: RenderSystem = attrs.field(
+        validator=type_validator(),
+        init=False,
+        repr=False
+    )
+
+    def __attrs_post_init__(self):
+        if self.tag_list_to_filter_out is None:
+            self.tag_list_to_filter_out = tag_list_to_filter_out
+
+    def compute_tree_representation(self):
+        self.tree_representation = DOMTreeRepresentation(
+            website_code=self.website_code,
+        )
+        self.tree_representation = remove_unwanted_tags(
+            tree_representation=self.tree_representation,
+            tag_list_to_filter_out=self.tag_list_to_filter_out
+        )
+        self.tree_representation.recompute_representation()
+
+    def compute_tree_regions_system(self):
+        self.tree_regions_system = TreeRegionsSystem(
+            tree_representation=self.tree_representation,
+            max_node_repr_length=self.MAX_NODE_REPR_LENGTH,
+            repr_length_compared_by=self.repr_length_compared_by
+        )
+
+    def compute_render_system(self):
+        self.render_system = RenderSystem(
+            tree_regions_system=self.tree_regions_system,
+            tree_representation=self.tree_representation
+        )
+
+    def start(self):
+        print("--- DOM REPRESENTATION ---")
+        print(" > Computing tree representation:")
+        self.compute_tree_representation()
+        print(" > Computing tree regions system:")
+        self.compute_tree_regions_system()
+        print(" > Computing render:")
+        self.compute_render_system()

betterhtmlchunking/render_system.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+import attrs
+
+from attrs_strict import type_validator
+
+import parsel_text
+
+from betterhtmlchunking.tree_representation import\
+    DOMTreeRepresentation
+
+from betterhtmlchunking.tree_regions_system import\
+    TreeRegionsSystem
+
+
+RegionOfInterestRenderT = dict[int, str]
+
+
+@attrs.define()
+class RenderSystem:
+    tree_regions_system: TreeRegionsSystem = attrs.field(
+        validator=type_validator()
+    )
+    tree_representation: DOMTreeRepresentation = attrs.field(
+        validator=type_validator()
+    )
+
+    html_render_with_pos_xpath: dict[int, RegionOfInterestRenderT] =\
+        attrs.field(
+            validator=type_validator(),
+            init=False
+        )
+    text_render_with_pos_xpath: dict[int, RegionOfInterestRenderT] =\
+        attrs.field(
+            validator=type_validator(),
+            init=False
+        )
+
+    # Render of the regions of interest, each one of them full:
+    html_render_roi: dict[int, str] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    text_render_roi: dict[int, str] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+
+    def get_roi_text_render_with_pos_xpath(self, roi_idx: int) -> str:
+        return "\n".join(
+            self.text_render_with_pos_xpath[roi_idx].values()
+        )
+
+    def get_roi_html_render_with_pos_xpath(self, roi_idx: int) -> str:
+        return "\n".join(
+            self.html_render_with_pos_xpath[roi_idx].values()
+        )
+
+    def render(self) -> None:
+        self.html_render_with_pos_xpath: dict[
+            int, RegionOfInterestRenderT] = {}
+        self.text_render_with_pos_xpath: dict[
+            int, RegionOfInterestRenderT] = {}
+
+        self.html_render_roi: dict[int, str] = {}
+        self.text_render_roi: dict[int, str] = {}
+
+        region_of_interest_idx: int = 0
+
+        # Execute the function:
+        for roi_idx, roi in\
+                self.tree_regions_system.sorted_roi_by_pos_xpath.items():
+            self.html_render_with_pos_xpath[roi_idx] = {}
+            self.text_render_with_pos_xpath[roi_idx] = {}
+
+            # print("*" * 50)
+            # print(roi.pos_xpath_list)
+
+            for pos_xpath in roi.pos_xpath_list:
+                # print(pos_xpath)
+
+                # HTML render:
+                prettified_pos_xpath_html: str =\
+                    self.tree_regions_system.tree_representation.xpaths_metadata[
+                        pos_xpath].bs4_elem.prettify(
+                            formatter="minimal"
+                        )
+                # print(prettified_pos_xpath_html)
+
+                # Text render:
+                pos_xpath_text: str =\
+                    parsel_text.get_bs4_soup_text(
+                        bs4_soup=self.tree_regions_system.tree_representation.xpaths_metadata[
+                            pos_xpath
+                        ].bs4_elem
+                    )
+
+                self.html_render_with_pos_xpath[
+                    roi_idx][pos_xpath] = prettified_pos_xpath_html
+                self.text_render_with_pos_xpath[
+                    roi_idx][pos_xpath] = pos_xpath_text
+
+                region_of_interest_idx += 1
+
+            self.html_render_roi[roi_idx] =\
+                self.get_roi_html_render_with_pos_xpath(
+                    roi_idx=roi_idx
+                )
+            self.text_render_roi[roi_idx] =\
+                self.get_roi_text_render_with_pos_xpath(
+                    roi_idx=roi_idx
+                )
+
+    def __attrs_post_init__(self):
+        self.render()

betterhtmlchunking/tree_regions_system.py

@@ -0,0 +1,379 @@
+#!/usr/bin/env python3
+
+import attrs
+
+from attrs_strict import type_validator
+
+import queue
+
+import treelib
+
+from betterhtmlchunking.tree_representation import\
+    DOMTreeRepresentation
+from betterhtmlchunking.tree_representation import\
+    get_xpath_depth
+
+from enum import StrEnum
+
+from typing import Iterator
+from typing import Any
+
+# import prettyprinter
+
+
+#################################
+#                               #
+#   --- TreeRegionsSystem ---   #
+#                               #
+#################################
+
+class ROIParsingState(StrEnum):
+    SEEK_END: str = "seek_end"
+    REGION_READY: str = "region_ready"
+    EOF: str = "EOF"
+
+
+@attrs.define()
+class RegionOfInterest:
+    pos_xpath_list: list[str] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    repr_length: int = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    node_is_roi: bool = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+
+    def __attrs_post_init__(self):
+        self.pos_xpath_list: list[str] = []
+        self.repr_length: int = 0
+        self.node_is_roi: bool = False
+
+
+class ReprLengthComparisionBy(StrEnum):
+    TEXT_LENGTH: str = "text_length"
+    HTML_LENGTH: str = "html_length"
+
+
+@attrs.define()
+class ROIMaker:
+    node_xpath: str = attrs.field(
+        validator=type_validator()
+    )
+    children_tags: list[str] = attrs.field(
+        validator=type_validator()
+    )
+    tree_representation: DOMTreeRepresentation = attrs.field(
+        validator=type_validator()
+    )
+    max_node_repr_length: int = attrs.field(
+        validator=type_validator()
+    )
+    repr_length_compared_by: ReprLengthComparisionBy = attrs.field(
+        validator=type_validator(),
+    )
+
+    PARSING_STATE: ROIParsingState = attrs.field(
+        validator=type_validator(),
+        default=ROIParsingState.SEEK_END
+    )
+    children_tags_iter: Iterator[str] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    actual_region_of_interest: Any = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    regions_of_interest_list: list[RegionOfInterest] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    children_to_enqueue: list[str] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+
+    def __attrs_post_init__(self) -> None:
+        # print(f"> ROIMaker: Node XPATH: {self.node_xpath}")
+
+        self.regions_of_interest_list: list[RegionOfInterest] = []
+        self.children_to_enqueue: list[str] = []
+        self.actual_region_of_interest = RegionOfInterest()
+
+        # Explore for ROIs on children.
+        self.children_tags_iter = iter(self.children_tags)
+
+        self.actual_region_of_interest.node_is_roi = False
+
+        self.step()
+        while self.PARSING_STATE != ROIParsingState.EOF:
+            self.step()
+
+        node_is_roi: bool = False
+        if len(self.children_tags) == 0:
+            node_is_roi = True
+        elif len(self.regions_of_interest_list) == 1:
+            roi = self.regions_of_interest_list[0]
+            if len(roi.pos_xpath_list) == len(self.children_tags):
+                node_is_roi = True
+
+        # Node itself is ROI.
+        if node_is_roi is True:
+            # print(f"> Node itself is ROI: {self.node_xpath}")
+            node: treelib.Node =\
+                self.tree_representation.tree.get_node(
+                    nid=self.node_xpath
+                )
+
+            node_repr_length: int =\
+                self.get_node_repr_length(node=node)
+
+            # prettyprinter.cpprint(node_repr_length)
+
+            # if node_repr_length > self.max_node_repr_length:
+            self.actual_region_of_interest.repr_length =\
+                node_repr_length
+            self.actual_region_of_interest.pos_xpath_list.append(
+                self.node_xpath
+            )
+            self.actual_region_of_interest.node_is_roi = True
+
+            self.regions_of_interest_list = []
+
+            self.regions_of_interest_list.append(
+                self.actual_region_of_interest
+            )
+
+        return None
+
+    def get_node_repr_length(self, node: treelib.Node) -> int:
+        match self.repr_length_compared_by:
+            case ReprLengthComparisionBy.TEXT_LENGTH:
+                node_repr_length: int = node.data.text_length
+            case ReprLengthComparisionBy.HTML_LENGTH:
+                node_repr_length: int = node.data.html_length
+
+        return node_repr_length
+
+    # Based on XMLStreamer.
+    def step(self):
+        match self.PARSING_STATE:
+            case ROIParsingState.SEEK_END:
+                # print("> SEEK END:")
+                try:
+                    children_tag: str = next(self.children_tags_iter)
+                    # print(f"children_tag: {children_tag}")
+                    node: treelib.Node =\
+                        self.tree_representation.tree.get_node(
+                            nid=children_tag
+                        )
+
+                    node_repr_length: int = self.get_node_repr_length(
+                        node=node
+                    )
+                    # print(f"node_repr_length: {node_repr_length}")
+
+                    if node_repr_length >= self.max_node_repr_length:
+                        self.PARSING_STATE = ROIParsingState.REGION_READY
+                        self.children_to_enqueue.append(children_tag)
+                    else:
+                        proposed_repr_length: int = node_repr_length +\
+                            self.actual_region_of_interest.repr_length
+
+                        # print(f"proposed_repr_length: {proposed_repr_length}")
+
+                        self.actual_region_of_interest.pos_xpath_list.append(
+                            node.identifier
+                        )
+                        self.actual_region_of_interest.repr_length =\
+                            proposed_repr_length
+
+                        if proposed_repr_length >= self.max_node_repr_length:
+                            self.PARSING_STATE = ROIParsingState.REGION_READY
+
+                except StopIteration:
+                    self.PARSING_STATE = ROIParsingState.EOF
+                    # print("StopIteration.")
+                    # prettyprinter.cpprint(self.actual_region_of_interest)
+                    if self.actual_region_of_interest.repr_length > 0 and\
+                            len(self.regions_of_interest_list) > 0:
+                        # print("Hanging xpaths.")
+                        self.regions_of_interest_list[-1].repr_length +=\
+                            self.actual_region_of_interest.repr_length
+                        self.regions_of_interest_list[-1].pos_xpath_list +=\
+                            self.actual_region_of_interest.pos_xpath_list
+                        self.actual_region_of_interest = RegionOfInterest()
+
+            case ROIParsingState.REGION_READY:
+                # print("> REGION READY:")
+                self.regions_of_interest_list.append(
+                    self.actual_region_of_interest
+                )
+
+                self.PARSING_STATE = ROIParsingState.SEEK_END
+                self.actual_region_of_interest = RegionOfInterest()
+
+
+def order_regions_of_interest_by_pos_xpath(
+    region_of_interest_list: list[RegionOfInterest],
+    pos_xpaths_list: list[str]
+        ) -> list[RegionOfInterest]:
+    # Create a mapping of xpath to its index in pos_xpaths_list
+    xpath_order = {
+        xpath: index for index, xpath in enumerate(pos_xpaths_list)
+    }
+
+    # Sort the region_of_interest_list
+    # based on the first pos_xpath_list entry for each region.
+    sorted_regions = sorted(
+        region_of_interest_list,
+        key=lambda region: xpath_order.get(
+            region.pos_xpath_list[0],
+            float("inf")
+        )
+    )
+
+    return sorted_regions
+
+
+@attrs.define()
+class TreeRegionsSystem:
+    tree_representation: DOMTreeRepresentation = attrs.field(
+        validator=type_validator()
+    )
+    max_node_repr_length: int = attrs.field(
+        validator=type_validator()
+    )
+    regions_of_interest_list: list[RegionOfInterest] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    sorted_roi_by_pos_xpath: dict[int, RegionOfInterest] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    repr_length_compared_by: ReprLengthComparisionBy = attrs.field(
+        validator=type_validator(),
+        default=ReprLengthComparisionBy.HTML_LENGTH
+    )
+
+    def __attrs_post_init__(self):
+        self.start()
+
+    def print_tree_node_states(self):
+        print("--- PRINT TREE NODE STATES ---")
+        for pos_xpath in self.tree_representation.pos_xpaths_list:
+            pad: str = get_xpath_depth(xpath=pos_xpath) * " " * 4
+            node = self.tree_representation.tree.get_node(pos_xpath)
+            print(f"{pad}|")
+            print(f"{pad}| {pos_xpath}")
+            print(f"{pad}| Text length: {node.data.text_length}")
+            print(f"{pad}| HTML length: {node.data.html_length}")
+
+    def get_node_repr_length(self, node: treelib.Node) -> int:
+        match self.repr_length_compared_by:
+            case ReprLengthComparisionBy.TEXT_LENGTH:
+                node_repr_length: int = node.data.text_length
+            case ReprLengthComparisionBy.HTML_LENGTH:
+                node_repr_length: int = node.data.html_length
+
+        return node_repr_length
+
+    def start(self):
+        self.regions_of_interest_list: list[RegionOfInterest] = []
+
+        subtrees_queue = queue.Queue()
+
+        subtrees_queue.put("/html")
+
+        while subtrees_queue.empty() is False:
+            # print("#" * 100)
+            node_xpath: str = subtrees_queue.get()
+            # print("--- NODE XPATH ---")
+            # print(node_xpath)
+
+            node: treelib.Node = self.tree_representation.tree.get_node(
+                node_xpath
+            )
+            # print(node.data.text_length)
+
+            children_tags: list[str] =\
+                self.tree_representation.get_children_tag_list(
+                    xpath=node_xpath
+                )
+
+            region_of_interest_maker = ROIMaker(
+                node_xpath=node_xpath,
+                children_tags=children_tags,
+                tree_representation=self.tree_representation,
+                max_node_repr_length=self.max_node_repr_length,
+                repr_length_compared_by=self.repr_length_compared_by
+            )
+
+            # print("--- REGIONS OF INTEREST LIST ---")
+            # prettyprinter.cpprint(region_of_interest_maker.regions_of_interest_list)
+
+            for roi in region_of_interest_maker.regions_of_interest_list:
+                # If we are based on text_length,
+                # tags like img (text_length == 0) are ignored.
+                # For that reason we base ROI on pos_xpath_list.
+                # if roi.text_length > 0:
+                if roi.pos_xpath_list != []:
+                    self.regions_of_interest_list.append(roi)
+
+            # print("--- CHILDREN TO ENQUEUE ---")
+            # prettyprinter.cpprint(region_of_interest_maker.children_to_enqueue)
+
+            """
+            Try to make ROIs under.
+            If ROI occupy all children, ROI contains node itself.
+
+            Those elements who are not ROI, are put into queue.
+            Elements who are ROI, are put into a separate dict.
+            """
+
+            for child_tag in region_of_interest_maker.children_to_enqueue:
+                subtrees_queue.put(child_tag)
+
+            """
+            for child_tag in children_tags:
+                subtrees_queue.put(child_tag)
+            """
+
+        sorted_regions: list[RegionOfInterest] =\
+            order_regions_of_interest_by_pos_xpath(
+                region_of_interest_list=\
+                self.regions_of_interest_list,
+                pos_xpaths_list=\
+                self.tree_representation.pos_xpaths_list
+            )
+
+        # This happen when there are no nodes to detect as RegionOfInterest
+        # or when max_node_repr_length is greater than total repr_length in
+        # the document.
+        if sorted_regions == [] and\
+                len(self.tree_representation.pos_xpaths_list) > 0:
+            node_xpath: str = self.tree_representation.pos_xpaths_list[0]
+
+            node: treelib.Node = self.tree_representation.tree.get_node(
+                node_xpath
+            )
+
+            node_repr_length: int = self.get_node_repr_length(
+                node=node
+            )
+            # print(node_repr_length)
+
+            roi = RegionOfInterest()
+            roi.pos_xpath_list = [node_xpath]
+            roi.repr_length = node_repr_length
+            roi.node_is_roi = True
+
+            sorted_regions = [roi]
+
+        self.sorted_roi_by_pos_xpath = dict(enumerate(sorted_regions))

betterhtmlchunking/tree_representation.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+
+import attrs
+from attrs_strict import type_validator
+
+import parsel_text
+
+import treelib
+
+import bs4
+
+from typing import Any
+
+# import prettyprinter
+
+# prettyprinter.install_extras()
+
+
+def get_parent_xpath(xpath: str) -> str:
+    if xpath.count("/") == 1:
+        return "root"
+    return "/".join(xpath.split("/")[:-1])
+
+
+def get_xpath_depth(xpath: str) -> int:
+    xpath: str = xpath.rstrip("/")
+    return xpath.count("/")
+
+
+def get_children_tags(node):
+    # Extract the tag names from the children of the given node
+    return [child.tag for child in node]
+
+
+def get_pos_xpath_from_bs4_elem(element) -> str:
+    components = []
+    child = element if isinstance(element, bs4.Tag) else element.parent
+
+    for parent in child.parents:
+        siblings = parent.find_all(child.name, recursive=False)
+
+        if len(siblings) == 1:
+            component = child.name
+        else:
+            index = next(
+                i for i, s in enumerate(siblings, 1) if s is child
+            )
+            component = f"{child.name}[{index}]"
+
+        components.append(component)
+        child = parent
+
+    components.reverse()
+    return "/" + "/".join(components)
+
+
+@attrs.define()
+class NodeMetadata:
+    idx: int = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    text_length: int = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    html_length: int = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    bs4_elem: Any = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    extra_metadata: Any = attrs.field(
+        validator=type_validator(),
+        default=None
+    )
+
+
+@attrs.define()
+class DOMTreeRepresentation:
+    website_code: str = attrs.field(
+        validator=type_validator()
+    )
+    soup: bs4.BeautifulSoup = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+
+    tree: treelib.Tree = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+
+    xpaths_metadata: dict[str, NodeMetadata] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+
+    pos_xpaths_list: list[str] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+    pos_sorted_xpaths: list[str] = attrs.field(
+        validator=type_validator(),
+        init=False
+    )
+
+    def __attrs_post_init__(self):
+        self.start()
+
+    def make_html_soup(self):
+        self.soup = bs4.BeautifulSoup(
+            self.website_code,
+            features="lxml"
+        )
+
+    def compute_xpaths_data(self):
+        children = self.soup.find_all(
+            name=True,
+            recursive=True
+        )
+
+        self.xpaths_metadata: dict[str, Any] = {}
+
+        for child in children:
+            pos_xpath: str = get_pos_xpath_from_bs4_elem(
+                element=child
+            )
+
+            child_text: str = parsel_text.get_bs4_soup_text(
+                bs4_soup=child
+            )
+            text_length: int = len(child_text)
+
+            child_html: str = child.prettify(
+                formatter="minimal"
+            )
+            html_length: int = len(child_html)
+
+            node_metadata = NodeMetadata()
+            node_metadata.text_length = text_length
+            node_metadata.html_length = html_length
+            node_metadata.bs4_elem = child
+
+            self.xpaths_metadata[pos_xpath] = node_metadata
+
+    def make_tree_representation(self):
+        # Initialize the tree.
+        self.tree = treelib.Tree()
+
+        # Add the root node:
+        self.tree.create_node(
+            tag="root",
+            identifier="root"
+        )
+
+        i = 0
+        # Add nodes to the tree:
+
+        for pos_xpath, node_metadata in self.xpaths_metadata.items():
+            parent_xpath: str = get_parent_xpath(xpath=pos_xpath)
+
+            node_metadata.idx = i
+
+            self.tree.create_node(
+                tag=pos_xpath,
+                identifier=pos_xpath,
+                parent=parent_xpath,
+                data=node_metadata
+            )
+
+            i += 1
+
+    def define_pos_xpaths_list(self):
+        self.pos_xpaths_list: list[str] = list(
+            self.xpaths_metadata.keys()
+        )
+
+    def sort_pos_xpaths(self):
+        self.pos_sorted_xpaths: list[str] = sorted(
+            self.pos_xpaths_list,
+            key=get_xpath_depth,
+            reverse=True
+        )
+
+    def get_children_tag_list(self, xpath: str) -> list[str]:
+        children_tags: list[str] = get_children_tags(
+            self.tree.children(xpath)
+        )
+        return children_tags
+
+    def delete_node(self, pos_xpath: str) -> None:
+        # Delete on treelib.Tree:
+        self.tree.remove_node(pos_xpath)
+        # print(self.tree)
+
+        # Delete on soup:
+        node = self.xpaths_metadata[pos_xpath].bs4_elem
+        node.decompose()
+        # print(self.soup.prettify())
+
+        keys_to_remove: list[str] = [
+            xpath for xpath in self.pos_xpaths_list
+            if xpath.startswith(pos_xpath)
+        ]
+
+        # Delete on metadata all which start with pos_xpath:
+        for xpath in keys_to_remove:
+            del self.xpaths_metadata[xpath]
+
+        self.define_pos_xpaths_list()
+        self.sort_pos_xpaths()
+
+        # After operating with node deletion
+        # you need to recompute the representation.
+
+    def recompute_representation(self):
+        self.compute_xpaths_data()
+        self.make_tree_representation()
+        self.define_pos_xpaths_list()
+        self.sort_pos_xpaths()
+
+    def start(self):
+        self.make_html_soup()
+        self.recompute_representation()

betterhtmlchunking/utils.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import treelib
+
+from betterhtmlchunking.tree_representation import DOMTreeRepresentation
+
+
+def wanted_xpath(
+    xpath: str,
+    tag_list_to_filter_out: list[str]
+        ) -> bool:
+    # Check if any of the unwanted tags are present in the given XPath
+    return not any(tag in xpath for tag in tag_list_to_filter_out)
+
+
+def remove_unwanted_tags(
+    tree_representation: DOMTreeRepresentation,
+    tag_list_to_filter_out: list[str]
+        ):
+    for pos_xpath in tree_representation.pos_xpaths_list:
+        if wanted_xpath(
+            xpath=pos_xpath,
+            tag_list_to_filter_out=tag_list_to_filter_out
+                ) is False:
+            try:
+                tree_representation.delete_node(pos_xpath=pos_xpath)
+            except treelib.exceptions.NodeIDAbsentError:
+                ...
+    return tree_representation

betterhtmlchunking-0.9.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Carlos A. Planchón
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

betterhtmlchunking-0.9.dist-info/METADATA

@@ -0,0 +1,153 @@
+Metadata-Version: 2.2
+Name: betterhtmlchunking
+Version: 0.9
+Summary: A Python library for intelligent HTML segmentation and ROI extraction. It builds a DOM tree from raw HTML and extracts content-rich regions for efficient web scraping and analysis.
+Author-email: "Carlos A. Planchón" <carlosandresplanchonprestes@gmail.com>
+License: MIT License
+Project-URL: repository, https://github.com/carlosplanchon/betterhtmlchunking.git
+Keywords: html,chunking,scraping,dom,roi,content extraction,web-scraping
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: attrs
+Requires-Dist: attrs-strict
+Requires-Dist: beautifulsoup4
+Requires-Dist: lxml
+Requires-Dist: treelib
+Requires-Dist: parsel_text
+
+```markdown
+# betterhtmlchunking
+
+A Python library for intelligently chunking HTML documents into structured, size-limited segments based on DOM tree analysis.
+
+## Overview
+
+This library processes HTML content to split it into semantically coherent chunks while respecting specified size constraints. It analyzes the DOM structure to identify optimal split points, preserving contextual information and document hierarchy.
+
+## Key Features
+
+- Custom DOM tree representation. 
+- Configurable chunk size limits (counting by text or HTML length).
+- Intelligent region-of-interest detection.
+- Dual output formats: HTML and plain text chunks.  
+- Preservation of structure relationships.
+- Customizable tag filtering.
+
+## Installation
+
+```bash
+pip install betterhtmlchunking
+```
+
+### Dependencies
+- Python 3.12+
+- attrs
+- treelib
+- beautifulsoup4
+- parsel-text
+- lxml
+- attrs-strict
+
+## Usage
+
+### Basic Example
+
+```python
+from betterhtmlchunking import DomRepresentation
+
+html_content = """
+<html>
+  <body>
+    <div id="content">
+      <h1>Document Title</h1>
+      <p>First paragraph...</p>
+      <p>Second paragraph...</p>
+    </div>
+  </body>
+</html>
+"""
+
+# Create document representation with 500 character chunks
+doc = DomRepresentation(
+    MAX_NODE_REPR_LENGTH=500,
+    website_code=html_content,
+    repr_length_compared_by="html_length"
+)
+
+# Access HTML chunks
+html_chunks = doc.render_system.html_render_roi
+for chunk_id, chunk in html_chunks.items():
+    print(f"Chunk {chunk_id}:\n{chunk}\n{'='*50}")
+
+# Access text chunks
+text_chunks = doc.render_system.text_render_roi
+for chunk_id, chunk in text_chunks.items():
+    print(f"Chunk {chunk_id}:\n{chunk}\n{'='*50}")
+```
+
+## Configuration
+
+### Key Parameters
+- `MAX_NODE_REPR_LENGTH`: Maximum allowed length for each chunk (in characters)
+- `repr_length_compared_by`: Length calculation method:
+  - `html_length`: HTML source length
+  - `text_length`: Rendered text length
+- `website_code`: Input HTML content
+
+### Advanced Features
+```python
+# Access the DOM tree structure
+tree = doc.tree_representation.tree
+
+# Get node metadata
+for node in tree.all_nodes():
+    print(f"XPath: {node.identifier}")
+    print(f"Text length: {node.data.text_length}")
+    print(f"HTML length: {node.data.html_length}")
+
+# Custom tag filtering (before processing)
+from betterhtmlchunking.tree_representation import DOMTreeRepresentation
+from betterhtmlchunking.utils import remove_unwanted_tags
+
+# Example usage of remove_unwanted_tags:
+tree_rep = DOMTreeRepresentation(website_code=html_content)
+filtered_rep = remove_unwanted_tags(tree_rep)
+```
+
+## How It Works
+
+1. **DOM Parsing**  
+   - Builds a tree representation of the HTML document.
+   - Calculates metadata (text length, HTML length) for each node.
+
+2. **Region Detection**  
+   - Uses **Breadth First Search (BFS)** to traverse the DOM tree in a level-order fashion, ensuring that each node is processed systematically.
+   - Combines nodes until the specified size limit is reached.
+   - Preserves parent-child relationships to maintain contextual integrity.
+
+3. **Chunk Generation**  
+   - Creates HTML chunks with original markup.
+   - Generates parallel text-only chunks.
+   - Maintains chunk order based on document structure.
+
+## Comparison to popular Chunking Techniques
+
+The actual practice (Feb. 2025) is to use **plain-text** or **token-based** chunking strategies, primarily aimed at keeping prompts within certain token limits for large language models. This approach is ideal for quick semantic retrieval or QA tasks on *unstructured* text.
+
+By contrast, **betterhtmlchunking** preserves the **HTML DOM structure**, calculating chunk boundaries based on each node’s text or HTML length. This approach is especially useful when you want to:
+- Retain or leverage the **hierarchical relationships** in the HTML (e.g., headings, nested divs)  
+- Filter out undesired tags or sections (like `<script>` or `<style>`)  
+- Pinpoint exactly where each chunk originated in the document (via positional XPaths)
+
+You can even combine the two techniques if you need both **structured extraction** (via betterhtmlchunking) and **LLM-friendly text chunking** (via LangChain) for advanced tasks such as summarization, semantic search, or large-scale QA pipelines.
+
+## License
+
+MIT License
+
+## Contributing
+Feel free to open issues or submit pull requests if you have suggestions or improvements.

betterhtmlchunking-0.9.dist-info/RECORD

@@ -0,0 +1,11 @@
+betterhtmlchunking/__init__.py,sha256=BBFsi9mQz1HccWlKF2jASNZxUK-z41f4gXPZY2bzfgE,78
+betterhtmlchunking/main.py,sha256=EDHE_C15FnXpEXKjTc2hCl4Uoqmm46NnMg_ZhISFIjg,2921
+betterhtmlchunking/render_system.py,sha256=CqL_WLNVqYV5UFqS0Q9pyV0Hb2pLwQBdUTxhUHqw5YU,3557
+betterhtmlchunking/tree_regions_system.py,sha256=wcKbmWD340Ipz2sRtQiZx1DEmGOPMMGen1v1FMM4lpg,12861
+betterhtmlchunking/tree_representation.py,sha256=qrcvVlLismL43csq4NDmnOz_rzLxn2Z5qJCk_C9mHxQ,5652
+betterhtmlchunking/utils.py,sha256=FB_QiTKQFdpt5CvhuXcPcJr5Mq-CV__tcEsnYD-QiAI,861
+betterhtmlchunking-0.9.dist-info/LICENSE,sha256=ng5atLTidZ1vzNYQp0LINoRma4D9cyw9ltbKTI8rvtE,1076
+betterhtmlchunking-0.9.dist-info/METADATA,sha256=AGs8EnwBJvecSk9pSjIHKzfSSooV27R2PBGKqgCI35s,5311
+betterhtmlchunking-0.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+betterhtmlchunking-0.9.dist-info/top_level.txt,sha256=tz4R-XL9TGFuEx5508yeFyoIS6m8l13O_EbYi91s5sM,19
+betterhtmlchunking-0.9.dist-info/RECORD,,

betterhtmlchunking-0.9.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

betterhtmlchunking-0.9.dist-info/top_level.txt

@@ -0,0 +1 @@
+betterhtmlchunking