natural-pdf 0.2.12__py3-none-any.whl → 0.2.15__py3-none-any.whl

natural_pdf/core/highlighting_service.py

@@ -92,6 +92,16 @@ class HighlightRenderer:
 
     def _draw_highlights(self):
         """Draws all highlight shapes, borders, vertices, and attributes."""
+        # Get the pdfplumber page offset for coordinate translation
+        page_offset_x = 0
+        page_offset_y = 0
+
+        if hasattr(self.page, "_page") and hasattr(self.page._page, "bbox"):
+            # PDFPlumber page bbox might have negative offsets
+            page_offset_x = -self.page._page.bbox[0]
+            page_offset_y = -self.page._page.bbox[1]
+            logger.debug(f"Applying highlight offset: x={page_offset_x}, y={page_offset_y}")
+
         for highlight in self.highlights:
             # Create a transparent overlay for this single highlight
             overlay = Image.new("RGBA", self.base_image.size, (0, 0, 0, 0))
@@ -101,7 +111,11 @@ class HighlightRenderer:
 
             if highlight.is_polygon:
                 scaled_polygon = [
-                    (p[0] * self.scale_factor, p[1] * self.scale_factor) for p in highlight.polygon
+                    (
+                        (p[0] + page_offset_x) * self.scale_factor,
+                        (p[1] + page_offset_y) * self.scale_factor,
+                    )
+                    for p in highlight.polygon
                 ]
                 # Draw polygon fill and border
                 draw.polygon(
@@ -117,11 +131,16 @@ class HighlightRenderer:
             else:  # Rectangle
                 x0, top, x1, bottom = highlight.bbox
                 x0_s, top_s, x1_s, bottom_s = (
-                    x0 * self.scale_factor,
-                    top * self.scale_factor,
-                    x1 * self.scale_factor,
-                    bottom * self.scale_factor,
+                    (x0 + page_offset_x) * self.scale_factor,
+                    (top + page_offset_y) * self.scale_factor,
+                    (x1 + page_offset_x) * self.scale_factor,
+                    (bottom + page_offset_y) * self.scale_factor,
                 )
+                logger.debug(f"Original bbox: ({x0}, {top}, {x1}, {bottom})")
+                logger.debug(
+                    f"Offset bbox: ({x0 + page_offset_x}, {top + page_offset_y}, {x1 + page_offset_x}, {bottom + page_offset_y})"
+                )
+                logger.debug(f"Scaled bbox: ({x0_s}, {top_s}, {x1_s}, {bottom_s})")
                 scaled_bbox = [x0_s, top_s, x1_s, bottom_s]
                 # Draw rectangle fill and border
                 draw.rectangle(
@@ -1482,11 +1501,22 @@ class HighlightingService:
                 offset_x = crop_offset[0] * scale_factor
                 offset_y = crop_offset[1] * scale_factor
 
+            # Add pdfplumber page offset for coordinate translation
+            page_offset_x = 0
+            page_offset_y = 0
+            if hasattr(page, "_page") and hasattr(page._page, "bbox"):
+                # PDFPlumber page bbox might have negative offsets
+                page_offset_x = -page._page.bbox[0]
+                page_offset_y = -page._page.bbox[1]
+
             # Draw the highlight
             if polygon:
                 # Scale polygon points and apply offset
                 scaled_polygon = [
-                    (p[0] * scale_factor - offset_x, p[1] * scale_factor - offset_y)
+                    (
+                        (p[0] + page_offset_x) * scale_factor - offset_x,
+                        (p[1] + page_offset_y) * scale_factor - offset_y,
+                    )
                     for p in polygon
                 ]
                 draw.polygon(
@@ -1496,10 +1526,10 @@ class HighlightingService:
                 # Scale bbox and apply offset
                 x0, y0, x1, y1 = bbox
                 scaled_bbox = [
-                    x0 * scale_factor - offset_x,
-                    y0 * scale_factor - offset_y,
-                    x1 * scale_factor - offset_x,
-                    y1 * scale_factor - offset_y,
+                    (x0 + page_offset_x) * scale_factor - offset_x,
+                    (y0 + page_offset_y) * scale_factor - offset_y,
+                    (x1 + page_offset_x) * scale_factor - offset_x,
+                    (y1 + page_offset_y) * scale_factor - offset_y,
                 ]
                 draw.rectangle(
                     scaled_bbox, fill=color, outline=(color[0], color[1], color[2], BORDER_ALPHA)

natural_pdf/elements/base.py

@@ -106,6 +106,7 @@ class DirectionalMixin:
         include_source: bool = False,
         until: Optional[str] = None,
         include_endpoint: bool = True,
+        offset: float = 0.0,
         **kwargs,
     ) -> "Region":
         """
@@ -118,6 +119,7 @@ class DirectionalMixin:
             include_source: Whether to include this element/region's area in the result
             until: Optional selector string to specify a boundary element
             include_endpoint: Whether to include the boundary element found by 'until'
+            offset: Pixel offset when excluding source/endpoint (default: 0.1)
             **kwargs: Additional parameters for the 'until' selector search
 
         Returns:
@@ -127,7 +129,7 @@ class DirectionalMixin:
 
         is_horizontal = direction in ("left", "right")
         is_positive = direction in ("right", "below")  # right/below are positive directions
-        pixel_offset = 1  # Offset for excluding elements/endpoints
+        pixel_offset = offset  # Use provided offset for excluding elements/endpoints
 
         # 1. Determine initial boundaries based on direction and include_source
         if is_horizontal:
@@ -260,6 +262,7 @@ class DirectionalMixin:
         include_source: bool = False,
         until: Optional[str] = None,
         include_endpoint: bool = True,
+        offset: float = 0.1,
         **kwargs,
     ) -> "Region":
         """
@@ -271,6 +274,7 @@ class DirectionalMixin:
             include_source: Whether to include this element/region in the result (default: False)
             until: Optional selector string to specify an upper boundary element
             include_endpoint: Whether to include the boundary element in the region (default: True)
+            offset: Pixel offset when excluding source/endpoint (default: 0.1)
             **kwargs: Additional parameters
 
         Returns:
@@ -295,6 +299,7 @@ class DirectionalMixin:
             include_source=include_source,
             until=until,
             include_endpoint=include_endpoint,
+            offset=offset,
             **kwargs,
         )
 
@@ -305,6 +310,7 @@ class DirectionalMixin:
         include_source: bool = False,
         until: Optional[str] = None,
         include_endpoint: bool = True,
+        offset: float = 0.1,
         **kwargs,
     ) -> "Region":
         """
@@ -316,6 +322,7 @@ class DirectionalMixin:
             include_source: Whether to include this element/region in the result (default: False)
             until: Optional selector string to specify a lower boundary element
             include_endpoint: Whether to include the boundary element in the region (default: True)
+            offset: Pixel offset when excluding source/endpoint (default: 0.1)
             **kwargs: Additional parameters
 
         Returns:
@@ -340,6 +347,7 @@ class DirectionalMixin:
             include_source=include_source,
             until=until,
             include_endpoint=include_endpoint,
+            offset=offset,
             **kwargs,
         )
 
@@ -350,6 +358,7 @@ class DirectionalMixin:
         include_source: bool = False,
         until: Optional[str] = None,
         include_endpoint: bool = True,
+        offset: float = 0.1,
         **kwargs,
     ) -> "Region":
         """
@@ -361,6 +370,7 @@ class DirectionalMixin:
             include_source: Whether to include this element/region in the result (default: False)
             until: Optional selector string to specify a left boundary element
             include_endpoint: Whether to include the boundary element in the region (default: True)
+            offset: Pixel offset when excluding source/endpoint (default: 0.1)
             **kwargs: Additional parameters
 
         Returns:
@@ -385,6 +395,7 @@ class DirectionalMixin:
             include_source=include_source,
             until=until,
             include_endpoint=include_endpoint,
+            offset=offset,
             **kwargs,
         )
 
@@ -395,6 +406,7 @@ class DirectionalMixin:
         include_source: bool = False,
         until: Optional[str] = None,
         include_endpoint: bool = True,
+        offset: float = 0.1,
         **kwargs,
     ) -> "Region":
         """
@@ -406,6 +418,7 @@ class DirectionalMixin:
             include_source: Whether to include this element/region in the result (default: False)
             until: Optional selector string to specify a right boundary element
             include_endpoint: Whether to include the boundary element in the region (default: True)
+            offset: Pixel offset when excluding source/endpoint (default: 0.1)
             **kwargs: Additional parameters
 
         Returns:
@@ -430,6 +443,7 @@ class DirectionalMixin:
             include_source=include_source,
             until=until,
             include_endpoint=include_endpoint,
+            offset=offset,
             **kwargs,
         )
 
@@ -1195,6 +1209,9 @@ class Element(
 
         return self
 
+    def exclude(self):
+        self.page.add_exclusion(self)
+
     def _get_render_specs(
         self,
         mode: Literal["show", "render"] = "show",

natural_pdf/elements/element_collection.py

@@ -888,6 +888,9 @@ class ElementCollection(
         self._elements.sort(key=key, reverse=reverse)
         return self
 
+    def exclude(self):
+        self.page.add_exclusion(self)
+
     def highlight(
         self,
         label: Optional[str] = None,
@@ -1902,13 +1905,87 @@ class ElementCollection(
 
         return ElementCollection(all_found_elements)
 
-    def extract_each_text(self, **kwargs) -> List[str]:
-        """
-        Extract text from each element in this region.
+    def extract_each_text(
+        self,
+        order: Optional[Union[str, Callable[[T], Any]]] = None,
+        *,
+        newlines: bool = True,
+        **kwargs,
+    ) -> List[str]:
+        """Return a list with the extracted text for every element.
+
+        Parameters
+        ----------
+        order
+            Controls the ordering of elements **before** extraction:
+
+            * ``None`` (default) – keep the collection's current order.
+            * ``callable`` – a function that will be used as ``key`` for :pyfunc:`sorted`.
+            * ``"ltr"`` – left-to-right ordering (x0, then y-top).
+            * ``"rtl"`` – right-to-left ordering (−x0, then y-top).
+            * ``"natural"`` – natural reading order (y-top, then x0).
+
+        Remaining keyword arguments are forwarded to each element's
+        :py:meth:`extract_text` method.
         """
-        return self.apply(
-            lambda element: element.extract_text(**kwargs) if element is not None else None
-        )
+
+        # -- Determine ordering --------------------------------------------------
+        elements: List[T] = list(self._elements)  # make a shallow copy we can sort
+
+        if order is not None and len(elements) > 1:
+            try:
+                if callable(order):
+                    elements.sort(key=order)
+                elif isinstance(order, str):
+                    preset = order.lower()
+                    if preset in {"ltr", "left-to-right"}:
+                        elements.sort(
+                            key=lambda el: (
+                                (
+                                    getattr(el, "page", None).index
+                                    if hasattr(el, "page") and el.page
+                                    else 0
+                                ),
+                                getattr(el, "x0", 0),
+                                getattr(el, "top", 0),
+                            )
+                        )
+                    elif preset in {"rtl", "right-to-left"}:
+                        elements.sort(
+                            key=lambda el: (
+                                (
+                                    getattr(el, "page", None).index
+                                    if hasattr(el, "page") and el.page
+                                    else 0
+                                ),
+                                -getattr(el, "x0", 0),
+                                getattr(el, "top", 0),
+                            )
+                        )
+                    elif preset in {"natural", "tdlr", "top-down"}:
+                        elements.sort(
+                            key=lambda el: (
+                                (
+                                    getattr(el, "page", None).index
+                                    if hasattr(el, "page") and el.page
+                                    else 0
+                                ),
+                                getattr(el, "top", 0),
+                                getattr(el, "x0", 0),
+                            )
+                        )
+                    else:
+                        # Unknown preset – silently ignore to keep original order
+                        pass
+            except Exception:
+                # If anything goes wrong, fall back to original order
+                pass
+
+        # -- Extract ----------------------------------------------------------------
+        return [
+            el.extract_text(newlines=newlines, **kwargs) if el is not None else None  # type: ignore[arg-type]
+            for el in elements
+        ]
 
     def correct_ocr(
         self,
@@ -2673,10 +2750,17 @@ class ElementCollection(
         else:
             v_dist = 0  # Vertically overlapping
 
-        # Use Chebyshev distance (max of horizontal and vertical)
-        # This creates a square proximity zone
-        distance = max(h_dist, v_dist)
+        # ------------------------------------------------------------------
+        # Decide connection logic based on vertical_gap parameter
+        # ------------------------------------------------------------------
+        if vertical_gap is not None:
+            # Consider elements connected when they vertically stack within
+            # the allowed gap **and** have some horizontal overlap
+            horizontal_overlap = not (h_dist > 0)
+            return horizontal_overlap and v_dist <= vertical_gap
 
+        # Fallback to legacy Chebyshev distance using ``threshold``
+        distance = max(h_dist, v_dist)
         return distance <= threshold
 
     def _merge_region_group(
@@ -2752,6 +2836,9 @@ class ElementCollection(
     def dissolve(
         self,
         padding: float = 2.0,
+        *,
+        vertical_gap: Optional[float] = None,
+        vertical: Optional[bool] = False,
         geometry: Literal["rect", "polygon"] = "rect",
         group_by: List[str] = None,
     ) -> "ElementCollection":
@@ -2764,8 +2851,19 @@ class ElementCollection(
         bounding boxes.
 
         Args:
-            padding: Maximum distance in points between elements to consider
-                them connected. Default is 2.0 points.
+            padding: Maximum chebyshev distance (in any direction) between
+                elements to consider them connected **when ``vertical_gap`` is
+                not provided**. Default 2.0 pt.
+
+            vertical_gap: If given, switches to *stack-aware* dissolve:
+                two elements are connected when their horizontal projections
+                overlap (any amount) **and** the vertical distance between them
+                is ≤ ``vertical_gap``.  This lets you combine multi-line labels
+                that share the same column but have blank space between lines.
+
+            vertical: If given, automatically sets vertical_gap to maximum to
+                allow for easy vertical stacking.
+
             geometry: Type of geometry to use for merged regions. Currently only
                 "rect" (bounding box) is supported. "polygon" will raise
                 NotImplementedError.
@@ -2807,6 +2905,9 @@ class ElementCollection(
         if geometry not in ["rect", "polygon"]:
             raise ValueError(f"Invalid geometry type: {geometry}. Must be 'rect' or 'polygon'")
 
+        if vertical:
+            vertical_gap = float("inf")
+
         from natural_pdf.elements.region import Region
 
         # Filter to elements with bbox (all elements that can be dissolved)
@@ -2835,7 +2936,9 @@ class ElementCollection(
             logger.debug(f"Processing group {group_key} with {len(group_elements)} elements")
 
             # Find connected components within this group
-            components = self._find_connected_components_elements(group_elements, padding)
+            components = self._find_connected_components_elements(
+                group_elements, padding, vertical_gap
+            )
 
             # Merge each component
             for component_elements in components:
@@ -2894,7 +2997,7 @@ class ElementCollection(
         return groups
 
     def _find_connected_components_elements(
-        self, elements: List["Element"], padding: float
+        self, elements: List["Element"], padding: float, vertical_gap: Optional[float] = None
     ) -> List[List["Element"]]:
         """Find connected components among elements using union-find."""
         if not elements:
@@ -2919,7 +3022,7 @@ class ElementCollection(
         # Check all pairs of elements for connectivity
         for i in range(len(elements)):
             for j in range(i + 1, len(elements)):
-                if self._are_elements_connected(elements[i], elements[j], padding):
+                if self._are_elements_connected(elements[i], elements[j], padding, vertical_gap):
                     union(i, j)
 
         # Group elements by their connected component
@@ -3004,7 +3107,9 @@ class ElementCollection(
 
         return merged_region
 
-    def _are_elements_connected(self, elem1: "Element", elem2: "Element", threshold: float) -> bool:
+    def _are_elements_connected(
+        self, elem1: "Element", elem2: "Element", threshold: float, vertical_gap: float | None
+    ) -> bool:
         """Check if two elements are connected (adjacent or overlapping)."""
         # Check if elements are on the same page
         # Handle edge cases where elements might not have a page attribute
@@ -3057,6 +3162,12 @@ class ElementCollection(
         # This creates a square proximity zone
         distance = max(h_dist, v_dist)
 
+        if vertical_gap is not None:
+            # 1. vertical distance ≤ vertical_gap
+            # 2. horizontal ranges overlap OR touch
+            h_overlap = (min(x1_1, x1_2) - max(x0_1, x0_2)) >= 0
+            return h_overlap and v_dist <= vertical_gap
+
         return distance <= threshold
 
     def _copy_element_attributes_to_region(
@@ -3163,3 +3274,30 @@ class ElementCollection(
         return self
 
     # ------------------------------------------------------------------
+
+    # ------------------------------------------------------------------
+    # Public alias: combine
+    # ------------------------------------------------------------------
+    def combine(
+        self,
+        padding: float = 2.0,
+        *,
+        vertical_gap: Optional[float] = None,
+        vertical: Optional[bool] = False,
+        geometry: Literal["rect", "polygon"] = "rect",
+        group_by: List[str] = None,
+    ) -> "ElementCollection":
+        """Alias for :py:meth:`dissolve` – retained for discoverability.
+
+        Many users find the verb *combine* more intuitive than *dissolve* when
+        merging nearby or stacked elements into unified Regions.  The parameters
+        are identical; see :py:meth:`dissolve` for full documentation.
+        """
+
+        return self.dissolve(
+            padding=padding,
+            vertical_gap=vertical_gap,
+            vertical=vertical,
+            geometry=geometry,
+            group_by=group_by,
+        )

natural_pdf/elements/rect.py

@@ -88,6 +88,40 @@ class RectangleElement(Element):
         """Get the stroke width of the rectangle."""
         return self._obj.get("linewidth", 0)
 
+    @property
+    def is_horizontal(self) -> bool:
+        """Check if this is a horizontal line based on coordinates."""
+        # Calculate absolute difference in coordinates
+        dx = abs(self.x1 - self.x0)
+        dy = abs(self.top - self.bottom)
+
+        # Define a tolerance for near-horizontal lines (e.g., 1 point)
+        tolerance = 1.0
+
+        # Horizontal if y-change is within tolerance and x-change is significant
+        return dy <= tolerance and dx > tolerance
+
+    @property
+    def is_vertical(self) -> bool:
+        """Check if this is a vertical line based on coordinates."""
+        # Calculate absolute difference in coordinates
+        dx = abs(self.x1 - self.x0)
+        dy = abs(self.top - self.bottom)
+
+        # Define a tolerance for near-vertical lines (e.g., 1 point)
+        tolerance = 1.0
+
+        # Vertical if x-change is within tolerance and y-change is significant
+        return dx <= tolerance and dy > tolerance
+
+    @property
+    def orientation(self) -> str:
+        """Get the orientation of the line ('horizontal', 'vertical', or 'diagonal')."""
+        if self.is_horizontal:
+            return "horizontal"
+        elif self.is_vertical:
+            return "vertical"
+
     def extract_text(self, **kwargs) -> str:
         """
         Extract text from inside this rectangle.

natural_pdf/elements/region.py

@@ -45,6 +45,7 @@ from natural_pdf.utils.locks import pdf_render_lock  # Import the lock
 
 # Import new utils
 from natural_pdf.utils.text_extraction import filter_chars_spatially, generate_text_layout
+from natural_pdf.vision.mixin import VisualSearchMixin
 
 # Import viewer widget support
 from natural_pdf.widgets.viewer import _IPYWIDGETS_AVAILABLE, InteractiveViewerWidget
@@ -80,6 +81,7 @@ class Region(
     ExtractionMixin,
     ShapeDetectionMixin,
     DescribeMixin,
+    VisualSearchMixin,
     Visualizable,
 ):
     """Represents a rectangular region on a page.
@@ -736,6 +738,9 @@ class Region(
             and self.bottom > element.top
         )
 
+    def exclude(self):
+        self.page.add_exclusion(self)
+
     def highlight(
         self,
         label: Optional[str] = None,
@@ -1227,7 +1232,13 @@ class Region(
             return [e for e in page_elements if self._is_element_in_region(e)]
 
     def extract_text(
-        self, apply_exclusions=True, debug=False, content_filter=None, **kwargs
+        self,
+        apply_exclusions: bool = True,
+        debug: bool = False,
+        *,
+        newlines: Union[bool, str] = True,
+        content_filter=None,
+        **kwargs,
     ) -> str:
         """
         Extract text from this region, respecting page exclusions and using pdfplumber's
@@ -1236,6 +1247,7 @@ class Region(
         Args:
             apply_exclusions: Whether to apply exclusion regions defined on the parent page.
             debug: Enable verbose debugging output for filtering steps.
+            newlines: Whether to strip newline characters from the extracted text.
             content_filter: Optional content filter to exclude specific text patterns. Can be:
                 - A regex pattern string (characters matching the pattern are EXCLUDED)
                 - A callable that takes text and returns True to KEEP the character
@@ -1309,6 +1321,18 @@ class Region(
             user_kwargs=final_kwargs,  # Pass kwargs including content_filter
         )
 
+        # Flexible newline handling (same logic as TextElement)
+        if isinstance(newlines, bool):
+            if newlines is False:
+                replacement = " "
+            else:
+                replacement = None
+        else:
+            replacement = str(newlines)
+
+        if replacement is not None:
+            result = result.replace("\n", replacement).replace("\r", replacement)
+
         logger.debug(f"Region {self.bbox}: extract_text finished, result length: {len(result)}.")
         return result
 
@@ -1692,7 +1716,21 @@ class Region(
         else:
             filtered_page = base_plumber_page
 
-        cropped = filtered_page.crop(self.bbox)
+        # Ensure bbox is within pdfplumber page bounds
+        page_bbox = filtered_page.bbox
+        clipped_bbox = (
+            max(self.bbox[0], page_bbox[0]),  # x0
+            max(self.bbox[1], page_bbox[1]),  # y0
+            min(self.bbox[2], page_bbox[2]),  # x1
+            min(self.bbox[3], page_bbox[3]),  # y1
+        )
+
+        # Only crop if the clipped bbox is valid (has positive width and height)
+        if clipped_bbox[2] > clipped_bbox[0] and clipped_bbox[3] > clipped_bbox[1]:
+            cropped = filtered_page.crop(clipped_bbox)
+        else:
+            # If the region is completely outside the page bounds, return empty list
+            return []
 
         # Extract all tables from the cropped area
         tables = cropped.extract_tables(table_settings)
@@ -1786,7 +1824,21 @@ class Region(
             filtered_page = base_plumber_page
 
         # Now crop the (possibly filtered) page to the region bbox
-        cropped = filtered_page.crop(self.bbox)
+        # Ensure bbox is within pdfplumber page bounds
+        page_bbox = filtered_page.bbox
+        clipped_bbox = (
+            max(self.bbox[0], page_bbox[0]),  # x0
+            max(self.bbox[1], page_bbox[1]),  # y0
+            min(self.bbox[2], page_bbox[2]),  # x1
+            min(self.bbox[3], page_bbox[3]),  # y1
+        )
+
+        # Only crop if the clipped bbox is valid (has positive width and height)
+        if clipped_bbox[2] > clipped_bbox[0] and clipped_bbox[3] > clipped_bbox[1]:
+            cropped = filtered_page.crop(clipped_bbox)
+        else:
+            # If the region is completely outside the page bounds, return empty table
+            return []
 
         # Extract the single largest table from the cropped area
         table = cropped.extract_table(table_settings)

natural_pdf/elements/text.py

@@ -2,7 +2,7 @@
 Text element classes for natural-pdf.
 """
 
-from typing import TYPE_CHECKING, Any, Dict, Optional
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union
 
 from natural_pdf.elements.base import Element
 
@@ -236,7 +236,13 @@ class TextElement(Element):
         return (0, 0, 0)
 
     def extract_text(
-        self, keep_blank_chars=True, strip: Optional[bool] = True, content_filter=None, **kwargs
+        self,
+        keep_blank_chars: bool = True,
+        strip: Optional[bool] = True,
+        *,
+        newlines: Union[bool, str] = True,
+        content_filter=None,
+        **kwargs,
     ) -> str:
         """
         Extract text from this element.
@@ -292,6 +298,18 @@ class TextElement(Element):
         if strip:
             result = result.strip()
 
+        # Flexible newline handling
+        if isinstance(newlines, bool):
+            if newlines is False:
+                replacement = " "  # single space when False
+            else:
+                replacement = None  # keep as-is when True
+        else:
+            replacement = str(newlines)
+
+        if replacement is not None:
+            result = result.replace("\n", replacement).replace("\r", replacement)
+
         return result
 
     def contains(self, substring: str, case_sensitive: bool = True) -> bool: