natural-pdf 0.2.11__tar.gz → 0.2.12__tar.gz

{natural_pdf-0.2.11/natural_pdf.egg-info → natural_pdf-0.2.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: natural-pdf
-Version: 0.2.11
+Version: 0.2.12
 Summary: A more intuitive interface for working with PDFs
 Author-email: Jonathan Soma <jonathan.soma@gmail.com>
 License-Expression: MIT

{natural_pdf-0.2.11 → natural_pdf-0.2.12}/natural_pdf/analyzers/guides.py

@@ -185,7 +185,9 @@ class GuidesList(UserList):
         self,
         markers: Union[str, List[str], "ElementCollection", Callable, None],
         obj: Optional[Union["Page", "Region", "FlowRegion"]] = None,
-        align: Literal["left", "right", "center", "between"] = "left",
+        align: Union[
+            Literal["left", "right", "center", "between"], Literal["top", "bottom"]
+        ] = "left",
         outer: bool = True,
         tolerance: float = 5,
         *,
@@ -203,7 +205,10 @@ class GuidesList(UserList):
                 - Callable: function that takes a page and returns markers
                 - None: no markers
             obj: Page/Region/FlowRegion to search (uses parent's context if None)
-            align: How to align guides relative to found elements
+            align: How to align guides relative to found elements:
+                - For vertical guides: 'left', 'right', 'center', 'between'
+                - For horizontal guides: 'top', 'bottom', 'center', 'between'
+                - Note: 'left'/'right' also work for horizontal (mapped to top/bottom)
             outer: Whether to add outer boundary guides
             tolerance: Tolerance for snapping to element edges
             apply_exclusions: Whether to apply exclusion zones when searching for text
@@ -224,19 +229,25 @@ class GuidesList(UserList):
             self._callable = None
             actual_markers = markers
 
+        # Normalize alignment for horizontal guides
+        if self._axis == "horizontal":
+            if align == "top":
+                align = "left"
+            elif align == "bottom":
+                align = "right"
+
         # Check if parent is in flow mode
         if self._parent.is_flow_region:
             # Create guides across all constituent regions
             all_guides = []
             for region in self._parent.context.constituent_regions:
-                # Normalize markers for this region
-                marker_texts = _normalize_markers(actual_markers, region)
+                # Pass markers directly - from_content will handle them properly
 
                 # Create guides for this region
                 region_guides = Guides.from_content(
                     obj=region,
                     axis=self._axis,
-                    markers=marker_texts,
+                    markers=actual_markers,  # Pass original markers, not normalized text
                     align=align,
                     outer=outer,
                     tolerance=tolerance,
@@ -312,14 +323,14 @@ class GuidesList(UserList):
             return self._parent
 
         # Original single-region logic
-        # Normalize markers to list of text strings
-        marker_texts = _normalize_markers(actual_markers, target_obj)
+        # Pass markers directly to from_content which will handle them properly
+        # (no need to normalize here since from_content now handles ElementCollection)
 
         # Create guides for this axis
         new_guides = Guides.from_content(
             obj=target_obj,
             axis=self._axis,
-            markers=marker_texts,
+            markers=actual_markers,  # Pass original markers, not normalized text
             align=align,
             outer=outer,
             tolerance=tolerance,
@@ -930,6 +941,82 @@ class GuidesList(UserList):
         self.data.clear()
         return self._parent
 
+    def from_stripes(
+        self,
+        stripes=None,
+        color=None,  # Explicitly specify stripe color
+    ) -> "Guides":
+        """Create guides from striped table rows or columns.
+
+        Creates guides at both edges of stripe elements (e.g., colored table rows).
+        Perfect for zebra-striped tables where you need guides at every row boundary.
+
+        Args:
+            stripes: Elements representing stripes. If None, auto-detects.
+            color: Specific color to look for (e.g., '#00ffff'). If None, finds most common.
+
+        Examples:
+            # Auto-detect zebra stripes
+            guides.horizontal.from_stripes()
+
+            # Specific color
+            guides.horizontal.from_stripes(color='#00ffff')
+
+            # Manual selection
+            stripes = page.find_all('rect[fill=#00ffff]')
+            guides.horizontal.from_stripes(stripes)
+
+            # Vertical stripes
+            guides.vertical.from_stripes(color='#e0e0e0')
+
+        Returns:
+            Parent Guides object for chaining
+        """
+        from collections import defaultdict
+
+        target_obj = self._parent.context
+        if target_obj is None:
+            raise ValueError("No context available for stripe detection")
+
+        if stripes is None:
+            if color:
+                # User specified color
+                stripes = target_obj.find_all(f"rect[fill={color}]")
+            else:
+                # Auto-detect most common non-white fill
+                all_rects = target_obj.find_all("rect[fill]")
+
+                # Group by fill color
+                fill_counts = defaultdict(list)
+                for rect in all_rects:
+                    if rect.fill and rect.fill not in ["#ffffff", "white", "none", "transparent"]:
+                        fill_counts[rect.fill].append(rect)
+
+                if not fill_counts:
+                    return self._parent  # No stripes found
+
+                # Find most common fill color
+                stripes = max(fill_counts.values(), key=len)
+
+        if not stripes:
+            return self._parent
+
+        # Get both edges of each stripe
+        edges = []
+        if self._axis == "horizontal":
+            for stripe in stripes:
+                edges.extend([stripe.top, stripe.bottom])
+        else:
+            for stripe in stripes:
+                edges.extend([stripe.x0, stripe.x1])
+
+        # Remove duplicates and sort
+        edges = sorted(set(edges))
+
+        # Add guides
+        self.extend(edges)
+        return self._parent
+
     def __add__(self, other):
         """Handle addition of GuidesList objects by returning combined data."""
         if isinstance(other, GuidesList):
@@ -1459,7 +1546,9 @@ class Guides:
         obj: Union["Page", "Region", "FlowRegion"],
         axis: Literal["vertical", "horizontal"] = "vertical",
         markers: Union[str, List[str], "ElementCollection", None] = None,
-        align: Literal["left", "right", "center", "between"] = "left",
+        align: Union[
+            Literal["left", "right", "center", "between"], Literal["top", "bottom"]
+        ] = "left",
         outer: bool = True,
         tolerance: float = 5,
         apply_exclusions: bool = True,
@@ -1475,7 +1564,9 @@ class Guides:
                 - List[str]: list of selectors or literal text strings
                 - ElementCollection: collection of elements to extract text from
                 - None: no markers
-            align: Where to place guides relative to found text
+            align: Where to place guides relative to found text:
+                - For vertical guides: 'left', 'right', 'center', 'between'
+                - For horizontal guides: 'top', 'bottom', 'center', 'between'
             outer: Whether to add guides at the boundaries
             tolerance: Maximum distance to search for text
             apply_exclusions: Whether to apply exclusion zones when searching for text
@@ -1483,6 +1574,13 @@ class Guides:
         Returns:
             New Guides object aligned to text content
         """
+        # Normalize alignment for horizontal guides
+        if axis == "horizontal":
+            if align == "top":
+                align = "left"
+            elif align == "bottom":
+                align = "right"
+
         # Handle FlowRegion
         if hasattr(obj, "constituent_regions"):
             guides = cls(context=obj)
@@ -1530,39 +1628,51 @@ class Guides:
         elif hasattr(obj, "width"):
             bounds = (0, 0, obj.width, obj.height)
 
-        # Normalize markers to list of text strings
-        marker_texts = _normalize_markers(markers, obj)
+        # Handle different marker types
+        elements_to_process = []
 
-        # Find each marker and determine guide position
-        for marker in marker_texts:
-            if hasattr(obj, "find"):
-                element = obj.find(f'text:contains("{marker}")', apply_exclusions=apply_exclusions)
-                if element:
-                    if axis == "vertical":
-                        if align == "left":
-                            guides_coords.append(element.x0)
-                        elif align == "right":
-                            guides_coords.append(element.x1)
-                        elif align == "center":
-                            guides_coords.append((element.x0 + element.x1) / 2)
-                        elif align == "between":
-                            # For between, collect left edges for processing later
-                            guides_coords.append(element.x0)
-                    else:  # horizontal
-                        if align == "left":  # top for horizontal
-                            guides_coords.append(element.top)
-                        elif align == "right":  # bottom for horizontal
-                            guides_coords.append(element.bottom)
-                        elif align == "center":
-                            guides_coords.append((element.top + element.bottom) / 2)
-                        elif align == "between":
-                            # For between, collect top edges for processing later
-                            guides_coords.append(element.top)
+        # Check if markers is an ElementCollection or has elements attribute
+        if hasattr(markers, "elements") or hasattr(markers, "_elements"):
+            # It's an ElementCollection - use elements directly
+            elements_to_process = getattr(markers, "elements", getattr(markers, "_elements", []))
+        elif hasattr(markers, "__iter__") and not isinstance(markers, str):
+            # Check if it's an iterable of elements (not strings)
+            try:
+                markers_list = list(markers)
+                if markers_list and hasattr(markers_list[0], "x0"):
+                    # It's a list of elements
+                    elements_to_process = markers_list
+            except:
+                pass
 
-        # Handle 'between' alignment - find midpoints between adjacent markers
-        if align == "between" and len(guides_coords) >= 2:
-            # We need to get the right and left edges of each marker
-            marker_bounds = []
+        if elements_to_process:
+            # Process elements directly without text search
+            for element in elements_to_process:
+                if axis == "vertical":
+                    if align == "left":
+                        guides_coords.append(element.x0)
+                    elif align == "right":
+                        guides_coords.append(element.x1)
+                    elif align == "center":
+                        guides_coords.append((element.x0 + element.x1) / 2)
+                    elif align == "between":
+                        # For between, collect left edges for processing later
+                        guides_coords.append(element.x0)
+                else:  # horizontal
+                    if align == "left":  # top for horizontal
+                        guides_coords.append(element.top)
+                    elif align == "right":  # bottom for horizontal
+                        guides_coords.append(element.bottom)
+                    elif align == "center":
+                        guides_coords.append((element.top + element.bottom) / 2)
+                    elif align == "between":
+                        # For between, collect top edges for processing later
+                        guides_coords.append(element.top)
+        else:
+            # Fall back to text-based search
+            marker_texts = _normalize_markers(markers, obj)
+
+            # Find each marker and determine guide position
             for marker in marker_texts:
                 if hasattr(obj, "find"):
                     element = obj.find(
@@ -1570,9 +1680,52 @@ class Guides:
                     )
                     if element:
                         if axis == "vertical":
-                            marker_bounds.append((element.x0, element.x1))
+                            if align == "left":
+                                guides_coords.append(element.x0)
+                            elif align == "right":
+                                guides_coords.append(element.x1)
+                            elif align == "center":
+                                guides_coords.append((element.x0 + element.x1) / 2)
+                            elif align == "between":
+                                # For between, collect left edges for processing later
+                                guides_coords.append(element.x0)
                         else:  # horizontal
-                            marker_bounds.append((element.top, element.bottom))
+                            if align == "left":  # top for horizontal
+                                guides_coords.append(element.top)
+                            elif align == "right":  # bottom for horizontal
+                                guides_coords.append(element.bottom)
+                            elif align == "center":
+                                guides_coords.append((element.top + element.bottom) / 2)
+                            elif align == "between":
+                                # For between, collect top edges for processing later
+                                guides_coords.append(element.top)
+
+        # Handle 'between' alignment - find midpoints between adjacent markers
+        if align == "between" and len(guides_coords) >= 2:
+            # We need to get the right and left edges of each marker
+            marker_bounds = []
+
+            if elements_to_process:
+                # Use elements directly
+                for element in elements_to_process:
+                    if axis == "vertical":
+                        marker_bounds.append((element.x0, element.x1))
+                    else:  # horizontal
+                        marker_bounds.append((element.top, element.bottom))
+            else:
+                # Fall back to text search
+                if "marker_texts" not in locals():
+                    marker_texts = _normalize_markers(markers, obj)
+                for marker in marker_texts:
+                    if hasattr(obj, "find"):
+                        element = obj.find(
+                            f'text:contains("{marker}")', apply_exclusions=apply_exclusions
+                        )
+                        if element:
+                            if axis == "vertical":
+                                marker_bounds.append((element.x0, element.x1))
+                            else:  # horizontal
+                                marker_bounds.append((element.top, element.bottom))
 
             # Sort markers by their left edge (or top edge for horizontal)
             marker_bounds.sort(key=lambda x: x[0])

{natural_pdf-0.2.11 → natural_pdf-0.2.12}/natural_pdf/core/page.py

@@ -815,11 +815,38 @@ class Page(
                             if debug:
                                 print(f"    ✗ Empty iterable returned from callable '{label}'")
                     elif region_result:
-                        logger.warning(
-                            f"Callable exclusion '{exclusion_label}' returned non-Region object: {type(region_result)}. Skipping."
-                        )
-                        if debug:
-                            print(f"    ✗ Callable returned non-Region/None: {type(region_result)}")
+                        # Check if it's a single Element that can be converted to a Region
+                        from natural_pdf.elements.base import Element
+
+                        if isinstance(region_result, Element) or (
+                            hasattr(region_result, "bbox") and hasattr(region_result, "expand")
+                        ):
+                            try:
+                                # Convert Element to Region using expand()
+                                expanded_region = region_result.expand()
+                                if isinstance(expanded_region, Region):
+                                    expanded_region.label = label
+                                    regions.append(expanded_region)
+                                    if debug:
+                                        print(
+                                            f"    ✓ Converted Element to Region from callable '{label}': {expanded_region}"
+                                        )
+                                else:
+                                    if debug:
+                                        print(
+                                            f"    ✗ Element.expand() did not return a Region: {type(expanded_region)}"
+                                        )
+                            except Exception as e:
+                                if debug:
+                                    print(f"    ✗ Failed to convert Element to Region: {e}")
+                        else:
+                            logger.warning(
+                                f"Callable exclusion '{exclusion_label}' returned non-Region object: {type(region_result)}. Skipping."
+                            )
+                            if debug:
+                                print(
+                                    f"    ✗ Callable returned non-Region/None: {type(region_result)}"
+                                )
                     else:
                         if debug:
                             print(
@@ -839,6 +866,27 @@ class Page(
                 if debug:
                     print(f"  - Added direct region '{label}': {exclusion_item}")
 
+            # Process direct Element objects - convert to Region
+            elif hasattr(exclusion_item, "bbox") and hasattr(exclusion_item, "expand"):
+                try:
+                    # Convert Element to Region using expand()
+                    expanded_region = exclusion_item.expand()
+                    if isinstance(expanded_region, Region):
+                        expanded_region.label = label
+                        regions.append(expanded_region)
+                        if debug:
+                            print(
+                                f"  - Converted direct Element to Region '{label}': {expanded_region}"
+                            )
+                    else:
+                        if debug:
+                            print(
+                                f"  - Element.expand() did not return a Region: {type(expanded_region)}"
+                            )
+                except Exception as e:
+                    if debug:
+                        print(f"  - Failed to convert Element to Region: {e}")
+
             # Process string selectors (from PDF-level exclusions)
             elif isinstance(exclusion_item, str):
                 selector_str = exclusion_item
@@ -1081,7 +1129,7 @@ class Page(
         )  # _apply_selector doesn't filter
 
         # Filter the results based on exclusions if requested
-        if apply_exclusions and self._exclusions and results_collection:
+        if apply_exclusions and results_collection:
             filtered_elements = self._filter_elements_by_exclusions(results_collection.elements)
             # Return the first element from the filtered list
             return filtered_elements[0] if filtered_elements else None
@@ -1176,7 +1224,7 @@ class Page(
         )  # _apply_selector doesn't filter
 
         # Filter the results based on exclusions if requested
-        if apply_exclusions and self._exclusions and results_collection:
+        if apply_exclusions and results_collection:
             filtered_elements = self._filter_elements_by_exclusions(results_collection.elements)
             return ElementCollection(filtered_elements)
         else:
@@ -1548,7 +1596,7 @@ class Page(
         all_elements = self._element_mgr.get_all_elements()
 
         # Apply exclusions if requested
-        if apply_exclusions and self._exclusions:
+        if apply_exclusions:
             return self._filter_elements_by_exclusions(
                 all_elements, debug_exclusions=debug_exclusions
             )

{natural_pdf-0.2.11 → natural_pdf-0.2.12}/natural_pdf/elements/region.py

@@ -1270,7 +1270,8 @@ class Region(
         # 3. Get Relevant Exclusions (overlapping this region)
         apply_exclusions_flag = kwargs.get("apply_exclusions", apply_exclusions)
         exclusion_regions = []
-        if apply_exclusions_flag and self._page._exclusions:
+        if apply_exclusions_flag:
+            # Always call _get_exclusion_regions to get both page and PDF level exclusions
             all_page_exclusions = self._page._get_exclusion_regions(
                 include_callable=True, debug=debug
             )
@@ -1281,10 +1282,11 @@ class Region(
             exclusion_regions = overlapping_exclusions
             if debug:
                 logger.debug(
-                    f"Region {self.bbox}: Applying {len(exclusion_regions)} overlapping exclusions."
+                    f"Region {self.bbox}: Found {len(all_page_exclusions)} total exclusions, "
+                    f"{len(exclusion_regions)} overlapping this region."
                 )
         elif debug:
-            logger.debug(f"Region {self.bbox}: Not applying exclusions.")
+            logger.debug(f"Region {self.bbox}: Not applying exclusions (apply_exclusions=False).")
 
         # 4. Spatially Filter Characters using Utility
         # Pass self as the target_region for precise polygon checks etc.

{natural_pdf-0.2.11 → natural_pdf-0.2.12/natural_pdf.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: natural-pdf
-Version: 0.2.11
+Version: 0.2.12
 Summary: A more intuitive interface for working with PDFs
 Author-email: Jonathan Soma <jonathan.soma@gmail.com>
 License-Expression: MIT

{natural_pdf-0.2.11 → natural_pdf-0.2.12}/natural_pdf.egg-info/SOURCES.txt

@@ -203,6 +203,26 @@ optimization/performance_results/image_heavy_snapshots.csv
 optimization/performance_results/image_heavy_snapshots.json
 optimization/performance_results/text_heavy_snapshots.csv
 optimization/performance_results/text_heavy_snapshots.json
+temp/debug_cell_extraction.py
+temp/debug_exclusion_overlap.py
+temp/debug_exclusions_guides.py
+temp/debug_extra_guide.py
+temp/debug_outer_boundaries.py
+temp/debug_st_search.py
+temp/fix_page_exclusions.py
+temp/test_exclusion_with_debug.py
+temp/test_find_exclusions_fix.py
+temp/test_find_exclusions_fix_no_recursion.py
+temp/test_fix_real_pdf.py
+temp/test_fix_working.py
+temp/test_fixed_pdf_exclusions.py
+temp/test_horizontal_top_bottom.py
+temp/test_marker_order.py
+temp/test_original_exclusions_now_work.py
+temp/test_pdf_exclusions_with_guides.py
+temp/test_region_exclusions_detailed.py
+temp/test_stripes_real_pdf.py
+temp/test_vertical_stripes.py
 tests/conftest.py
 tests/test_annotate.py
 tests/test_arabic_performance.py
@@ -220,8 +240,10 @@ tests/test_dissolve_single_elements.py
 tests/test_dissolve_vertical_offset_issue.py
 tests/test_document_qa.py
 tests/test_element_addition.py
+tests/test_element_collection_guides.py
 tests/test_element_collection_show_cols.py
 tests/test_element_collection_slicing.py
+tests/test_element_exclusions.py
 tests/test_element_show_crop_highlights.py
 tests/test_empty_pseudo_class.py
 tests/test_exclusions.py
@@ -244,12 +266,15 @@ tests/test_guides_extract_table.py
 tests/test_guides_extract_table_collections.py
 tests/test_guides_extract_table_exclusions.py
 tests/test_guides_extract_table_real.py
+tests/test_guides_from_stripes.py
 tests/test_guides_integration.py
+tests/test_guides_marker_sorting.py
 tests/test_highlight_detection.py
 tests/test_highlight_detection_comprehensive.py
 tests/test_highlight_protocol.py
 tests/test_highlight_protocol_simple.py
 tests/test_highlight_regions.py
+tests/test_horizontal_guides_alignment.py
 tests/test_include_boundaries_comprehensive.py
 tests/test_include_boundaries_debug.py
 tests/test_include_boundaries_final.py
@@ -268,6 +293,7 @@ tests/test_multi_page_table_discovery.py
 tests/test_optional_deps.py
 tests/test_page_exclusion_lists.py
 tests/test_pdf_add_exclusion_elementcollection.py
+tests/test_pdf_exclusions_in_find_methods.py
 tests/test_region_show_crop_highlights.py
 tests/test_region_viewer.py
 tests/test_sections_end_only.py

natural_pdf-0.2.12/temp/debug_cell_extraction.py

@@ -0,0 +1,42 @@
+"""Debug cell text extraction with exclusions"""
+from natural_pdf import PDF
+from natural_pdf.analyzers.guides import Guides
+
+pdf = PDF("pdfs/m27.pdf")
+page = pdf.pages[0]
+
+# Add exclusions
+pdf.add_exclusion(lambda page: page.find(text="PREMISE").above(), label="header")
+
+# Check exclusions are registered
+print("Exclusions on page:")
+exclusions = page._get_exclusion_regions(debug=True)
+
+# Create guides and build grid
+headers = page.find(text="NUMBER").right(include_source=True).expand(top=3, bottom=3).find_all('text')
+guides = Guides(page)
+guides.vertical.from_content(headers, align='left')
+guides.horizontal.from_stripes()
+
+# Build grid and get cells
+grid_result = guides.build_grid(include_outer_boundaries=True)
+cells = grid_result["regions"]["cells"]
+
+print(f"\nTotal cells: {len(cells)}")
+
+# Check first row cells (these should be in excluded area)
+first_row_cells = [c for c in cells if c.bbox[1] < 90]  # y < 90
+print(f"\nFirst row cells: {len(first_row_cells)}")
+
+for i, cell in enumerate(first_row_cells[:3]):
+    print(f"\nCell {i}:")
+    print(f"  Bbox: {cell.bbox}")
+    print(f"  Raw text: {repr(cell.extract_text(apply_exclusions=False))}")
+    print(f"  With exclusions: {repr(cell.extract_text(apply_exclusions=True))}")
+
+# Now test the full table extraction
+print("\n\nFull table extraction:")
+result = guides.extract_table(include_outer_boundaries=True, apply_exclusions=True, header=False)
+df = result.to_df()
+print("\nFirst row of dataframe:")
+print(df.iloc[0].to_dict() if not df.empty else "Empty")

natural_pdf-0.2.12/temp/debug_exclusion_overlap.py

@@ -0,0 +1,43 @@
+"""Debug how exclusions work with overlapping regions"""
+from natural_pdf import PDF
+from natural_pdf.analyzers.guides import Guides
+
+pdf = PDF("pdfs/m27.pdf")
+page = pdf.pages[0]
+
+# Add exclusion
+pdf.add_exclusion(lambda page: page.find(text="PREMISE").above(), label="header")
+
+# Get the exclusion region
+exclusions = page._get_exclusion_regions()
+excl_region = exclusions[0]
+print(f"Exclusion region: {excl_region.bbox}")
+print(f"Exclusion bottom: {excl_region.bbox[3]}")
+
+# Create a test cell that overlaps the exclusion
+# Cell 1 from before: (32.06, 0.5, 73.18288, 79.53999999999996)
+test_cell = page.region(32.06, 0.5, 73.18288, 79.53999999999996)
+
+print(f"\nTest cell: {test_cell.bbox}")
+print(f"Cell overlaps exclusion: top={test_cell.bbox[1]} < excl_bottom={excl_region.bbox[3]}")
+
+# Extract text from different y-ranges
+print("\nText in different parts of the cell:")
+
+# Part above exclusion line (should be empty)
+upper_part = page.region(32.06, 0.5, 73.18288, 59.12)
+print(f"Upper part (0.5 to 59.12): '{upper_part.extract_text(apply_exclusions=True)}'")
+
+# Part below exclusion line (should have text)  
+lower_part = page.region(32.06, 59.12, 73.18288, 79.54)
+print(f"Lower part (59.12 to 79.54): '{lower_part.extract_text()}'")
+
+# The whole cell
+print(f"Whole cell with exclusions: '{test_cell.extract_text(apply_exclusions=True)}'")
+print(f"Whole cell without exclusions: '{test_cell.extract_text(apply_exclusions=False)}'")
+
+# Check what text elements are in this region
+print("\nText elements in cell:")
+cell_texts = test_cell.find_all('text')
+for t in cell_texts[:5]:
+    print(f"  '{t.text}' at y={t.top:.2f}-{t.bottom:.2f}")