natural-pdf 0.2.4__tar.gz → 0.2.6__tar.gz

{natural_pdf-0.2.4 → natural_pdf-0.2.6}/.gitignore

@@ -1,3 +1,4 @@
+temp
 Untitled*.ipynb
 importtime_output.txt
 .notebook_cache.json

{natural_pdf-0.2.4/natural_pdf.egg-info → natural_pdf-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: natural-pdf
-Version: 0.2.4
+Version: 0.2.6
 Summary: A more intuitive interface for working with PDFs
 Author-email: Jonathan Soma <jonathan.soma@gmail.com>
 License-Expression: MIT

{natural_pdf-0.2.4 → natural_pdf-0.2.6}/natural_pdf/analyzers/guides.py

@@ -143,7 +143,7 @@ class GuidesList(UserList):
 
     def from_content(
         self,
-        markers: Union[str, List[str], "ElementCollection", None],
+        markers: Union[str, List[str], "ElementCollection", Callable, None],
         obj: Optional[Union["Page", "Region", "FlowRegion"]] = None,
         align: Literal["left", "right", "center", "between"] = "left",
         outer: bool = True,
@@ -160,6 +160,7 @@ class GuidesList(UserList):
                 - str: single selector (e.g., 'text:contains("Name")') or literal text
                 - List[str]: list of selectors or literal text strings
                 - ElementCollection: collection of elements to extract text from
+                - Callable: function that takes a page and returns markers
                 - None: no markers
             obj: Page/Region/FlowRegion to search (uses parent's context if None)
             align: How to align guides relative to found elements
@@ -174,13 +175,22 @@ class GuidesList(UserList):
         if target_obj is None:
             raise ValueError("No object provided and no context available")
 
+        # Store callable markers for later evaluation
+        if callable(markers):
+            self._callable = markers
+            # For now, evaluate with the current target object to get initial guides
+            actual_markers = markers(target_obj)
+        else:
+            self._callable = None
+            actual_markers = markers
+
         # Check if parent is in flow mode
         if self._parent.is_flow_region:
             # Create guides across all constituent regions
             all_guides = []
             for region in self._parent.context.constituent_regions:
                 # Normalize markers for this region
-                marker_texts = _normalize_markers(markers, region)
+                marker_texts = _normalize_markers(actual_markers, region)
 
                 # Create guides for this region
                 region_guides = Guides.from_content(
@@ -263,7 +273,7 @@ class GuidesList(UserList):
 
         # Original single-region logic
         # Normalize markers to list of text strings
-        marker_texts = _normalize_markers(markers, target_obj)
+        marker_texts = _normalize_markers(actual_markers, target_obj)
 
         # Create guides for this axis
         new_guides = Guides.from_content(
@@ -1541,11 +1551,15 @@ class Guides:
         # Add outer guides if requested
         if outer and bounds:
             if axis == "vertical":
-                guides_coords.insert(0, bounds[0])  # x0
-                guides_coords.append(bounds[2])  # x1
+                if outer == True or outer == "first":
+                    guides_coords.insert(0, bounds[0])  # x0
+                if outer == True or outer == "last":
+                    guides_coords.append(bounds[2])  # x1
             else:
-                guides_coords.insert(0, bounds[1])  # y0
-                guides_coords.append(bounds[3])  # y1
+                if outer == True or outer == "first":
+                    guides_coords.insert(0, bounds[1])  # y0
+                if outer == True or outer == "last":
+                    guides_coords.append(bounds[3])  # y1
 
         # Remove duplicates and sort
         guides_coords = sorted(list(set(guides_coords)))
@@ -3302,7 +3316,7 @@ class Guides:
         markers: Union[str, List[str], "ElementCollection", None] = None,
         obj: Optional[Union["Page", "Region"]] = None,
         align: Literal["left", "right", "center", "between"] = "left",
-        outer: bool = True,
+        outer: Union[str, bool] = True,
         tolerance: float = 5,
         apply_exclusions: bool = True,
     ) -> "Guides":
@@ -3319,7 +3333,10 @@ class Guides:
                 - None: no markers
             obj: Page or Region to search (uses self.context if None)
             align: How to align guides relative to found elements
-            outer: Whether to add outer boundary guides
+            outer: Whether to add outer boundary guides. Can be:
+                - bool: True/False to add/not add both
+                - "first": To add boundary before the first element
+                - "last": To add boundary before the last element
             tolerance: Tolerance for snapping to element edges
             apply_exclusions: Whether to apply exclusion zones when searching for text
 
@@ -3445,7 +3462,15 @@ class Guides:
 
     def extract_table(
         self,
-        target: Optional[Union["Page", "Region"]] = None,
+        target: Optional[
+            Union[
+                "Page",
+                "Region",
+                "PageCollection",
+                "ElementCollection",
+                List[Union["Page", "Region"]],
+            ]
+        ] = None,
         source: str = "guides_temp",
         cell_padding: float = 0.5,
         include_outer_boundaries: bool = False,
@@ -3457,8 +3482,11 @@ class Guides:
         cell_extraction_func: Optional[Callable[["Region"], Optional[str]]] = None,
         show_progress: bool = False,
         content_filter: Optional[Union[str, Callable[[str], bool], List[str]]] = None,
+        apply_exclusions: bool = True,
         *,
         multi_page: Literal["auto", True, False] = "auto",
+        header: Union[str, List[str], None] = "first",
+        skip_repeating_headers: Optional[bool] = None,
     ) -> "TableResult":
         """
         Extract table data directly from guides without leaving temporary regions.
@@ -3469,8 +3497,11 @@ class Guides:
         3. Cleans up all temporary regions
         4. Returns the TableResult
 
+        When passed a collection (PageCollection, ElementCollection, or list), this method
+        will extract tables from each element and combine them into a single result.
+
         Args:
-            target: Page or Region to create regions on (uses self.context if None)
+            target: Page, Region, or collection of Pages/Regions to extract from (uses self.context if None)
             source: Source label for temporary regions (will be cleaned up)
             cell_padding: Internal padding for cell regions in points
             include_outer_boundaries: Whether to add boundaries at edges if missing
@@ -3482,7 +3513,15 @@ class Guides:
             cell_extraction_func: Optional callable for custom cell text extraction
             show_progress: Controls progress bar for text method
             content_filter: Content filtering function or patterns
+            apply_exclusions: Whether to apply exclusion regions during text extraction (default: True)
             multi_page: Controls multi-region table creation for FlowRegions
+            header: How to handle headers when extracting from collections:
+                - "first": Use first row of first element as headers (default)
+                - "all": Expect headers on each element, use from first element
+                - None: No headers, use numeric indices
+                - List[str]: Custom column names
+            skip_repeating_headers: Whether to remove duplicate header rows when extracting from collections.
+                Defaults to True when header is "first" or "all", False otherwise.
 
         Returns:
             TableResult: Extracted table data
@@ -3494,20 +3533,49 @@ class Guides:
             ```python
             from natural_pdf.analyzers import Guides
 
-            # Create guides from detected lines
+            # Single page extraction
             guides = Guides.from_lines(page, source_label="detected")
-
-            # Extract table directly - no temporary regions left behind
             table_data = guides.extract_table()
-
-            # Convert to pandas DataFrame
             df = table_data.to_df()
+
+            # Multiple page extraction
+            guides = Guides(pages[0])
+            guides.vertical.from_content(['Column 1', 'Column 2'])
+            table_result = guides.extract_table(pages, header=['Col1', 'Col2'])
+            df = table_result.to_df()
+
+            # Region collection extraction
+            regions = pdf.find_all('region[type=table]')
+            guides = Guides(regions[0])
+            guides.vertical.from_lines(n=3)
+            table_result = guides.extract_table(regions)
             ```
         """
-        target_obj = target or self.context
-        if not target_obj:
+        from natural_pdf.core.page_collection import PageCollection
+        from natural_pdf.elements.element_collection import ElementCollection
+
+        target_obj = target if target is not None else self.context
+        if target_obj is None:
             raise ValueError("No target object available. Provide target parameter or context.")
 
+        # Check if target is a collection - if so, delegate to _extract_table_from_collection
+        if isinstance(target_obj, (PageCollection, ElementCollection, list)):
+            # For collections, pass through most parameters as-is
+            return self._extract_table_from_collection(
+                elements=target_obj,
+                header=header,
+                skip_repeating_headers=skip_repeating_headers,
+                method=method,
+                table_settings=table_settings,
+                use_ocr=use_ocr,
+                ocr_config=ocr_config,
+                text_options=text_options,
+                cell_extraction_func=cell_extraction_func,
+                show_progress=show_progress,
+                content_filter=content_filter,
+                apply_exclusions=apply_exclusions,
+            )
+
         # Get the page for cleanup later
         if hasattr(target_obj, "x0") and hasattr(target_obj, "top"):  # Region
             page = target_obj._page
@@ -3552,6 +3620,7 @@ class Guides:
                 cell_extraction_func=cell_extraction_func,
                 show_progress=show_progress,
                 content_filter=content_filter,
+                apply_exclusions=apply_exclusions,
             )
 
             return table_result
@@ -3577,6 +3646,165 @@ class Guides:
             except Exception as cleanup_err:
                 logger.warning(f"Failed to clean up temporary regions: {cleanup_err}")
 
+    def _extract_table_from_collection(
+        self,
+        elements: Union["PageCollection", "ElementCollection", List[Union["Page", "Region"]]],
+        header: Union[str, List[str], None] = "first",
+        skip_repeating_headers: Optional[bool] = None,
+        method: Optional[str] = None,
+        table_settings: Optional[dict] = None,
+        use_ocr: bool = False,
+        ocr_config: Optional[dict] = None,
+        text_options: Optional[Dict] = None,
+        cell_extraction_func: Optional[Callable[["Region"], Optional[str]]] = None,
+        show_progress: bool = True,
+        content_filter: Optional[Union[str, Callable[[str], bool], List[str]]] = None,
+        apply_exclusions: bool = True,
+    ) -> "TableResult":
+        """
+        Extract tables from multiple pages or regions using this guide pattern.
+
+        This method applies the guide to each element, extracts tables, and combines
+        them into a single TableResult. Dynamic guides (using lambdas) are evaluated
+        for each element.
+
+        Args:
+            elements: PageCollection, ElementCollection, or list of Pages/Regions to extract from
+            header: How to handle headers:
+                - "first": Use first row of first element as headers (default)
+                - "all": Expect headers on each element, use from first element
+                - None: No headers, use numeric indices
+                - List[str]: Custom column names
+            skip_repeating_headers: Whether to remove duplicate header rows.
+                Defaults to True when header is "first" or "all", False otherwise.
+            method: Table extraction method (passed to extract_table)
+            table_settings: Settings for pdfplumber table extraction
+            use_ocr: Whether to use OCR for text extraction
+            ocr_config: OCR configuration parameters
+            text_options: Dictionary of options for the 'text' method
+            cell_extraction_func: Optional callable for custom cell text extraction
+            show_progress: Show progress bar for multi-element extraction (default: True)
+            content_filter: Content filtering function or patterns
+            apply_exclusions: Whether to apply exclusion regions during extraction
+
+        Returns:
+            TableResult: Combined table data from all elements
+
+        Example:
+            ```python
+            # Create guide with static vertical, dynamic horizontal
+            guide = Guides(regions[0])
+            guide.vertical.from_content(columns, outer="last")
+            guide.horizontal.from_content(lambda r: r.find_all('text:starts-with(NF-)'))
+
+            # Extract from all regions
+            table_result = guide._extract_table_from_collection(regions, header=columns)
+            df = table_result.to_df()
+            ```
+        """
+        from natural_pdf.core.page_collection import PageCollection
+        from natural_pdf.elements.element_collection import ElementCollection
+        from natural_pdf.tables.result import TableResult
+
+        # Convert to list if it's a collection
+        if isinstance(elements, (PageCollection, ElementCollection)):
+            element_list = list(elements)
+        else:
+            element_list = elements
+
+        if not element_list:
+            return TableResult([])
+
+        # Determine header handling
+        if skip_repeating_headers is None:
+            skip_repeating_headers = header in ["first", "all"] or isinstance(header, list)
+
+        all_rows = []
+        header_row = None
+
+        # Configure progress bar
+        iterator = element_list
+        if show_progress and len(element_list) > 1:
+            try:
+                from tqdm.auto import tqdm
+
+                iterator = tqdm(
+                    element_list, desc="Extracting tables from elements", unit="element"
+                )
+            except ImportError:
+                pass
+
+        for i, element in enumerate(iterator):
+            # Create a new Guides object for this element
+            element_guide = Guides(element)
+
+            # Copy vertical guides (usually static)
+            if hasattr(self.vertical, "_callable") and self.vertical._callable is not None:
+                # If vertical is dynamic (lambda), evaluate it
+                element_guide.vertical.from_content(self.vertical._callable(element))
+            else:
+                # Copy static vertical positions
+                element_guide.vertical.data = self.vertical.data.copy()
+
+            # Handle horizontal guides
+            if hasattr(self.horizontal, "_callable") and self.horizontal._callable is not None:
+                # If horizontal is dynamic (lambda), evaluate it
+                element_guide.horizontal.from_content(self.horizontal._callable(element))
+            else:
+                # Copy static horizontal positions
+                element_guide.horizontal.data = self.horizontal.data.copy()
+
+            # Extract table from this element
+            table_result = element_guide.extract_table(
+                method=method,
+                table_settings=table_settings,
+                use_ocr=use_ocr,
+                ocr_config=ocr_config,
+                text_options=text_options,
+                cell_extraction_func=cell_extraction_func,
+                show_progress=False,  # Don't show nested progress
+                content_filter=content_filter,
+                apply_exclusions=apply_exclusions,
+            )
+
+            # Convert to list of rows
+            rows = list(table_result)
+
+            # Handle headers based on strategy
+            if i == 0:  # First element
+                if header == "first" or header == "all":
+                    # Use first row as header
+                    if rows:
+                        header_row = rows[0]
+                        rows = rows[1:]  # Remove header from data
+                elif isinstance(header, list):
+                    # Custom headers provided
+                    header_row = header
+            else:  # Subsequent elements
+                if header == "all" and skip_repeating_headers and rows:
+                    # Expect and remove header row
+                    if rows and header_row and rows[0] == header_row:
+                        rows = rows[1:]
+                    elif rows:
+                        # Still remove first row if it looks like a header
+                        rows = rows[1:]
+
+            # Add rows to combined result
+            all_rows.extend(rows)
+
+        # Create final TableResult
+        if isinstance(header, list):
+            # Custom headers - prepend to data
+            final_result = TableResult(all_rows)
+        elif header_row is not None:
+            # Prepend discovered header
+            final_result = TableResult([header_row] + all_rows)
+        else:
+            # No headers
+            final_result = TableResult(all_rows)
+
+        return final_result
+
     def _get_flow_orientation(self) -> Literal["vertical", "horizontal", "unknown"]:
         """Determines if a FlowRegion's constituent parts are arranged vertically or horizontally."""
         if not self.is_flow_region or len(self.context.constituent_regions) < 2:

{natural_pdf-0.2.4 → natural_pdf-0.2.6}/natural_pdf/core/element_manager.py

@@ -939,6 +939,11 @@ class ElementManager:
         self.load_elements()
         return self._elements.get("chars", [])
 
+    def invalidate_cache(self):
+        """Invalidate the cached elements, forcing a reload on next access."""
+        self._elements = None
+        logger.debug(f"Page {self._page.number}: ElementManager cache invalidated")
+
     @property
     def words(self):
         """Get all word elements."""