natural-pdf 25.3.16__py3-none-any.whl

natural_pdf/utils/reading_order.py

@@ -0,0 +1,227 @@
+"""
+Reading order utilities for natural-pdf.
+"""
+from typing import List, Dict, Any, Callable, Optional
+
+
+def establish_reading_order(elements: List[Dict[str, Any]], 
+                           algorithm: str = 'basic') -> List[Dict[str, Any]]:
+    """
+    Establish reading order for a collection of elements.
+    
+    Args:
+        elements: List of elements to order
+        algorithm: Algorithm to use ('basic', 'column', 'complex')
+        
+    Returns:
+        List of elements in reading order
+    """
+    if algorithm == 'basic':
+        return _basic_reading_order(elements)
+    elif algorithm == 'column':
+        return _column_reading_order(elements)
+    elif algorithm == 'complex':
+        return _complex_reading_order(elements)
+    else:
+        # Default to basic
+        return _basic_reading_order(elements)
+
+
+def _basic_reading_order(elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Basic top-to-bottom, left-to-right reading order.
+    
+    Args:
+        elements: List of elements to order
+        
+    Returns:
+        List of elements in reading order
+    """
+    # Simple sort by y0 (top), then by x0 (left)
+    return sorted(elements, key=lambda e: (
+        e.get('top', e.get('y0', 0)), 
+        e.get('x0', 0)
+    ))
+
+
+def _column_reading_order(elements: List[Dict[str, Any]], 
+                         column_threshold: float = 0.2,
+                         x_tolerance: float = 10.0) -> List[Dict[str, Any]]:
+    """
+    Reading order that accounts for columns.
+    
+    This is more complex as it needs to detect columns first,
+    then read each column in order.
+    
+    Args:
+        elements: List of elements to order
+        column_threshold: Percentage overlap threshold for column detection (0.0 to 1.0)
+        x_tolerance: Horizontal tolerance for determining column edges
+        
+    Returns:
+        List of elements in reading order
+    """
+    if not elements:
+        return []
+    
+    # 1. Group elements by line
+    lines = group_elements_by_line(elements)
+    
+    # 2. For each line, find the x-coordinate ranges (potential column boundaries)
+    line_x_ranges = []
+    for line in lines:
+        for el in line:
+            x0 = el.get('x0', 0)
+            x1 = el.get('x1', 0)
+            line_x_ranges.append((x0, x1))
+    
+    # If we don't have enough ranges to detect columns, just use basic ordering
+    if len(line_x_ranges) < 3:
+        return _basic_reading_order(elements)
+    
+    # 3. Detect columns by clustering x-coordinate ranges
+    def overlaps(range1, range2, threshold=column_threshold):
+        """Determine if two ranges overlap by more than threshold percentage."""
+        # Calculate overlap
+        overlap_start = max(range1[0], range2[0])
+        overlap_end = min(range1[1], range2[1])
+        overlap = max(0, overlap_end - overlap_start)
+        
+        # Calculate lengths
+        len1 = range1[1] - range1[0]
+        len2 = range2[1] - range2[0]
+        
+        # Calculate overlap as percentage of the shorter range
+        shorter_len = min(len1, len2)
+        if shorter_len == 0:
+            return False
+            
+        return overlap / shorter_len >= threshold
+    
+    # Cluster x-ranges into columns
+    columns = []
+    for x_range in line_x_ranges:
+        # Skip zero-width ranges
+        if x_range[1] - x_range[0] <= 0:
+            continue
+            
+        # Try to find an existing column to add to
+        added = False
+        for col in columns:
+            if any(overlaps(x_range, r) for r in col):
+                col.append(x_range)
+                added = True
+                break
+                
+        # If not added to an existing column, create a new one
+        if not added:
+            columns.append([x_range])
+    
+    # 4. Get column boundaries by averaging x-ranges in each column
+    column_bounds = []
+    for col in columns:
+        left = sum(r[0] for r in col) / len(col)
+        right = sum(r[1] for r in col) / len(col)
+        column_bounds.append((left, right))
+    
+    # Sort columns by x-coordinate (left to right)
+    column_bounds.sort(key=lambda b: b[0])
+    
+    # 5. Assign each element to a column
+    element_columns = {}
+    for el in elements:
+        # Get element x-coordinates
+        el_x0 = el.get('x0', 0)
+        el_x1 = el.get('x1', 0)
+        el_center = (el_x0 + el_x1) / 2
+        
+        # Find the column this element belongs to
+        for i, (left, right) in enumerate(column_bounds):
+            # Extend bounds by tolerance
+            extended_left = left - x_tolerance
+            extended_right = right + x_tolerance
+            
+            # Check if center point is within extended column bounds
+            if extended_left <= el_center <= extended_right:
+                element_columns[el] = i
+                break
+        else:
+            # If no column found, assign to nearest column
+            distances = [(i, min(abs(el_center - left), abs(el_center - right)))
+                         for i, (left, right) in enumerate(column_bounds)]
+            nearest_col = min(distances, key=lambda d: d[1])[0]
+            element_columns[el] = nearest_col
+    
+    # 6. Sort elements by column, then by vertical position
+    sorted_elements = []
+    for col_idx, _ in enumerate(column_bounds):
+        # Get elements in this column
+        col_elements = [el for el in elements if element_columns.get(el) == col_idx]
+        # Sort by top coordinate
+        col_elements.sort(key=lambda e: e.get('top', e.get('y0', 0)))
+        # Add to final list
+        sorted_elements.extend(col_elements)
+    
+    return sorted_elements
+
+
+def _complex_reading_order(elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Complex reading order that accounts for various document structures.
+    
+    This considers columns, text flow around images, tables, etc.
+    
+    Args:
+        elements: List of elements to order
+        
+    Returns:
+        List of elements in reading order
+    """
+    # TODO: Implement complex layout analysis
+    # For now, fall back to column-aware reading order
+    return _column_reading_order(elements)
+
+
+def group_elements_by_line(elements: List[Dict[str, Any]], 
+                          tolerance: float = 3.0) -> List[List[Dict[str, Any]]]:
+    """
+    Group elements into lines based on vertical position.
+    
+    Args:
+        elements: List of elements to group
+        tolerance: Maximum vertical distance for elements to be considered on the same line
+        
+    Returns:
+        List of lists, where each sublist contains elements on the same line
+    """
+    if not elements:
+        return []
+    
+    # Sort by top coordinate
+    sorted_elements = sorted(elements, key=lambda e: e.get('top', e.get('y0', 0)))
+    
+    lines = []
+    current_line = [sorted_elements[0]]
+    current_top = sorted_elements[0].get('top', sorted_elements[0].get('y0', 0))
+    
+    for element in sorted_elements[1:]:
+        element_top = element.get('top', element.get('y0', 0))
+        
+        # If element is close enough to current line's top, add to current line
+        if abs(element_top - current_top) <= tolerance:
+            current_line.append(element)
+        else:
+            # Otherwise, start a new line
+            lines.append(current_line)
+            current_line = [element]
+            current_top = element_top
+    
+    # Add the last line
+    if current_line:
+        lines.append(current_line)
+    
+    # Sort elements within each line by x0
+    for line in lines:
+        line.sort(key=lambda e: e.get('x0', 0))
+    
+    return lines

natural_pdf/utils/visualization.py

@@ -0,0 +1,151 @@
+"""
+Visualization utilities for natural-pdf.
+"""
+from typing import List, Dict, Tuple, Optional, Union, Any
+import io
+import math
+import random
+from PIL import Image, ImageDraw, ImageFont
+
+# Define a list of visually distinct colors for highlighting
+# Format: (R, G, B, alpha)
+HIGHLIGHT_COLORS = [
+    (255, 255, 0, 100),    # Yellow (semi-transparent)
+    (255, 0, 0, 100),      # Red (semi-transparent)
+    (0, 255, 0, 100),      # Green (semi-transparent)
+    (0, 0, 255, 100),      # Blue (semi-transparent)
+    (255, 0, 255, 100),    # Magenta (semi-transparent)
+    (0, 255, 255, 100),    # Cyan (semi-transparent)
+    (255, 165, 0, 100),    # Orange (semi-transparent)
+    (128, 0, 128, 100),    # Purple (semi-transparent)
+    (0, 128, 0, 100),      # Dark Green (semi-transparent)
+    (0, 0, 128, 100),      # Navy (semi-transparent)
+]
+
+# Keep track of the next color to use
+_next_color_index = 0
+
+def get_next_highlight_color() -> Tuple[int, int, int, int]:
+    """
+    Get the next highlight color in the cycle.
+    
+    Returns:
+        Tuple of (R, G, B, alpha) values
+    """
+    global _next_color_index
+    color = HIGHLIGHT_COLORS[_next_color_index % len(HIGHLIGHT_COLORS)]
+    _next_color_index += 1
+    return color
+
+def reset_highlight_colors():
+    """Reset the highlight color cycle."""
+    global _next_color_index
+    _next_color_index = 0
+
+def get_random_highlight_color() -> Tuple[int, int, int, int]:
+    """
+    Get a random highlight color.
+    
+    Returns:
+        Tuple of (R, G, B, alpha) values
+    """
+    return random.choice(HIGHLIGHT_COLORS)
+
+def create_legend(labels_colors: Dict[str, Tuple[int, int, int, int]], 
+                 width: int = 200, 
+                 item_height: int = 30) -> Image.Image:
+    """
+    Create a legend image for the highlighted elements.
+    
+    Args:
+        labels_colors: Dictionary mapping labels to colors
+        width: Width of the legend image
+        item_height: Height of each legend item
+        
+    Returns:
+        PIL Image with the legend
+    """
+    # Calculate the height based on the number of labels
+    height = len(labels_colors) * item_height + 10  # 10px padding
+    
+    # Create a white image
+    legend = Image.new('RGBA', (width, height), (255, 255, 255, 255))
+    draw = ImageDraw.Draw(legend)
+    
+    # Try to load a font, use default if not available
+    try:
+        font = ImageFont.truetype("Arial", 12)
+    except IOError:
+        font = ImageFont.load_default()
+    
+    # Draw each legend item
+    y = 5  # Start with 5px padding
+    for label, color in labels_colors.items():
+        # Get the color components
+        r, g, b, alpha = color
+        
+        # Calculate the apparent color when drawn on white background
+        # Alpha blending formula: result = (source * alpha) + (dest * (1-alpha))
+        # Where alpha is normalized to 0-1 range
+        alpha_norm = alpha / 255.0
+        apparent_r = int(r * alpha_norm + 255 * (1 - alpha_norm))
+        apparent_g = int(g * alpha_norm + 255 * (1 - alpha_norm))
+        apparent_b = int(b * alpha_norm + 255 * (1 - alpha_norm))
+        
+        # Use solid color that matches the apparent color of the semi-transparent highlight
+        legend_color = (apparent_r, apparent_g, apparent_b, 255)
+        
+        # Draw the color box
+        draw.rectangle([(10, y), (30, y + item_height - 5)], fill=legend_color)
+        
+        # Draw the label text
+        draw.text((40, y + item_height // 4), label, fill=(0, 0, 0, 255), font=font)
+        
+        # Move to the next position
+        y += item_height
+    
+    return legend
+
+def merge_images_with_legend(image: Image.Image, 
+                            legend: Image.Image, 
+                            position: str = 'right') -> Image.Image:
+    """
+    Merge an image with a legend.
+    
+    Args:
+        image: Main image
+        legend: Legend image
+        position: Position of the legend ('right', 'bottom', 'top', 'left')
+        
+    Returns:
+        Merged image
+    """
+    if position == 'right':
+        # Create a new image with extra width for the legend
+        merged = Image.new('RGBA', (image.width + legend.width, max(image.height, legend.height)), 
+                          (255, 255, 255, 255))
+        merged.paste(image, (0, 0))
+        merged.paste(legend, (image.width, 0))
+    elif position == 'bottom':
+        # Create a new image with extra height for the legend
+        merged = Image.new('RGBA', (max(image.width, legend.width), image.height + legend.height), 
+                          (255, 255, 255, 255))
+        merged.paste(image, (0, 0))
+        merged.paste(legend, (0, image.height))
+    elif position == 'top':
+        # Create a new image with extra height for the legend
+        merged = Image.new('RGBA', (max(image.width, legend.width), image.height + legend.height),
+                          (255, 255, 255, 255))
+        merged.paste(legend, (0, 0))
+        merged.paste(image, (0, legend.height))
+    elif position == 'left':
+        # Create a new image with extra width for the legend
+        merged = Image.new('RGBA', (image.width + legend.width, max(image.height, legend.height)),
+                          (255, 255, 255, 255))
+        merged.paste(legend, (0, 0))
+        merged.paste(image, (legend.width, 0))
+    else:
+        # Invalid position, return the original image
+        merged = image
+    
+    return merged

natural_pdf-25.3.16.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023-2025 Jonathan Soma
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

natural_pdf-25.3.16.dist-info/METADATA

@@ -0,0 +1,268 @@
+Metadata-Version: 2.2
+Name: natural-pdf
+Version: 25.3.16
+Summary: A more intuitive interface for working with PDFs
+Home-page: https://github.com/jsoma/natural-pdf
+Author: Jonathan Soma
+Author-email: jonathan.soma@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pdfplumber>=0.7.0
+Requires-Dist: Pillow>=8.0.0
+Requires-Dist: colour>=0.1.5
+Requires-Dist: numpy>=1.20.0
+Requires-Dist: doclayout_yolo>=0.0.3
+Requires-Dist: torch>=2.0.0
+Requires-Dist: torchvision>=0.15.0
+Requires-Dist: transformers>=4.30.0
+Requires-Dist: huggingface_hub>=0.19.0
+Provides-Extra: easyocr
+Requires-Dist: easyocr>=1.7.0; extra == "easyocr"
+Provides-Extra: paddle
+Requires-Dist: paddlepaddle>=2.5.0; extra == "paddle"
+Requires-Dist: paddleocr>=2.7.0; extra == "paddle"
+Provides-Extra: qa
+Provides-Extra: core
+Requires-Dist: pdfplumber>=0.7.0; extra == "core"
+Requires-Dist: Pillow>=8.0.0; extra == "core"
+Requires-Dist: colour>=0.1.5; extra == "core"
+Requires-Dist: numpy>=1.20.0; extra == "core"
+Provides-Extra: ai
+Requires-Dist: doclayout_yolo>=0.0.3; extra == "ai"
+Requires-Dist: torch>=2.0.0; extra == "ai"
+Requires-Dist: torchvision>=0.15.0; extra == "ai"
+Requires-Dist: transformers>=4.30.0; extra == "ai"
+Requires-Dist: huggingface_hub>=0.19.0; extra == "ai"
+Provides-Extra: all
+Requires-Dist: easyocr>=1.7.0; extra == "all"
+Requires-Dist: paddlepaddle>=2.5.0; extra == "all"
+Requires-Dist: paddleocr>=2.7.0; extra == "all"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Natural PDF
+
+A friendly library for working with PDFs, built on top of [pdfplumber](https://github.com/jsvine/pdfplumber).
+
+Natural PDF lets you find and extract content from PDFs using simple code that makes sense.
+
+[Complete documentation here](https://jsoma.github.io/natural-pdf)
+
+## Features
+
+- **Fluent API** for chaining operations
+- **CSS-like selectors** for finding elements
+- **Spatial navigation** with intuitive methods like `above()`, `below()`, and `select_until()`
+- **Element collections** for batch operations
+- **Visual highlighting** for debugging
+- **Region visualization** with direct image extraction of specific regions
+- **Text style analysis** for document structure
+- **Exclusion zones** for headers, footers, and other areas to ignore
+- **OCR integration** for extracting text from scanned documents
+- **Document layout analysis** for detecting document structure with ML models
+- **Table extraction** with multiple detection methods
+- **Structured logging** with configurable levels and handlers
+
+## Installation
+
+```bash
+pip install natural-pdf
+```
+
+or if you're picky...
+
+```bash
+# Minimal installation without AI models (faster, smaller)
+pip install natural-pdf[core]
+
+# With all OCR engines
+pip install natural-pdf[easyocr,paddle]
+```
+
+## Quick Start
+
+```python
+from natural_pdf import PDF
+
+# Open a PDF
+pdf = PDF('document.pdf')
+
+# Get the first page
+page = pdf.pages[0]
+
+# Find elements using CSS-like selectors
+heading = page.find('text:contains("Summary"):bold')
+
+# Extract content below the heading
+content = heading.below().extract_text()
+print(content)
+
+# Exclude headers and footers
+page.add_exclusion(page.find('text:contains("CONFIDENTIAL")').above())
+page.add_exclusion(page.find_all('line')[-1].below())
+
+# Extract clean text
+clean_text = page.extract_text()
+print(clean_text)
+```
+
+## Selectors
+
+The library supports CSS-like selectors for finding elements:
+
+```python
+# Find text containing a specific string
+element = page.find('text:contains("Revenue")')
+
+# Find bold text with a specific font size
+headings = page.find_all('text[size>=12]:bold')
+
+# Find thick red lines
+lines = page.find_all('line[width>=2][color~=(1,0,0)]')
+```
+
+## Spatial Navigation
+
+Navigate through the document with intuitive spatial methods:
+
+```python
+# Get content below a heading
+heading = page.find('text:contains("Introduction")')
+content = heading.below().extract_text()
+
+# Get content from one element to another
+start = page.find('text:contains("Start")')
+end = page.find('text:contains("End")')
+region = start.select_until(end)
+content = region.extract_text()
+```
+
+## Exclusion Zones
+
+Exclude headers, footers, or other areas from extraction:
+
+```python
+# Page-level exclusion
+page.add_exclusion(page.find('text:contains("Page")').above())
+page.add_exclusion(page.find_all('line')[-1].below())
+
+# PDF-level exclusion with lambdas
+pdf.add_exclusion(
+    lambda page: page.find('text:contains("Header")').above(),
+    label="headers"
+)
+
+# Extract text with exclusions applied
+text = pdf.extract_text()
+
+# Extract from a specific region with exclusions
+summary = page.find('text:contains("Summary")')
+conclusion = page.find('text:contains("Conclusion")')
+region = page.create_region(summary.x0, summary.top, conclusion.x1, conclusion.bottom)
+region_text = region.extract_text(apply_exclusions=True)  # Excludes headers/footers
+
+# Disable exclusions for a specific extraction
+full_text = page.extract_text(apply_exclusions=False)
+```
+
+Exclusions work efficiently with different region types:
+- Regions without intersection with exclusion zones → exclusions ignored entirely
+- Rectangular regions with header/footer exclusions → optimized cropping
+- Complex regions with partial exclusions → advanced filtering with warning
+
+## OCR Integration
+
+Extract text from scanned documents using OCR with multiple engine options:
+
+```python
+# Using the default EasyOCR engine
+pdf = PDF('scanned_document.pdf', ocr={
+    'enabled': 'auto',  # Only use OCR when necessary
+    'languages': ['en'],
+    'min_confidence': 0.5
+})
+
+# Using PaddleOCR for better Asian language support
+pdf = PDF('scanned_document.pdf', 
+          ocr_engine='paddleocr',
+          ocr={
+              'enabled': True,
+              'languages': ['zh-cn', 'en'],  # Chinese and English
+              'min_confidence': 0.3,
+              'model_settings': {
+                  'use_angle_cls': False,  # PaddleOCR-specific setting
+                  'rec_batch_num': 6
+              }
+          })
+
+# Extract text, OCR will be used if needed
+text = page.extract_text()
+
+# Force OCR regardless of existing text
+ocr_text = page.extract_text(ocr=True)
+
+# Find OCR-detected text with high confidence
+high_confidence = page.find_all('text[source=ocr][confidence>=0.8]')
+
+# Visualize OCR results with color-coded confidence levels
+for elem in page.find_all('text[source=ocr]'):
+    if elem.confidence >= 0.8:
+        color = (0, 1, 0, 0.3)  # Green for high confidence
+    elif elem.confidence >= 0.5:
+        color = (1, 1, 0, 0.3)  # Yellow for medium confidence
+    else:
+        color = (1, 0, 0, 0.3)  # Red for low confidence
+        
+    elem.highlight(color=color, label=f"OCR ({elem.confidence:.2f})")
+page.save_image('ocr_results.png', labels=True)
+```
+
+## Logging
+
+The library includes a structured logging system to provide visibility into its operations:
+
+```python
+import logging
+from natural_pdf import PDF, configure_logging
+
+# Configure logging with INFO level to console
+configure_logging(level=logging.INFO)
+
+# Or log to a file with DEBUG level
+file_handler = logging.FileHandler("natural_pdf.log")
+file_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+configure_logging(level=logging.DEBUG, handler=file_handler)
+
+# Now operations will generate logs
+pdf = PDF("document.pdf")
+# Log: natural_pdf.core.pdf - INFO - Initializing PDF from document.pdf
+
+# Run layout detection with verbose logging
+regions = pdf.pages[0].analyze_layout(
+    model="paddle",
+    model_params={"verbose": True}
+)
+# Log: natural_pdf.analyzers.layout.paddle - INFO - Starting PaddleLayout detection...
+# Log: natural_pdf.analyzers.layout.paddle - DEBUG - Parameters: confidence=0.2...
+```
+
+Logs follow a hierarchical structure matching the library's module organization:
+- `natural_pdf.core` - Core PDF operations
+- `natural_pdf.analyzers` - Layout analysis operations
+- `natural_pdf.ocr` - OCR engine operations
+
+## More details
+
+[Complete documentation here](https://jsoma.github.io/natural-pdf)