natural-pdf 25.3.16__py3-none-any.whl

examples/table_structure_detection.py

@@ -0,0 +1,155 @@
+"""
+Table structure detection example using Table Transformer.
+
+This example demonstrates how to use the Table Transformer (TATR)
+to detect tables and their structure in PDF documents.
+
+Note: This example requires additional dependencies:
+    - torch
+    - torchvision
+    - transformers
+    
+These will be automatically installed when you install natural-pdf.
+"""
+import os
+from natural_pdf import PDF
+
+# Get the current directory of this script
+script_dir = os.path.dirname(os.path.realpath(__file__))
+# Get the parent directory (project root)
+root_dir = os.path.dirname(script_dir)
+# Setup paths
+pdf_path = os.path.join(root_dir, "pdfs", "01-practice.pdf")
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+print(f"Analyzing table structure in: {pdf_path}")
+
+# Load the PDF - this file has a single page with a table
+pdf = PDF(pdf_path)
+page = pdf.pages[0]  # Get the first page
+
+print("Running YOLO layout analysis first (excluding tables)...")
+# First run YOLO detector but exclude tables
+page.analyze_layout(
+    model="yolo",
+    confidence=0.3,
+    exclude_classes=["table", "table_caption", "table_footnote"]
+)
+
+print(f"Found {len(page.detected_layout_regions)} general layout regions")
+
+print("Now running Table Transformer detection...")
+# Then run Table Transformer detection and add to existing regions
+page.analyze_layout(
+    model="tatr",
+    confidence=0.4,  # Table detection confidence threshold
+    existing="append"
+)
+
+print(f"Found {len(page.detected_layout_regions)} total regions (including table structure)")
+
+# Example of method chaining
+print("\nDemonstrating method chaining for layout analysis and highlighting:")
+# Create a highlighted image with a single method chain
+page.clear_highlights()\
+    .analyze_layout(model="tatr", confidence=0.3)\
+    .highlight_layout()\
+    .to_image(path=os.path.join(output_dir, "chained_analysis.png"), show_labels=True)
+print("Created highlighted image with method chaining")
+
+# Group regions by type and model
+regions_by_type = {}
+for region in page.detected_layout_regions:
+    region_type = region.region_type
+    if region_type not in regions_by_type:
+        regions_by_type[region_type] = []
+    regions_by_type[region_type].append(region)
+
+# Print a summary of all detected regions by type
+print("\nAll detected regions:")
+for region_type, type_regions in regions_by_type.items():
+    model_name = type_regions[0].model if hasattr(type_regions[0], 'model') else "unknown"
+    print(f"  - {region_type} ({model_name}): {len(type_regions)} regions")
+
+# Highlight all regions using method chaining
+output_path = os.path.join(output_dir, "all_detected_regions.png")
+page.clear_highlights()\
+    .highlight_layout()\
+    .to_image(path=output_path, show_labels=True)
+print(f"\nSaved combined layout visualization to {output_path}")
+
+# Highlight only YOLO regions using selector and chaining
+output_path = os.path.join(output_dir, "yolo_regions.png")
+page.clear_highlights()\
+    .find_all('region[model=yolo]')\
+    .highlight(label="YOLO Regions")
+page.to_image(path=output_path, show_labels=True)
+print(f"Saved YOLO layout visualization to {output_path}")
+
+# Highlight only Table Transformer regions using selector and chaining
+output_path = os.path.join(output_dir, "table_structure.png")
+page.clear_highlights()\
+    .find_all('region[model=tatr]')\
+    .highlight(label="Table Structure")
+page.to_image(path=output_path, show_labels=True)
+print(f"Saved Table Transformer visualization to {output_path}")
+
+# Find tables and process their content
+tables = page.find_all('region[type=table]')
+if tables:
+    print(f"\nFound {len(tables)} tables")
+    
+    # Get the first table
+    table = tables[0]
+    print(f"Table details:")
+    print(f"  Confidence: {table.confidence:.2f}")
+    print(f"  Bounding box: {table.bbox}")
+    
+    # Find rows, columns, and headers within this table
+    # Note: Original class names with spaces are converted to hyphenated format in selectors
+    rows = page.find_all('region[type=table-row]')
+    columns = page.find_all('region[type=table-column]')
+    headers = page.find_all('region[type=table-column-header]')
+    
+    print(f"  Structure: {len(rows)} rows, {len(columns)} columns, {len(headers)} headers")
+    
+    # Extract text from the table
+    table_text = table.extract_text()
+    print(f"  Content preview: {table_text[:150]}..." if len(table_text) > 150 else table_text)
+    
+    # Highlight the table structure with distinct colors
+    page.clear_highlights()
+    
+    # First highlight the table
+    table.highlight(label="Table", color=(1, 0, 0, 0.3))
+    
+    # Then highlight the structure elements
+    for row in rows:
+        row.highlight(label="Row", color=(0, 1, 0, 0.3))
+    for column in columns:
+        column.highlight(label="Column", color=(0, 0, 1, 0.3))
+    for header in headers:
+        header.highlight(label="Header", color=(0, 1, 1, 0.3))
+        
+    # Save the highlighted table structure
+    output_path = os.path.join(output_dir, "table_structure_detail.png")
+    page.to_image(path=output_path, show_labels=True)
+    print(f"  Saved detailed table structure visualization to {output_path}")
+    
+    # Now find text elements within the table
+    print("\nExtracting text from table cells:")
+    table_text_elements = table.find_all('text')
+    print(f"  Found {len(table_text_elements)} text elements in the table")
+    
+    # Show the first few text elements
+    for i, elem in enumerate(table_text_elements[:5]):
+        print(f"  Text {i+1}: '{elem.text}'")
+    
+    # You can also extract text just from table headers
+    if headers:
+        header = headers[0]
+        header_text = header.extract_text()
+        print(f"\nHeader text: {header_text}")
+else:
+    print("\nNo tables detected on this page")

examples/tatr_cells_test.py

@@ -0,0 +1,56 @@
+import os
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+
+# Get absolute path for the PDF
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+pdf_path = os.path.join(root_dir, "pdfs", "HARRY ROQUE_redacted.pdf")
+
+print(f"Loading PDF: {pdf_path}")
+pdf = PDF(pdf_path)
+
+# Create output directory if it doesn't exist
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+# Use a specific page
+page = pdf.pages[6]
+
+# Test 1: Analyze layout with create_cells=True
+print("\n-- Testing layout detection with cell creation --")
+regions = page.analyze_layout(model='tatr', create_cells=True)
+
+# Count tables and cells
+tables = page.find_all('region[type=table][model=tatr]')
+cells = page.find_all('region[type=table-cell][model=tatr]')
+
+print(f"Found {len(tables)} tables")
+print(f"Found {len(cells)} table cells")
+
+# Test 2: Create cells explicitly from a table
+if tables:
+    print("\n-- Testing explicit cell creation from a table --")
+    table = tables[0]
+    # Create cells if not already created
+    explicit_cells = table.create_cells()
+    print(f"Created {len(explicit_cells)} cells explicitly")
+    
+    # Highlight the first few cells
+    for i, cell in enumerate(explicit_cells[:5]):
+        cell.highlight(label=f"Cell {i+1}", color=(255, 0, 0, 50))
+    
+    # Highlight the table
+    table.highlight(label="Table", color=(0, 0, 255, 50))
+
+# Save the highlighted image
+output_path = os.path.join(output_dir, "tatr_cells_test.png")
+print(f"\nSaving highlighted image to: {output_path}")
+page.to_image(path=output_path, show_labels=True)
+
+print("\nTest completed successfully!")

examples/tatr_ocr_table_test.py

@@ -0,0 +1,94 @@
+import os
+import sys
+from pathlib import Path
+import pandas as pd
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+
+# Get absolute path for the PDF
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+pdf_path = os.path.join(root_dir, "pdfs", "HARRY ROQUE_redacted.pdf")
+
+print(f"Loading PDF: {pdf_path}")
+pdf = PDF(pdf_path)
+
+# Create output directory if it doesn't exist
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+# Use a specific page
+page = pdf.pages[3]  # Try page 3 (this should be correct - pages are indexed from 0)
+
+# Run document layout analysis to find tables
+print("\n-- Running layout analysis to find tables --")
+regions = page.analyze_layout(model='tatr')
+
+# Find the first table
+table = page.find('region[type=table][model=tatr]')
+if not table:
+    print("No tables found.")
+    sys.exit(1)
+
+print(f"Found table at coordinates: {table.bbox}")
+
+# Find table structure elements
+rows = page.find_all(f'region[type=table-row][model=tatr]')
+columns = page.find_all(f'region[type=table-column][model=tatr]')
+headers = page.find_all(f'region[type=table-column-header][model=tatr]')
+
+# Filter to elements that are part of this table
+def is_in_table(region, table):
+    region_center_x = (region.x0 + region.x1) / 2
+    region_center_y = (region.top + region.bottom) / 2
+    return (table.x0 <= region_center_x <= table.x1 and
+            table.top <= region_center_y <= table.bottom)
+
+table_rows = [r for r in rows if is_in_table(r, table)]
+table_columns = [c for c in columns if is_in_table(c, table)]
+table_headers = [h for h in headers if is_in_table(h, table)]
+
+# Print structure info
+print(f"Table has {len(table_rows)} rows, {len(table_columns)} columns, and {len(table_headers)} headers")
+
+# Create cells and check OCR on some of them
+cells = table.create_cells()
+print(f"Created {len(cells)} cells")
+
+# Try OCR on a few individual cells to debug
+print("\n-- Testing OCR on individual cells --")
+if cells:
+    sample_cells = cells[:50]  # First 50 cells
+    
+    for i, cell in enumerate(sample_cells):
+        # print(f"Cell {i+1}:", cell.bbox)
+        
+        # Try OCR with very low confidence
+        ocr_config = {
+            "enabled": True,
+            "min_confidence": 0.01,
+            "detection_params": {
+                "text_threshold": 0.001,  # Lower threshold to detect more text (default is 0.7)
+                "mag_ratio": 4.0,  # Double the magnification during detectio
+                "link_threshold": 1
+            },
+            "recognition_params": {
+                "min_size": 6
+            }
+        }
+        
+        ocr_elements = cell.apply_ocr(**ocr_config)
+        if ocr_elements:
+            print(f"  OCR detected {len(ocr_elements)} text elements:")
+            for elem in ocr_elements:
+                print(f"    '{elem.text}' (conf: {elem.confidence:.2f})")
+            
+        # Get regular text
+        text = cell.extract_text().strip()
+        if text:
+            print(f"  Regular extraction: '{text}'")
+
+print("\nTest completed successfully!")

examples/text_search_example.py

@@ -0,0 +1,122 @@
+"""
+Example demonstrating enhanced text search capabilities in Natural PDF.
+
+This showcases:
+1. Multi-word searching with keep_spaces enabled (default)
+2. Case-insensitive searching
+3. Regular expression searching
+4. Turning off keep_spaces to see the difference
+"""
+
+import os
+import sys
+import argparse
+from pathlib import Path
+
+# Add parent directory to path for running the example
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from natural_pdf import PDF, configure_logging
+import logging
+
+
+def main(pdf_path=None):
+    # Use a default PDF if none provided
+    if not pdf_path:
+        pdf_path = os.path.join(os.path.dirname(__file__), '..', 'pdfs', '2019 Statistics.pdf')
+        
+    print(f"Using PDF: {pdf_path}")
+    print("-" * 50)
+    
+    # Create PDF with default settings (keep_spaces=True)
+    pdf = PDF(pdf_path)
+    page = pdf.pages[0]
+    
+    # Display basic page info
+    print(f"Page dimensions: {page.width} x {page.height}")
+    
+    # 1. Basic multi-word search with default keep_spaces=True
+    print("\nMulti-word search with keep_spaces=True (default):")
+    print("-" * 50)
+    
+    # Search for a multi-word phrase
+    results = page.find_all('text:contains("annual report")', case=False)
+    print(f"Found {len(results)} results for 'annual report' (case-insensitive)")
+    for i, result in enumerate(results):
+        print(f"  Result {i+1}: '{result.text}'")
+        # Highlight the results
+        result.highlight(label=f"Match {i+1}: 'annual report'", color=(1, 0.7, 0, 0.3))
+    
+    # 2. Case-sensitive search
+    print("\nCase-sensitive search:")
+    print("-" * 50)
+    
+    # Search with case sensitivity
+    results = page.find_all('text:contains("Annual Report")', case=True)
+    print(f"Found {len(results)} results for 'Annual Report' (case-sensitive)")
+    for i, result in enumerate(results):
+        print(f"  Result {i+1}: '{result.text}'")
+        # Highlight with a different color
+        result.highlight(label=f"Match {i+1}: 'Annual Report'", color=(0, 0.7, 1, 0.3))
+    
+    # 3. Regular expression search
+    print("\nRegular expression search:")
+    print("-" * 50)
+    
+    # Use regex to find patterns
+    pattern = "report\\s+\\d{4}"  # "report" followed by whitespace and 4 digits
+    results = page.find_all(f'text:contains("{pattern}")', regex=True, case=False)
+    print(f"Found {len(results)} results for regex pattern '{pattern}'")
+    for i, result in enumerate(results):
+        print(f"  Result {i+1}: '{result.text}'")
+        # Highlight with another color
+        result.highlight(label=f"Match {i+1}: regex '{pattern}'", color=(0, 1, 0, 0.3))
+    
+    # Save highlighted page as an image
+    output_path = os.path.join(os.path.dirname(__file__), '..', 'output', 'text_search_results.png')
+    page.save_image(output_path, labels=True)
+    print(f"\nSaved highlighted results to: {output_path}")
+    
+    # 4. Create a new PDF with keep_spaces=False to compare
+    print("\nComparing with keep_spaces=False (legacy behavior):")
+    print("-" * 50)
+    
+    # Create a new PDF with keep_spaces=False
+    pdf_legacy = PDF(pdf_path, keep_spaces=False)
+    page_legacy = pdf_legacy.pages[0]
+    
+    # Try the same multi-word search
+    results_legacy = page_legacy.find_all('text:contains("annual report")', case=False)
+    print(f"Found {len(results_legacy)} results for 'annual report' (case-insensitive)")
+    
+    # Try regex to find occurrences in separate words
+    pattern = "annual\\s+report"  # "annual" followed by whitespace and "report"
+    regex_results = page_legacy.find_all(f'text:contains("{pattern}")', regex=True, case=False)
+    print(f"With regex '{pattern}': Found {len(regex_results)} results")
+    
+    # Show conclusion
+    print("\nConclusion:")
+    print("-" * 50)
+    print("1. With keep_spaces=True (default):")
+    print("   - Multi-word phrases can be found directly with :contains()")
+    print("   - Text maintains its natural spacing within word elements")
+    print("\n2. With keep_spaces=False (legacy):")
+    print("   - Words are split at spaces, making multi-word search less effective")
+    print("   - Regular expressions with \\s patterns can help bridge words")
+    
+    return pdf
+
+
+if __name__ == "__main__":
+    # Set up command line arguments
+    parser = argparse.ArgumentParser(description="Demonstrate Natural PDF's enhanced text search capabilities")
+    parser.add_argument("--pdf", help="Path to a PDF file to analyze")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose logging")
+    args = parser.parse_args()
+    
+    # Configure logging
+    log_level = logging.DEBUG if args.verbose else logging.INFO
+    configure_logging(level=log_level)
+    
+    # Run the example
+    pdf = main(args.pdf)

examples/text_style_example.py

@@ -0,0 +1,110 @@
+"""
+Example demonstrating the text style analysis feature of natural-pdf.
+"""
+import os
+import sys
+
+# Add the parent directory to the path to import the package
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from natural_pdf import PDF
+
+def text_style_example(pdf_path):
+    """Demonstrates the text style analysis feature."""
+    # Open the PDF
+    with PDF(pdf_path) as pdf:
+        page = pdf.pages[0]
+        
+        print(f"PDF loaded: {pdf_path}")
+        print(f"PDF has {len(pdf)} pages")
+        
+        # Create an output directory for saving images
+        output_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'output'))
+        os.makedirs(output_dir, exist_ok=True)
+        
+        # EXAMPLE 1: Analyze text styles
+        print("\nEXAMPLE 1: Analyzing text styles")
+        print("-" * 60)
+        
+        # Analyze the styles
+        styles = page.analyze_text_styles()
+        
+        # Display what was found
+        print("Text style analysis results:")
+        for label, elements in styles.items():
+            print(f"- {label}: {len(elements)} elements")
+            
+            # Show a sample of each style
+            if len(elements) > 0:
+                sample = elements[0]
+                # Get style properties
+                size = getattr(sample, 'size', 'N/A')
+                font = getattr(sample, 'fontname', 'N/A')
+                
+                # Determine if bold/italic based on font name
+                is_bold = False
+                is_italic = False
+                if hasattr(sample, 'fontname') and sample.fontname:
+                    font_lower = sample.fontname.lower()
+                    is_bold = ('bold' in font_lower or 'black' in font_lower or 
+                               sample.fontname.endswith('-B'))
+                    is_italic = ('italic' in font_lower or 'oblique' in font_lower or 
+                                 sample.fontname.endswith('-I'))
+                
+                style_desc = []
+                if is_bold:
+                    style_desc.append("bold")
+                if is_italic:
+                    style_desc.append("italic")
+                    
+                style_text = ", ".join(style_desc) if style_desc else "regular"
+                
+                print(f"  Sample: '{sample.text}' (size={size}, {style_text}, font={font})")
+                
+        # EXAMPLE 2: Visualize text styles with highlighting
+        print("\nEXAMPLE 2: Visualizing text styles")
+        print("-" * 60)
+        
+        # Highlight the styles
+        page.highlight_text_styles()
+        
+        # Save the image with a legend
+        output_file = os.path.join(output_dir, "text_styles.png")
+        page.to_image(path=output_file, show_labels=True)
+        print(f"Saved text style visualization to: {output_file}")
+        
+        # Clear highlights for the next example
+        page.clear_highlights()
+        
+        # EXAMPLE 3: Using highlight_all with text styles
+        print("\nEXAMPLE 3: Using highlight_all with text styles")
+        print("-" * 60)
+        
+        # Highlight all elements including text styles
+        page.highlight_all(include_text_styles=True)
+        
+        # Save the image with a legend
+        output_file = os.path.join(output_dir, "highlight_all_styles.png")
+        page.to_image(path=output_file, show_labels=True)
+        print(f"Saved highlight_all with text styles to: {output_file}")
+        
+        print("\nEnd of text style demonstration.")
+
+if __name__ == "__main__":
+    # Default to example PDF if no path is provided
+    if len(sys.argv) < 2:
+        # Use the example PDF in the pdfs directory
+        pdf_path = os.path.abspath(os.path.join(
+            os.path.dirname(__file__), '..', 'pdfs', '01-practice.pdf'))
+        if not os.path.exists(pdf_path):
+            print("Example PDF not found. Please provide a path to a PDF file.")
+            print("Usage: python text_style_example.py [path/to/file.pdf]")
+            sys.exit(1)
+    else:
+        pdf_path = sys.argv[1]
+        # Check if the file exists
+        if not os.path.exists(pdf_path):
+            print(f"File not found: {pdf_path}")
+            sys.exit(1)
+    
+    text_style_example(pdf_path)

examples/tiny-text.py

@@ -0,0 +1,61 @@
+import os
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+
+# Get absolute path for the PDF
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+pdf_path = os.path.join(root_dir, "pdfs", "HARRY ROQUE_redacted.pdf")
+
+print(f"Loading PDF: {pdf_path}")
+pdf = PDF(pdf_path, ocr={
+    "enabled": True,
+    "engine": "easyocr",
+    "languages": ["en"],
+    "detection_params": {
+        "text_threshold": 0.001,
+        "mag_ratio": 3.0,  # Quadruple the magnification during detection
+        "canvas_size": 5000,
+    },
+    "recognition_params": {
+        "min_size": 4,
+        "contrast_ths": 0.5
+    }
+})
+
+# Create output directory if it doesn't exist
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+# Use a specific page
+page = pdf.pages[6]
+# Run document layout analysis
+regions = page.analyze_layout(model='tatr')
+
+print(f"Found {len(regions)} regions")
+
+# # Apply OCR explicitly
+# print("Applying OCR...")
+ocr_elements = page.apply_ocr()
+print(f"Found {len(ocr_elements)} OCR elements")
+
+# Print some sample elements
+print("\nSample OCR elements:")
+for i, elem in enumerate(ocr_elements[:30]):
+    print(f"{i+1}. {elem}")
+
+# Highlight the OCR text elements
+print("\nHighlighting OCR elements...")
+for elem in ocr_elements:
+    elem.highlight(label=f"OCR ({elem.confidence:.2f})")
+
+output_path = os.path.join(output_dir, "ocr_highlight_all_test.png")
+print(f"Saving highlight_all image to: {output_path}")
+page.to_image(path=output_path, show_labels=True)
+
+print("\nTest completed successfully!")