natural-pdf 25.3.16__py3-none-any.whl

examples/ocr_default_test.py

@@ -0,0 +1,97 @@
+"""
+Test to ensure OCR is disabled by default.
+"""
+import os
+import sys
+
+# Add the parent directory to the path to import the package
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from natural_pdf import PDF
+
+def test_ocr_default():
+    """Test that OCR is disabled by default but can be enabled explicitly."""
+    # Use the scanned PDF for testing OCR
+    pdf_path = os.path.abspath(os.path.join(
+        os.path.dirname(__file__), '..', 'pdfs', 'needs-ocr.pdf'))
+    
+    if not os.path.exists(pdf_path):
+        # Fall back to a different PDF
+        pdf_path = os.path.abspath(os.path.join(
+            os.path.dirname(__file__), '..', 'pdfs', 'HARRY ROQUE_redacted.pdf'))
+        
+    if not os.path.exists(pdf_path):
+        print("No suitable PDF file found for OCR testing. Please provide a scanned PDF file.")
+        return
+    
+    print(f"Testing with PDF: {pdf_path}")
+    
+    # Test 1: OCR should be OFF by default
+    print("\nTEST 1: Default Behavior (OCR should be OFF)")
+    print("-" * 60)
+    
+    with PDF(pdf_path) as pdf:
+        # Print initial OCR config
+        print(f"Initial OCR config: {pdf._ocr_config}")
+        print(f"OCR enabled? {pdf._ocr_config.get('enabled', False)}")
+        
+        # Extract text without OCR
+        page = pdf.pages[0]
+        text = page.extract_text()
+        
+        print(f"Extracted {len(text)} characters without explicit OCR")
+        print(f"First 100 chars: {text[:100]}...")
+    
+    # Test 2: Explicit OCR enable via constructor
+    print("\nTEST 2: Explicit OCR Enable via Constructor")
+    print("-" * 60)
+    
+    with PDF(pdf_path, ocr=True) as pdf:
+        # Print OCR config
+        print(f"OCR config: {pdf._ocr_config}")
+        print(f"OCR enabled? {pdf._ocr_config.get('enabled', False)}")
+        
+        # Extract text with OCR
+        page = pdf.pages[0]
+        text = page.extract_text()
+        
+        print(f"Extracted {len(text)} characters with OCR enabled in constructor")
+        print(f"First 100 chars: {text[:100]}...")
+    
+    # Test 3: Explicit OCR enable via extract_text parameter
+    print("\nTEST 3: Explicit OCR Enable via extract_text parameter")
+    print("-" * 60)
+    
+    with PDF(pdf_path) as pdf:
+        # Print initial OCR config
+        print(f"Initial OCR config: {pdf._ocr_config}")
+        print(f"OCR enabled? {pdf._ocr_config.get('enabled', False)}")
+        
+        # Extract text with OCR parameter
+        page = pdf.pages[0]
+        text = page.extract_text(ocr=True)
+        
+        print(f"Extracted {len(text)} characters with OCR enabled in extract_text")
+        print(f"First 100 chars: {text[:100]}...")
+    
+    # Test 4: OCR via with_ocr builder
+    print("\nTEST 4: OCR via with_ocr builder")
+    print("-" * 60)
+    
+    with PDF(pdf_path) as pdf:
+        # Configure OCR with builder
+        pdf.with_ocr(enabled=True, languages=["en"])
+        
+        # Print updated OCR config
+        print(f"Updated OCR config: {pdf._ocr_config}")
+        print(f"OCR enabled? {pdf._ocr_config.get('enabled', False)}")
+        
+        # Extract text with OCR configured via builder
+        page = pdf.pages[0]
+        text = page.extract_text()
+        
+        print(f"Extracted {len(text)} characters with OCR enabled via builder")
+        print(f"First 100 chars: {text[:100]}...")
+
+if __name__ == "__main__":
+    test_ocr_default()

examples/ocr_engine_comparison.py

@@ -0,0 +1,235 @@
+"""
+OCR Engine Comparison Example.
+
+This example compares the performance of different OCR engines with natural-pdf.
+
+Requires both EasyOCR and PaddleOCR to be installed:
+pip install easyocr
+pip install paddlepaddle paddleocr
+"""
+import os
+import sys
+import time
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+from natural_pdf.ocr import EasyOCREngine, PaddleOCREngine
+
+# Get the current directory of this script
+script_dir = os.path.dirname(os.path.realpath(__file__))
+# Get the parent directory (project root)
+root_dir = os.path.dirname(script_dir)
+# Default PDF path (replace with a scanned document path for better results)
+default_pdf = os.path.join(root_dir, "pdfs", "HARRY ROQUE_redacted.pdf")
+# Output directory
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+print("OCR Engine Comparison")
+print("====================")
+
+# Check if both OCR engines are available
+easyocr_available = False
+paddleocr_available = False
+
+try:
+    import easyocr
+    easyocr_available = True
+    print("EasyOCR is available.")
+except ImportError:
+    print("EasyOCR is not available. Some comparisons will be skipped.")
+
+try:
+    import paddleocr
+    import paddle
+    paddleocr_available = True
+    print("PaddleOCR is available.")
+except ImportError:
+    print("PaddleOCR is not available. Some comparisons will be skipped.")
+
+if not easyocr_available and not paddleocr_available:
+    print("No OCR engines available. Please install at least one OCR engine.")
+    sys.exit(1)
+
+# Common OCR configuration for fair comparison
+ocr_config = {
+    "languages": ["en"],
+    "device": "cpu",
+    "min_confidence": 0.3
+}
+
+# Set up testing information
+engines = []
+if easyocr_available:
+    engines.append(("EasyOCR", "easyocr"))
+if paddleocr_available:
+    engines.append(("PaddleOCR", "paddleocr"))
+
+# Function to run OCR with an engine and measure performance
+def test_engine(engine_name, engine_id, page_number=0):
+    print(f"\nTesting {engine_name}:")
+    
+    try:
+        # Start timing
+        start_time = time.time()
+        
+        # Load PDF with this engine
+        print(f"  Loading PDF with {engine_name} engine...")
+        pdf = PDF(default_pdf, ocr_engine=engine_id, ocr=ocr_config)
+        
+        # Get the specified page
+        print(f"  Accessing page {page_number}...")
+        page = pdf.pages[page_number]
+        
+        # Check if OCR is properly configured
+        if hasattr(pdf, '_ocr_engine'):
+            print(f"  OCR engine: {pdf._ocr_engine.__class__.__name__}")
+            print(f"  OCR config: {pdf._ocr_config}")
+        else:
+            print("  Warning: PDF does not have _ocr_engine attribute")
+        
+        # Force OCR explicitly
+        print(f"  Extracting OCR elements explicitly...")
+        ocr_elements = page.extract_ocr_elements()
+        print(f"  Found {len(ocr_elements)} OCR elements")
+        
+        if len(ocr_elements) == 0:
+            print("  Warning: No OCR elements found - trying to debug")
+            # Try direct extract_text with OCR flag
+            print("  Trying page.extract_text(ocr=True)...")
+            text = page.extract_text(ocr=True)
+            print(f"  Extract_text with ocr=True returned {len(text)} characters")
+        else:
+            # Extract text
+            print(f"  Extracting text...")
+            text = page.extract_text()
+            print(f"  Extracted {len(text)} characters")
+        
+        extraction_time = time.time() - start_time
+        
+        # Calculate average confidence
+        avg_confidence = sum(elem.confidence for elem in ocr_elements) / len(ocr_elements) if ocr_elements else 0
+        
+        # Create a highlighted image
+        print(f"  Creating highlighted image...")
+        page.clear_highlights()
+        for elem in ocr_elements:
+            if elem.confidence >= 0.7:
+                color = (0, 204, 0, 76)  # Green for high confidence
+            elif elem.confidence >= 0.5:
+                color = (230, 230, 0, 76)  # Yellow for medium confidence
+            else:
+                color = (204, 0, 0, 76)  # Red for low confidence
+            
+            elem.highlight(label=f"{engine_name}", color=color)
+        
+        # Save the image
+        output_path = os.path.join(output_dir, f"{engine_name.lower()}_results.png")
+        page.to_image(path=output_path, show_labels=True)
+        
+        # Return results
+        return {
+            "engine": engine_name,
+            "extraction_time": extraction_time,
+            "text_length": len(text),
+            "element_count": len(ocr_elements),
+            "avg_confidence": avg_confidence,
+            "output_path": output_path
+        }
+    
+    except Exception as e:
+        print(f"  Error during {engine_name} test: {e}")
+        import traceback
+        traceback.print_exc()
+        return {
+            "engine": engine_name,
+            "extraction_time": 0,
+            "text_length": 0,
+            "element_count": 0,
+            "avg_confidence": 0,
+            "output_path": "error",
+            "error": str(e)
+        }
+
+# Run tests for each available engine
+results = []
+for engine_name, engine_id in engines:
+    result = test_engine(engine_name, engine_id)
+    results.append(result)
+    
+    # Print some stats
+    print(f"  Extraction time: {result['extraction_time']:.2f} seconds")
+    print(f"  Text length: {result['text_length']} characters")
+    print(f"  Element count: {result['element_count']} elements")
+    print(f"  Average confidence: {result['avg_confidence']:.2f}")
+    print(f"  Output image: {result['output_path']}")
+
+# Compare results
+if len(results) > 1:
+    print("\nComparison Results:")
+    print(f"{'Engine':<10} {'Time (s)':<10} {'Text Len':<10} {'Elements':<10} {'Avg Conf':<10}")
+    print(f"{'-'*60}")
+    for result in results:
+        print(f"{result['engine']:<10} {result['extraction_time']:.2f}s      {result['text_length']:<10} {result['element_count']:<10} {result['avg_confidence']:.2f}")
+    
+    # Highlight differences
+    fastest = min(results, key=lambda x: x['extraction_time'])
+    most_elements = max(results, key=lambda x: x['element_count'])
+    highest_confidence = max(results, key=lambda x: x['avg_confidence'])
+    
+    print(f"\nFastest engine: {fastest['engine']} ({fastest['extraction_time']:.2f}s)")
+    print(f"Most elements: {most_elements['engine']} ({most_elements['element_count']} elements)")
+    print(f"Highest confidence: {highest_confidence['engine']} ({highest_confidence['avg_confidence']:.2f})")
+
+# Additional comparison with engine-specific optimizations
+print("\nRunning comparison with engine-specific optimizations:")
+
+# Custom configurations for each engine
+if easyocr_available and paddleocr_available:
+    # EasyOCR with customized settings
+    easyocr_custom = PDF(default_pdf, 
+                         ocr_engine="easyocr",
+                         ocr={
+                             "languages": ["en"],
+                             "device": "cpu",
+                             "min_confidence": 0.3,
+                             "model_settings": {
+                                 "detail": 1,
+                                 "paragraph": False,
+                                 "contrast_ths": 0.05,
+                                 "text_threshold": 0.5
+                             }
+                         })
+    
+    # PaddleOCR with customized settings
+    paddleocr_custom = PDF(default_pdf,
+                          ocr_engine="paddleocr",
+                          ocr={
+                              "languages": ["en"],
+                              "device": "cpu",
+                              "min_confidence": 0.3,
+                              "model_settings": {
+                                  "use_angle_cls": True,
+                                  "det_db_thresh": 0.2,
+                                  "det_db_box_thresh": 0.3
+                              }
+                          })
+    
+    # Compare text extraction
+    easyocr_text = easyocr_custom.pages[0].extract_text()
+    paddleocr_text = paddleocr_custom.pages[0].extract_text()
+    
+    print(f"\nOptimized EasyOCR text length: {len(easyocr_text)}")
+    print(f"Optimized PaddleOCR text length: {len(paddleocr_text)}")
+    
+    # Compare element counts
+    easyocr_elements = easyocr_custom.pages[0].extract_ocr_elements()
+    paddleocr_elements = paddleocr_custom.pages[0].extract_ocr_elements()
+    
+    print(f"Optimized EasyOCR element count: {len(easyocr_elements)}")
+    print(f"Optimized PaddleOCR element count: {len(paddleocr_elements)}")
+
+print("\nDone!")

examples/ocr_example.py

@@ -0,0 +1,89 @@
+"""
+OCR example using PaddleOCR.
+
+This example demonstrates how to use OCR to extract text from PDF documents,
+both for whole pages and specific regions.
+
+Note: This example requires the 'paddleocr' package:
+pip install paddlepaddle paddleocr
+"""
+import os
+import sys
+from natural_pdf import PDF
+
+# Get the current directory of this script
+script_dir = os.path.dirname(os.path.realpath(__file__))
+# Get the parent directory (project root)
+root_dir = os.path.dirname(script_dir)
+# Default PDF path (replace with a scanned document path for better results)
+default_pdf = os.path.join(root_dir, "pdfs", "HARRY ROQUE_redacted.pdf")
+# Output directory
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+print("OCR Example")
+print("==========")
+
+# 1. Loading a PDF with OCR enabled
+print("\n1. Loading PDF with OCR enabled")
+pdf = PDF(default_pdf, ocr={
+    "enabled": "auto",  # Auto mode: only use OCR when necessary
+    "languages": ["en"],
+    # For more options, see OCR-NOTES.md
+})
+
+# 2. Extract text from a page with auto OCR
+page = pdf.pages[0]
+print(f"\n2. Extracting text from page {page.number} with auto OCR")
+text = page.extract_text()
+print(f"Extracted {len(text)} characters.")
+print("First 150 characters:\n", text[:150] + "..." if len(text) > 150 else text)
+
+# 3. Force OCR on a page
+print("\n3. Force OCR on a page")
+ocr_text = page.extract_text(ocr=True)  # Force OCR regardless of existing text
+print(f"Extracted {len(ocr_text)} characters with forced OCR.")
+print("First 150 characters:\n", ocr_text[:150] + "..." if len(ocr_text) > 150 else ocr_text)
+
+# 4. Extract OCR elements directly
+print("\n4. Extracting OCR elements directly")
+ocr_elements = page.extract_ocr_elements()
+print(f"Found {len(ocr_elements)} OCR text elements.")
+for i, elem in enumerate(ocr_elements[:3]):  # Show first 3 elements
+    print(f"  Element {i+1}: '{elem.text}' (confidence: {elem.confidence:.2f})")
+
+# 5. Apply OCR to a specific region
+print("\n5. Applying OCR to a specific region")
+# Create a region (adjust coordinates for your PDF)
+region = page.create_region(100, 100, 400, 200)  # x0, y0, x1, y1
+region.highlight(label="OCR Region")
+
+# Apply OCR to this region
+region_elements = region.apply_ocr()
+print(f"Found {len(region_elements)} OCR text elements in the region.")
+
+# Extract text from the region (uses OCR since we already applied it)
+region_text = region.extract_text()
+print(f"Region text: '{region_text[:50]}...'" if len(region_text) > 50 else f"Region text: '{region_text}'")
+
+# 6. Finding OCR text elements with selectors
+print("\n6. Finding OCR text elements with selectors")
+# Find OCR elements with specific properties
+high_confidence_ocr = page.find_all('text[source=ocr][confidence>=0.8]')
+print(f"Found {len(high_confidence_ocr)} high-confidence OCR elements.")
+
+# Find OCR elements containing specific text
+matching_ocr = page.find_all('text[source=ocr]:contains("the")')
+print(f"Found {len(matching_ocr)} OCR elements containing 'the'.")
+
+# 7. Visualize OCR results
+print("\n7. Visualizing OCR results")
+# Highlight all OCR elements
+for elem in ocr_elements:
+    elem.highlight(label=f"OCR ({elem.confidence:.2f})")
+
+# Save the highlighted page
+output_path = os.path.join(output_dir, "ocr_results.png")
+page.to_image(path=output_path, show_labels=True)
+print(f"Saved visualization to {output_path}")
+print("\nDone!")

examples/ocr_simplified_params.py

@@ -0,0 +1,79 @@
+import os
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+
+# Get absolute path for the PDF
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+pdf_path = os.path.join(root_dir, "pdfs", "HARRY ROQUE_redacted.pdf")
+
+print(f"Loading PDF: {pdf_path}")
+
+# Example 1: Initialize PDF with flattened OCR parameters
+pdf = PDF(pdf_path, ocr={
+    "enabled": True,
+    "languages": ["en"],
+    "min_confidence": 0.3,
+    # OCR parameters directly in config root:
+    "text_threshold": 0.1,    # Was previously in detection_params
+    "link_threshold": 0.1,    # Was previously in detection_params
+    "paragraph": True,        # Was previously in recognition_params
+    "detail": 1               # Was previously in recognition_params
+})
+
+# Use a specific page
+page = pdf.pages[3]
+
+# Example 2: Apply OCR with flattened parameters
+print("\nApplying OCR with flattened parameters")
+ocr_elements = page.apply_ocr(
+    # Direct parameters:
+    text_threshold=0.15,
+    link_threshold=0.15,
+    mag_ratio=1.5,
+    canvas_size=1024,
+    batch_size=4
+)
+
+print(f"Found {len(ocr_elements)} OCR text elements")
+
+# Print sample of OCR results
+print("\nSample OCR results:")
+for i, elem in enumerate(ocr_elements[:5]):
+    print(f"{i+1}. '{elem.text}' (conf: {elem.confidence:.2f})")
+    if i >= 4:
+        break
+
+# Example 3: Extract text with OCR using flattened parameters
+print("\nExtracting text with OCR using flattened parameters")
+text = page.extract_text(ocr={
+    "enabled": True,
+    "min_confidence": 0.2,
+    # Direct parameters:
+    "text_threshold": 0.2,
+    "contrast_ths": 0.05
+})
+
+# Display first 100 characters of text
+print(f"\nExtracted text (first 100 chars):")
+print(text[:100] + "...")
+
+# Create output directory if it doesn't exist
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+# Highlight OCR elements
+for elem in ocr_elements[:10]:
+    elem.highlight(label=f"OCR: {elem.text}")
+
+# Save image
+output_path = os.path.join(output_dir, "ocr_simplified.png")
+print(f"\nSaving highlighted image to: {output_path}")
+page.to_image(path=output_path, show_labels=True)
+
+print("\nTest completed successfully!")

examples/ocr_visualization.py

@@ -0,0 +1,102 @@
+"""
+OCR Visualization Example
+
+This example demonstrates the new OCR visualization feature that renders
+OCR text with white background boxes on the image.
+"""
+import os
+import sys
+
+# Add project directory to the path to import the library
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+sys.path.insert(0, root_dir)
+
+from natural_pdf import PDF
+
+# Get paths
+default_pdf = os.path.join(root_dir, "pdfs", "needs-ocr.pdf") 
+if not os.path.exists(default_pdf):
+    default_pdf = os.path.join(root_dir, "pdfs", "01-practice.pdf")
+
+# Output directory
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+def main():
+    """Main example function."""
+    print("OCR Visualization Example")
+    print("=========================")
+    
+    # 1. Load a PDF with OCR enabled
+    print("\n1. Loading PDF with OCR enabled")
+    pdf = PDF(default_pdf, ocr={
+        "enabled": True,
+        "languages": ["en"],
+        "min_confidence": 0.3  # Lower confidence to get more results
+    })
+    
+    # 2. First check if we have OCR text by extracting text with OCR
+    print("\n2. Extracting text with OCR")
+    page = pdf.pages[0]
+    text = page.extract_text(ocr=True)  # Force OCR
+    print(f"Extracted {len(text)} characters with OCR")
+    
+    # 3. Find OCR text elements
+    print("\n3. Finding OCR text elements")
+    ocr_elements = page.find_all('text[source=ocr]')
+    print(f"Found {len(ocr_elements)} OCR text elements on the page")
+    
+    # If we don't have OCR elements, fall back to forcing OCR directly
+    if not ocr_elements:
+        print("No OCR elements found. Running OCR directly...")
+        # Extract OCR elements directly
+        ocr_elements = page.extract_ocr_elements()
+        print(f"Found {len(ocr_elements)} OCR text elements from direct extraction")
+    
+    # 4. Highlight the OCR elements
+    print(f"\n4. Highlighting {len(ocr_elements)} OCR elements")
+    for element in ocr_elements:
+        # Add color highlighting based on confidence score
+        confidence = getattr(element, 'confidence', 0.5)  # Default if not available
+        if confidence >= 0.8:
+            color = (0, 1, 0, 0.3)  # Green for high confidence
+        elif confidence >= 0.5:
+            color = (1, 1, 0, 0.3)  # Yellow for medium confidence
+        else:
+            color = (1, 0, 0, 0.3)  # Red for low confidence
+            
+        element.highlight(color=color, label=f"OCR ({confidence:.2f})")
+    
+    # 5. Visualize the regular highlights (no OCR text)
+    print("\n5. Saving highlighted image without OCR text")
+    highlighted_path = os.path.join(output_dir, "ocr_highlighted.png")
+    page.to_image(path=highlighted_path, show_labels=True, render_ocr=False)
+    print(f"Saved highlighted image to {highlighted_path}")
+    
+    # 6. Visualize with OCR text on white background
+    print("\n6. Saving image with rendered OCR text")
+    ocr_text_path = os.path.join(output_dir, "ocr_rendered_text.png")
+    try:
+        page.to_image(path=ocr_text_path, show_labels=True, render_ocr=True)
+        print(f"Saved OCR text rendering to {ocr_text_path}")
+    except ValueError as e:
+        print(f"Error rendering OCR text: {e}")
+    
+    # 7. Create a clean white page with just OCR text (no highlights)
+    if ocr_elements:
+        print("\n7. Creating clean white page with just OCR text")
+        # Clear previous highlights
+        page.clear_highlights()
+        # Save with OCR text rendering only
+        clean_text_path = os.path.join(output_dir, "ocr_clean_text.png")
+        try:
+            page.to_image(path=clean_text_path, render_ocr=True)
+            print(f"Saved clean OCR text rendering to {clean_text_path}")
+        except ValueError as e:
+            print(f"Error rendering clean OCR text: {e}")
+    
+    print("\nDone!")
+
+if __name__ == "__main__":
+    main()