natural-pdf 25.3.16__py3-none-any.whl

examples/color_name_example.py

@@ -0,0 +1,71 @@
+"""
+Example demonstrating the use of color names in selectors.
+"""
+import sys
+from pathlib import Path
+
+# Add the parent directory to the path to import the local package
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+
+def main():
+    """Run the example."""
+    # Get the PDF file path from command line args or use default
+    if len(sys.argv) > 1:
+        pdf_path = sys.argv[1]
+    else:
+        # Use a default sample PDF
+        pdf_path = str(Path(__file__).parent.parent / "pdfs" / "01-practice.pdf")
+    
+    # Create a PDF object
+    pdf = PDF(pdf_path)
+    page = pdf.pages[0]
+    
+    print("\n=== Using Color Names in Selectors ===\n")
+
+    # Different ways to specify the same red color
+    print("Finding red text using different color specifications:")
+    
+    # Traditional RGB tuple
+    red_text1 = page.find_all('text[color~=(1,0,0)]')
+    print(f"- Using RGB tuple (1,0,0): Found {len(red_text1)} elements")
+    
+    # Using named color
+    red_text2 = page.find_all('text[color~=red]')
+    print(f"- Using named color 'red': Found {len(red_text2)} elements")
+    
+    # Using hex color
+    red_text3 = page.find_all('text[color~=#ff0000]')
+    print(f"- Using hex color '#ff0000': Found {len(red_text3)} elements")
+    
+    # Compare results
+    print("\nAre the results the same?", 
+          len(red_text1) == len(red_text2) == len(red_text3))
+    
+    # Highlight the found elements
+    page.clear_highlights()
+    red_text1.highlight(label="Red (RGB tuple)")
+    
+    # Try a different color by name
+    blue_text = page.find_all('text[color~=blue]')
+    blue_text.highlight(label="Blue (named color)")
+    
+    green_text = page.find_all('text[color~=#00ff00]')
+    green_text.highlight(label="Green (hex color)")
+    
+    print("\nHighlighting the found elements...")
+    
+    # Save the highlighted image
+    output_path = str(Path(__file__).parent.parent / "output" / "color_names.png")
+    page.to_image(path=output_path, show_labels=True)
+    print(f"Image saved to {output_path}")
+    
+    # Show more information about the colors
+    if red_text1:
+        print("\nExample red text element:")
+        print(f"- Text: {red_text1.first.text}")
+        print(f"- Color: {red_text1.first.color}")
+
+if __name__ == "__main__":
+    main()

examples/color_test.py

@@ -0,0 +1,62 @@
+"""
+Test script to verify color conversion in the highlight system.
+"""
+import os
+import sys
+from typing import List, Dict, Tuple, Optional, Union, Any, Set
+
+# Add the parent directory to the path to import the package
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+def test_color_conversion():
+    """Test the color conversion logic directly without relying on the PDF."""
+    print("Testing color conversion logic...")
+    
+    # Test the same logic we added to highlighting.py
+    def normalize_color(color) -> Tuple[int, int, int, int]:
+        """Normalize color tuple to 0-255 integer format."""
+        if isinstance(color, tuple):
+            # Convert values to integers in 0-255 range
+            processed_color = []
+            for i, c in enumerate(color):
+                if isinstance(c, float):
+                    # 0.0-1.0 float format
+                    if c <= 1.0:
+                        processed_color.append(int(c * 255))
+                    # Already in 0-255 range but as float
+                    else:
+                        processed_color.append(int(c))
+                else:
+                    processed_color.append(c)
+                    
+            # Default alpha value if needed
+            if len(processed_color) == 3:
+                processed_color.append(100)  # Default alpha
+                
+            return tuple(processed_color)
+        else:
+            # Default if invalid color is provided
+            return (255, 255, 0, 100)  # Yellow with semi-transparency
+    
+    # Test various color formats
+    test_cases = [
+        ((255, 0, 0, 128), "Integer RGB with alpha"),
+        ((255, 0, 0), "Integer RGB without alpha"),
+        ((0.0, 1.0, 0.0, 0.5), "Float RGB with alpha (0-1)"),
+        ((0.0, 1.0, 0.0), "Float RGB without alpha (0-1)"),
+        ((0.5, 0.5, 255, 0.7), "Mixed float and integer"),
+        ((0.5, 0.5, 255), "Mixed without alpha"),
+        ((128.5, 64.3, 200.7, 50.9), "Float values > 1"),
+        (None, "None case")
+    ]
+    
+    for color, desc in test_cases:
+        print(f"\nTesting: {desc}")
+        print(f"Input:  {color}")
+        result = normalize_color(color)
+        print(f"Output: {result}")
+    
+    print("\nTest complete!")
+    
+if __name__ == "__main__":
+    test_color_conversion()

examples/debug_ocr.py

@@ -0,0 +1,91 @@
+"""
+Debug OCR issues.
+"""
+import os
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+from natural_pdf.ocr import EasyOCREngine
+
+# Get current directory
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+default_pdf = os.path.join(root_dir, "pdfs", "HARRY ROQUE_redacted.pdf")
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+print("OCR Debug Test")
+print("=============")
+
+# Check if OCR engines are available
+try:
+    import easyocr
+    print("EasyOCR is available.")
+except ImportError:
+    print("EasyOCR is not available.")
+    
+try:
+    import paddleocr
+    import paddle
+    print("PaddleOCR is available.")
+except ImportError:
+    print("PaddleOCR is not available.")
+
+# Test with EasyOCR directly (explicit configuration)
+print("\n1. Testing with explicit EasyOCR engine and forced enabled")
+pdf = PDF(default_pdf, 
+          ocr_engine="easyocr", 
+          ocr={
+              "enabled": True,
+              "languages": ["en"],
+              "min_confidence": 0.3
+          })
+
+# Get the page
+print("Getting page...")
+page = pdf.pages[0]
+
+# Print OCR config
+print(f"PDF OCR config: {pdf._ocr_config}")
+print(f"OCR engine type: {type(pdf._ocr_engine)}")
+
+# Generate page image for debugging
+print("Generating debug image of the page...")
+img = page.to_image()
+img_path = os.path.join(output_dir, "debug_page_image.png")
+img.save(img_path)
+print(f"Saved page image to {img_path}")
+
+# Force OCR extraction
+print("Forcing OCR extraction...")
+ocr_elements = page.extract_ocr_elements()
+print(f"Extracted {len(ocr_elements)} OCR elements")
+
+# Print details of first few elements if any
+if ocr_elements:
+    for i, elem in enumerate(ocr_elements[:3]):
+        print(f"Element {i+1}: '{elem.text}' (conf: {elem.confidence:.2f})")
+else:
+    print("No OCR elements found!")
+
+# Extract text with OCR
+print("Extracting text with OCR=True...")
+text = page.extract_text(ocr=True)
+print(f"Extracted {len(text)} characters of text")
+print(f"First 100 chars: {text[:100]}...")
+
+# Create a debug image
+print("Creating debug visualization...")
+page.clear_highlights()
+for elem in ocr_elements:
+    elem.highlight(label=f"OCR ({elem.confidence:.2f})")
+    
+output_path = os.path.join(output_dir, "ocr_debug.png")
+page.to_image(path=output_path, show_labels=True)
+print(f"Saved debug image to {output_path}")
+
+print("\nTest complete!")

examples/direct_ocr_test.py

@@ -0,0 +1,148 @@
+"""
+Direct OCR test script to debug OCR issues.
+"""
+import os
+import sys
+from PIL import Image
+import numpy as np
+
+# Add the project directory to the path to import the library
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from natural_pdf import PDF
+
+# Select a PDF file to test
+PDF_FILE = "./pdfs/HARRY ROQUE_redacted.pdf"
+if not os.path.exists(PDF_FILE):
+    PDF_FILE = "./pdfs/01-practice.pdf"  # Fallback to another file if needed
+
+def test_direct_ocr():
+    """Test OCR engines directly."""
+    
+    # Create output directory
+    output_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "output")
+    os.makedirs(output_dir, exist_ok=True)
+    
+    # Direct test with EasyOCR
+    print("\n=== Direct test with EasyOCR ===")
+    try:
+        import easyocr
+        # Use the provided PDF file
+        with PDF(PDF_FILE) as pdf:
+            # Get the first page
+            page = pdf.pages[0]
+            # Convert to image
+            image = page.to_image()
+            image_path = os.path.join(output_dir, "easyocr_test_input.png")
+            image.save(image_path)
+            print(f"Saved image to {image_path}")
+            
+            # Run EasyOCR directly
+            reader = easyocr.Reader(['en'])
+            results = reader.readtext(np.array(image))
+            print(f"EasyOCR found {len(results)} text elements")
+            
+            # Print results
+            for i, (bbox, text, conf) in enumerate(results[:5]):
+                print(f"Result {i+1}: '{text}' (Confidence: {conf:.2f})")
+            
+            print("EasyOCR direct test successful")
+    except ImportError:
+        print("EasyOCR not available")
+    except Exception as e:
+        print(f"Error in EasyOCR direct test: {e}")
+        import traceback
+        traceback.print_exc()
+        
+    # Direct test with PaddleOCR
+    print("\n=== Direct test with PaddleOCR ===")
+    try:
+        import paddleocr
+        # Use the provided PDF file
+        with PDF(PDF_FILE) as pdf:
+            # Get the first page
+            page = pdf.pages[0]
+            # Convert to image
+            image = page.to_image()
+            image_path = os.path.join(output_dir, "paddleocr_test_input.png")
+            image.save(image_path)
+            print(f"Saved image to {image_path}")
+            
+            # Run PaddleOCR directly
+            reader = paddleocr.PaddleOCR(lang='en')
+            results = reader.ocr(np.array(image), cls=False)
+            
+            if results is not None and len(results) > 0:
+                page_result = results[0] if isinstance(results[0], list) else results
+                print(f"PaddleOCR found {len(page_result)} text elements")
+                
+                # Print results
+                for i, detection in enumerate(page_result[:5]):
+                    if len(detection) >= 2:
+                        bbox = detection[0]
+                        text_conf = detection[1]
+                        text = text_conf[0] if isinstance(text_conf, tuple) and len(text_conf) >= 2 else str(text_conf)
+                        conf = text_conf[1] if isinstance(text_conf, tuple) and len(text_conf) >= 2 else 1.0
+                        print(f"Result {i+1}: '{text}' (Confidence: {conf:.2f})")
+            else:
+                print(f"PaddleOCR returned no results: {results}")
+            
+            print("PaddleOCR direct test complete")
+    except ImportError:
+        print("PaddleOCR not available")
+    except Exception as e:
+        print(f"Error in PaddleOCR direct test: {e}")
+        import traceback
+        traceback.print_exc()
+        
+def test_library_ocr():
+    """Test OCR integration with the library."""
+    
+    print("\n=== Test library integration with EasyOCR ===")
+    try:
+        # Create a PDF with explicit OCR config
+        with PDF(PDF_FILE, ocr={"enabled": True, "languages": ["en"]}, ocr_engine="easyocr") as pdf:
+            # Get the first page
+            page = pdf.pages[0]
+            
+            # Extract text with OCR
+            print("Running OCR through library...")
+            elements = page.extract_ocr_elements()
+            
+            print(f"Library OCR found {len(elements)} text elements")
+            
+            # Print results
+            for i, elem in enumerate(elements[:5]):
+                print(f"Result {i+1}: '{elem.text}' (Confidence: {elem.confidence:.2f})")
+                
+            print("Library OCR with EasyOCR test complete")
+    except Exception as e:
+        print(f"Error in library OCR with EasyOCR test: {e}")
+        import traceback
+        traceback.print_exc()
+        
+    print("\n=== Test library integration with PaddleOCR ===")
+    try:
+        # Create a PDF with explicit OCR config
+        with PDF(PDF_FILE, ocr={"enabled": True, "languages": ["en"]}, ocr_engine="paddleocr") as pdf:
+            # Get the first page
+            page = pdf.pages[0]
+            
+            # Extract text with OCR
+            print("Running OCR through library...")
+            elements = page.extract_ocr_elements()
+            
+            print(f"Library OCR found {len(elements)} text elements")
+            
+            # Print results
+            for i, elem in enumerate(elements[:5]):
+                print(f"Result {i+1}: '{elem.text}' (Confidence: {elem.confidence:.2f})")
+                
+            print("Library OCR with PaddleOCR test complete")
+    except Exception as e:
+        print(f"Error in library OCR with PaddleOCR test: {e}")
+        import traceback
+        traceback.print_exc()
+
+if __name__ == "__main__":
+    test_direct_ocr()
+    test_library_ocr()

examples/direct_paddle_test.py

@@ -0,0 +1,99 @@
+"""
+Direct test of PaddlePaddle's PPStructure functionality.
+
+This script bypasses our library and directly uses paddleocr to test layout detection.
+"""
+import os
+import sys
+from pathlib import Path
+import cv2
+
+try:
+    from paddleocr import PPStructure
+except ImportError:
+    print("PaddleOCR not installed. Run: pip install paddlepaddle paddleocr")
+    sys.exit(1)
+
+# Get the current directory of this script
+script_dir = os.path.dirname(os.path.realpath(__file__))
+# Get the parent directory (project root)
+root_dir = os.path.dirname(script_dir)
+# Default PDF path
+default_pdf = os.path.join(root_dir, "pdfs", "2019 Statistics.pdf")
+
+# Check command line args
+if len(sys.argv) > 1:
+    image_path = sys.argv[1]
+else:
+    # Convert first page of PDF to image since PPStructure needs an image
+    import fitz  # PyMuPDF
+    pdf_path = default_pdf
+    print(f"Converting first page of {pdf_path} to image...")
+    
+    pdf_doc = fitz.open(pdf_path)
+    page = pdf_doc[0]
+    
+    # Render page at higher resolution
+    zoom = 2.0  # Increase resolution
+    mat = fitz.Matrix(zoom, zoom)
+    pix = page.get_pixmap(matrix=mat)
+    
+    # Save as image
+    image_path = os.path.join(root_dir, "output", "direct_paddle_test.png")
+    pix.save(image_path)
+    print(f"Saved image to {image_path}")
+
+# Ensure image exists
+if not os.path.exists(image_path):
+    print(f"Image doesn't exist: {image_path}")
+    sys.exit(1)
+
+print(f"Running PPStructure on {image_path}...")
+
+# Initialize PP-Structure with minimal settings
+table_engine = PPStructure(show_log=True)
+
+try:
+    # Run layout analysis
+    result = table_engine(image_path)
+    
+    # Print results
+    print(f"Found {len(result)} layout regions:")
+    for i, region in enumerate(result):
+        region_type = region.get('type', 'unknown')
+        bbox = region.get('bbox', [])
+        confidence = region.get('score', 0)
+        print(f"{i+1}. Type: {region_type}, Confidence: {confidence:.4f}, BBox: {bbox}")
+        
+        # Check for OCR text inside the region
+        if 'res' in region:
+            if isinstance(region['res'], dict) and 'text' in region['res']:
+                print(f"   Text: {region['res']['text'][:50]}...")
+            elif isinstance(region['res'], dict) and 'cells' in region['res']:
+                print(f"   Table with {len(region['res']['cells'])} cells")
+            else:
+                print(f"   Has result data: {type(region['res'])}")
+    
+    # Try directly with PaddleOCR for layout analysis
+    from paddleocr import PaddleOCR
+    print("\nTrying with direct PaddleOCR...")
+    
+    ocr_engine = PaddleOCR(lang="en", show_log=True)
+    layout_result = ocr_engine.ocr(image_path, det=True, rec=True, cls=False)
+    
+    if layout_result:
+        print(f"PaddleOCR found text elements on page 1: {len(layout_result[0])}")
+        
+        # Print first few elements
+        for i, line in enumerate(layout_result[0][:5]):
+            points = line[0]  # Coordinates: [[x1,y1], [x2,y2], [x3,y3], [x4,y4]]
+            text = line[1][0]  # Text content
+            confidence = line[1][1]  # Confidence score
+            print(f"  {i+1}. Text: '{text}', Confidence: {confidence:.4f}")
+    else:
+        print("PaddleOCR found no elements")
+    
+except Exception as e:
+    print(f"Error: {e}")
+    import traceback
+    traceback.print_exc()

examples/direct_qa_example.py

@@ -0,0 +1,165 @@
+"""
+Direct Document QA example that closely mirrors the original pdfplumber implementation.
+
+This example shows how to:
+1. Use pdfplumber directly to extract words and images
+2. Use transformers pipelines for document QA
+3. Compare with the Natural PDF implementation
+
+It's intentionally similar to the original code provided by the user.
+"""
+
+import os
+import sys
+import argparse
+import pdfplumber
+from PIL import Image, ImageDraw
+import numpy as np
+
+# Add parent directory to path to run without installing
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+# For comparison
+from natural_pdf import PDF, configure_logging
+import logging
+
+def pdfplumber_qa(pdf_path, question, debug=False):
+    """Run QA using direct pdfplumber code similar to the original example."""
+    # Open PDF
+    pdf = pdfplumber.open(pdf_path)
+    page = pdf.pages[0]
+    
+    # Get image
+    image = page.to_image(resolution=300).original
+    
+    # Extract words
+    words = page.extract_words()
+    
+    # Build word boxes in the expected format
+    def get_box(word):
+        return [
+            word['text'],
+            [int(word["x0"]), int(word["top"]), int(word["x1"]), int(word["bottom"])]
+        ]
+    
+    word_boxes = [get_box(word) for word in words]
+    
+    # Debug visualization
+    if debug:
+        os.makedirs("output", exist_ok=True)
+        
+        # Save image
+        image.save("output/direct_qa_image.png")
+        
+        # Save visualization
+        vis_image = image.copy()
+        draw = ImageDraw.Draw(vis_image)
+        
+        for i, (text, box) in enumerate(word_boxes):
+            x0, y0, x1, y1 = box
+            draw.rectangle((x0, y0, x1, y1), outline=(255, 0, 0), width=2)
+            draw.text((x0, y0), str(i), fill=(255, 0, 0))
+        
+        vis_image.save("output/direct_qa_boxes.png")
+        
+    # Use transformers pipeline
+    try:
+        from transformers import pipeline
+        
+        pipe = pipeline("document-question-answering", model="impira/layoutlm-document-qa")
+        
+        # Run query
+        query = { "image": image, "question": question, "word_boxes": word_boxes }
+        
+        result = pipe(query)[0]
+        
+        # Create result dictionary similar to Natural PDF's format
+        return {
+            "answer": result.get("answer", ""),
+            "confidence": result.get("score", 0.0),
+            "start": result.get("start", 0),
+            "end": result.get("end", 0),
+            "found": True if result.get("answer") else False
+        }
+        
+    except Exception as e:
+        print(f"Error in direct QA: {e}")
+        return {
+            "answer": "",
+            "confidence": 0.0,
+            "error": str(e),
+            "found": False
+        }
+
+def main():
+    parser = argparse.ArgumentParser(description="Direct Document QA Example")
+    parser.add_argument("pdf_path", nargs="?", default="../pdfs/0500000US42001.pdf", 
+                      help="Path to PDF document")
+    parser.add_argument("--question", default="How many votes for Harris and Walz?",
+                      help="Question to ask about the document")
+    parser.add_argument("--debug", action="store_true",
+                      help="Save debug information for troubleshooting")
+    parser.add_argument("--compare", action="store_true",
+                      help="Compare with Natural PDF implementation")
+    
+    args = parser.parse_args()
+    
+    # Configure logging for Natural PDF
+    if args.debug:
+        configure_logging(level=logging.DEBUG)
+    else:
+        configure_logging(level=logging.INFO)
+    
+    print(f"Document: {args.pdf_path}")
+    print(f"Question: {args.question}")
+    
+    # Run direct pdfplumber QA
+    print("\n=== Direct pdfplumber implementation ===")
+    result = pdfplumber_qa(args.pdf_path, args.question, debug=args.debug)
+    
+    if result.get("found", False):
+        print(f"Answer: {result['answer']}")
+        print(f"Confidence: {result['confidence']:.2f}")
+    else:
+        print(f"No answer found: {result.get('error', '')}")
+    
+    # Compare with Natural PDF if requested
+    if args.compare:
+        print("\n=== Natural PDF implementation ===")
+        
+        # Use Natural PDF
+        pdf = PDF(args.pdf_path)
+        page = pdf.pages[0]
+        
+        # Ask the question
+        natural_result = page.ask(args.question, debug=args.debug)
+        
+        if natural_result.get("found", False):
+            print(f"Answer: {natural_result['answer']}")
+            print(f"Confidence: {natural_result['confidence']:.2f}")
+            
+            # Highlight the answer
+            if natural_result.get("source_elements"):
+                for element in natural_result["source_elements"]:
+                    element.highlight(color=(1, 0.5, 0, 0.5))
+                
+                # Save the image
+                page.save_image("output/natural_pdf_answer.png")
+                print("Saved highlighted answer to output/natural_pdf_answer.png")
+        else:
+            print(f"No answer found: {natural_result.get('error', '')}")
+        
+        # Compare results
+        if result.get("found", False) and natural_result.get("found", False):
+            print("\n=== Comparison ===")
+            print(f"Direct answer: '{result['answer']}' (confidence: {result['confidence']:.2f})")
+            print(f"Natural PDF answer: '{natural_result['answer']}' (confidence: {natural_result['confidence']:.2f})")
+            
+            # Calculate similarity
+            if result['answer'] == natural_result['answer']:
+                print("Results match exactly!")
+            else:
+                print("Results differ.")
+
+if __name__ == "__main__":
+    main()