natural-pdf 25.3.16__py3-none-any.whl

examples/region_exclusion_test.py

@@ -0,0 +1,149 @@
+"""
+Test the improved exclusion handling in Region.extract_text() method.
+"""
+
+import os
+import sys
+import logging
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF, configure_logging
+
+# Configure logging
+configure_logging(level=logging.DEBUG)
+
+
+def test_region_with_exclusions(pdf_path):
+    """
+    Test extracting text from regions with various exclusion scenarios.
+    """
+    with PDF(pdf_path) as pdf:
+        page = pdf.pages[0]
+        print(f"\nTesting with PDF: {pdf_path} (page {page.number})")
+        
+        print("\n=== 1. Creating Test Exclusion Zones ===")
+        # Create top (header) and bottom (footer) exclusions
+        # Top 15% of the page
+        top_exclusion = page.create_region(0, 0, page.width, page.height * 0.15)
+        top_exclusion.highlight(label="Header Exclusion", color=(1, 0, 0, 0.3))
+        page.add_exclusion(top_exclusion)
+        print(f"Added header exclusion: {top_exclusion.bbox}")
+        
+        # Bottom 10% of the page
+        bottom_exclusion = page.create_region(0, page.height * 0.9, page.width, page.height)
+        bottom_exclusion.highlight(label="Footer Exclusion", color=(0, 0, 1, 0.3))
+        page.add_exclusion(bottom_exclusion)
+        print(f"Added footer exclusion: {bottom_exclusion.bbox}")
+        
+        # Middle partial-width exclusion
+        middle_exclusion = page.create_region(0, page.height * 0.4, page.width * 0.3, page.height * 0.6)
+        middle_exclusion.highlight(label="Side Exclusion", color=(0, 1, 0, 0.3))
+        page.add_exclusion(middle_exclusion)
+        print(f"Added side exclusion: {middle_exclusion.bbox}")
+        
+        print("\n=== 2. Testing Region That Doesn't Intersect Exclusions ===")
+        # Create a region that doesn't intersect with any exclusion
+        non_intersecting = page.create_region(
+            page.width * 0.4,
+            page.height * 0.5, 
+            page.width * 0.9,
+            page.height * 0.7
+        )
+        non_intersecting.highlight(label="Non-Intersecting", color=(1, 1, 0, 0.3))
+        
+        # Extract with and without applying exclusions - should be the same
+        text_with_exclusions = non_intersecting.extract_text(apply_exclusions=True, debug=True)
+        text_without_exclusions = non_intersecting.extract_text(apply_exclusions=False)
+        print(f"Non-intersecting region text length:")
+        print(f"  - With exclusions: {len(text_with_exclusions)} chars")
+        print(f"  - Without exclusions: {len(text_without_exclusions)} chars")
+        print(f"  - Same result: {text_with_exclusions == text_without_exclusions}")
+        
+        print("\n=== 3. Testing Region With Header/Footer Intersection ===")
+        # Create a region that intersects with header and footer
+        full_height = page.create_region(
+            page.width * 0.3,
+            0, 
+            page.width * 0.8,
+            page.height
+        )
+        full_height.highlight(label="Full Height Region", color=(1, 0, 1, 0.3))
+        
+        # Extract with and without applying exclusions
+        text_with_exclusions = full_height.extract_text(apply_exclusions=True, debug=True)
+        text_without_exclusions = full_height.extract_text(apply_exclusions=False)
+        print(f"Full height region text length:")
+        print(f"  - With exclusions: {len(text_with_exclusions)} chars")
+        print(f"  - Without exclusions: {len(text_without_exclusions)} chars")
+        print(f"  - Exclusions removed {len(text_without_exclusions) - len(text_with_exclusions)} chars")
+        
+        # Test the specific case that was causing issues
+        middle_to_footer = page.create_region(
+            page.width * 0.3,
+            page.height * 0.4,  # Middle of page
+            page.width * 0.8,
+            page.height        # All the way to bottom (overlapping footer)
+        )
+        middle_to_footer.highlight(label="Middle to Footer", color=(0.5, 0.5, 0, 0.3))
+        
+        text_with_exclusions = middle_to_footer.extract_text(apply_exclusions=True, debug=True)
+        text_without_exclusions = middle_to_footer.extract_text(apply_exclusions=False)
+        print(f"\nMiddle-to-footer region text length:")
+        print(f"  - With exclusions: {len(text_with_exclusions)} chars")
+        print(f"  - Without exclusions: {len(text_without_exclusions)} chars")
+        if len(text_with_exclusions) > 0:
+            print(f"  - Working correctly! Content found with exclusions applied")
+        else:
+            print(f"  - Still failing! No content found with exclusions applied")
+        
+        print("\n=== 4. Testing Region With Complex Exclusion Intersection ===")
+        # Create a region that intersects with the side exclusion
+        complex_region = page.create_region(
+            page.width * 0.1,
+            page.height * 0.3, 
+            page.width * 0.5,
+            page.height * 0.7
+        )
+        complex_region.highlight(label="Complex Region", color=(0, 1, 1, 0.3))
+        
+        # Extract with and without applying exclusions
+        text_with_exclusions = complex_region.extract_text(apply_exclusions=True, debug=True)
+        text_without_exclusions = complex_region.extract_text(apply_exclusions=False)
+        print(f"Complex region text length:")
+        print(f"  - With exclusions: {len(text_with_exclusions)} chars")
+        print(f"  - Without exclusions: {len(text_without_exclusions)} chars")
+        print(f"  - Exclusions removed {len(text_without_exclusions) - len(text_with_exclusions)} chars")
+        
+        # Save the image with all regions and exclusions highlighted
+        print("\n=== 5. Saving Visual Test Image ===")
+        output_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'output'))
+        os.makedirs(output_dir, exist_ok=True)
+        output_file = os.path.join(output_dir, "region_exclusion_test.png")
+        page.save_image(output_file, labels=True)
+        print(f"Saved test visualization to: {output_file}")
+
+
+def main():
+    """Main entry point."""
+    # Get the PDF path from command line or use a default
+    if len(sys.argv) > 1:
+        pdf_path = sys.argv[1]
+    else:
+        # Look for any PDF in the pdfs directory
+        pdfs_dir = Path(__file__).parent.parent / "pdfs"
+        pdf_files = list(pdfs_dir.glob("*.pdf"))
+        
+        if pdf_files:
+            pdf_path = str(pdf_files[0])
+        else:
+            print("No PDF file found. Please provide a path to a PDF file.")
+            sys.exit(1)
+    
+    test_region_with_exclusions(pdf_path)
+
+
+if __name__ == "__main__":
+    main()

examples/region_expand_example.py

@@ -0,0 +1,109 @@
+"""
+Example demonstrating the region.expand() method in Natural PDF.
+
+This example shows how to expand or shrink regions in various ways.
+"""
+import os
+import sys
+
+# Add the parent directory to the path so we can import natural_pdf module
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from natural_pdf import PDF
+
+def main():
+    # If a PDF path is provided, use it; otherwise use the default example
+    if len(sys.argv) > 1:
+        pdf_path = sys.argv[1]
+    else:
+        # Use a default PDF path
+        pdf_path = "pdfs/Atlanta_Public_Schools_GA_sample.pdf"
+        if not os.path.exists(pdf_path):
+            print(f"Default PDF not found at {pdf_path}")
+            print("Please provide a PDF path as an argument")
+            return
+    
+    print(f"Processing PDF: {pdf_path}")
+    pdf = PDF(pdf_path)
+    page = pdf.pages[0]
+    
+    # Example 1: Basic expansion in different directions
+    print("\n1. Basic region expansion")
+    
+    # Find a text element to start with
+    text = page.find('text')
+    if not text:
+        print("No text found on page")
+        return
+        
+    # Create a region from the text element (its bounding box)
+    region = page.create_region(text.x0, text.top, text.x1, text.bottom)
+    print(f"Original region: {region.bbox}")
+    
+    # Expand the region in different directions
+    expanded_right = region.expand(right=50)
+    print(f"Expanded right by 50: {expanded_right.bbox}")
+    
+    expanded_all = region.expand(left=10, right=20, top_expand=15, bottom_expand=25)
+    print(f"Expanded in all directions: {expanded_all.bbox}")
+    
+    # Shrink the region with negative values
+    shrunk = region.expand(left=-5, right=-5, top_expand=-2, bottom_expand=-2)
+    print(f"Shrunk with negative values: {shrunk.bbox}")
+    
+    # Example 2: Using expansion factors
+    print("\n2. Expansion with factors")
+    
+    # Double the width
+    double_width = region.expand(width_factor=2.0)
+    print(f"Double width (width_factor=2.0): {double_width.bbox}")
+    
+    # Increase height by 50%
+    taller = region.expand(height_factor=1.5)
+    print(f"50% taller (height_factor=1.5): {taller.bbox}")
+    
+    # Both width and height factors
+    bigger = region.expand(width_factor=1.5, height_factor=1.25)
+    print(f"Wider and taller: {bigger.bbox}")
+    
+    # Example 3: Combining with spatial navigation
+    print("\n3. Combining with spatial navigation")
+    
+    # Find a heading (assuming it's bold or larger text)
+    heading = page.find('text[size>=12]')
+    if heading:
+        print(f"Found heading: '{heading.text}'")
+        
+        # Create a region below the heading and expand it
+        content_region = heading.below(height=100, full_width=False)
+        print(f"Region below heading: {content_region.bbox}")
+        
+        # Expand the region to include more content
+        expanded_region = content_region.expand(right=100, bottom_expand=50)
+        print(f"Expanded region: {expanded_region.bbox}")
+        
+        # Extract text from the expanded region
+        text = expanded_region.extract_text()
+        print(f"Text in expanded region: {text[:100]}...")
+    
+    # Example 4: Visual demonstration with highlighting
+    print("\n4. Visual demonstration with highlighting")
+    
+    # Choose a region to work with
+    demo_region = page.create_region(100, 100, 300, 200)
+    
+    # Highlight the original region
+    demo_region.highlight(color=(1, 0, 0), label="Original")
+    
+    # Highlight expanded versions with different colors
+    demo_region.expand(left=20, right=20).highlight(color=(0, 1, 0), label="Wider")
+    demo_region.expand(top_expand=20, bottom_expand=20).highlight(color=(0, 0, 1), label="Taller")
+    demo_region.expand(width_factor=1.5, height_factor=1.5).highlight(color=(1, 0.5, 0), label="1.5x Larger")
+    
+    # Save the highlighted page
+    highlight_path = "region_expand_highlight.png"
+    page.to_image(path=highlight_path, show_labels=True)
+    print(f"Highlighted regions saved to {highlight_path}")
+
+if __name__ == "__main__":
+    main()

examples/region_image_example.py

@@ -0,0 +1,116 @@
+"""
+Example demonstrating the new region.to_image() and region.save_image() functionality.
+
+This example shows how to:
+1. Create regions in various ways
+2. Generate images of just the region
+3. Save region images to files
+4. Compare different rendering options
+"""
+
+import os
+import sys
+import argparse
+
+# Add parent directory to path to run without installing
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from natural_pdf import PDF
+
+def main():
+    parser = argparse.ArgumentParser(description="Region Image Example")
+    parser.add_argument("pdf_path", nargs="?", default="../pdfs/0500000US42001.pdf", 
+                      help="Path to PDF document")
+    args = parser.parse_args()
+    
+    print(f"Opening PDF: {args.pdf_path}")
+    
+    # Open the PDF
+    pdf = PDF(args.pdf_path)
+    page = pdf.pages[0]
+    
+    # Create output directory
+    os.makedirs("output", exist_ok=True)
+    
+    # Method 1: Find a text element and create a region below it
+    print("Creating regions...")
+    title = page.find('text:bold')
+    if not title:
+        title = page.find('text')
+    
+    region_below = title.below(height=100, width="element")
+    
+    # Method 2: Create a region from a specific part of the page
+    page_width, page_height = page.width, page.height
+    center_region = page.create_region(
+        page_width / 4,     # Left quarter of page
+        page_height / 4,    # Top quarter of page
+        page_width * 3/4,   # Right three-quarters
+        page_height * 3/4   # Bottom three-quarters
+    )
+    
+    # Method 3: Use layout detection to find regions
+    page.analyze_layout(confidence=0.3)
+    layout_regions = page.find_all('region')
+    
+    # Generate and save images for each region
+    print("Generating region images...")
+    
+    # Example 1: Basic region image with default settings
+    region_below.save_image("output/region_below.png")
+    print(f"Saved basic region image to output/region_below.png")
+    
+    # Example 2: Region image with highlighted content
+    # First highlight some elements in the region
+    elements = region_below.find_all('text')
+    if elements:
+        elements[0].highlight(color=(1, 0, 0, 0.3), label="First Element")
+        
+        # Save with highlights included
+        region_below.save_image(
+            "output/region_with_highlights.png", 
+            include_highlights=True
+        )
+        print(f"Saved region with highlights to output/region_with_highlights.png")
+        
+        # Save without highlights
+        region_below.save_image(
+            "output/region_without_highlights.png", 
+            include_highlights=False
+        )
+        print(f"Saved region without highlights to output/region_without_highlights.png")
+    
+    # Example 3: Region image without border
+    center_region.save_image(
+        "output/center_region_with_border.png"
+    )
+    print(f"Saved center region with border to output/center_region_with_border.png")
+    
+    center_region.save_image(
+        "output/center_region_without_border.png",
+        crop_only=True
+    )
+    print(f"Saved center region without border to output/center_region_without_border.png")
+    
+    # Example 4: High-resolution region image
+    if layout_regions:
+        first_layout = layout_regions[0]
+        first_layout.highlight(label=f"Region Type: {first_layout.region_type}")
+        
+        # Save at different resolutions
+        first_layout.save_image(
+            "output/layout_region_low_res.png",
+            resolution=72
+        )
+        print(f"Saved layout region at 72 DPI to output/layout_region_low_res.png")
+        
+        first_layout.save_image(
+            "output/layout_region_high_res.png",
+            resolution=300
+        )
+        print(f"Saved layout region at 300 DPI to output/layout_region_high_res.png")
+    
+    print("\nDone! Check the output directory for the generated images.")
+
+if __name__ == "__main__":
+    main()

examples/region_ocr_test.py

@@ -0,0 +1,119 @@
+"""
+Test to identify and fix issues with region-specific OCR.
+"""
+import os
+import sys
+
+# Add the parent directory to the path to import the package
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from natural_pdf import PDF
+from PIL import Image, ImageDraw
+
+def test_region_ocr():
+    """Test OCR applied to specific regions."""
+    # Use a PDF that may work well with OCR
+    pdf_path = os.path.abspath(os.path.join(
+        os.path.dirname(__file__), '..', 'pdfs', 'Nigeria 2021_MICS_SFR_English.pdf'))
+    
+    if not os.path.exists(pdf_path):
+        # Fall back to another PDF
+        pdf_path = os.path.abspath(os.path.join(
+            os.path.dirname(__file__), '..', 'pdfs', '0500000US42001.pdf'))
+        
+    if not os.path.exists(pdf_path):
+        print("No suitable PDF file found for region OCR testing.")
+        return
+    
+    print(f"Testing with PDF: {pdf_path}")
+    
+    # Output directory
+    output_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'output'))
+    os.makedirs(output_dir, exist_ok=True)
+    
+    with PDF(pdf_path) as pdf:
+        # Get the first page
+        page = pdf.pages[0]
+        
+        # Save the entire page image for reference
+        page_img = page.to_image(path=os.path.join(output_dir, "region_ocr_full_page.png"))
+        
+        # Create a region in the middle of the page
+        half_width = page.width / 2
+        half_height = page.height / 2
+        region_width = page.width / 3
+        region_height = page.height / 3
+        
+        region = page.create_region(
+            half_width - region_width/2, 
+            half_height - region_height/2,
+            half_width + region_width/2, 
+            half_height + region_height/2
+        )
+        
+        # Highlight the region
+        region.highlight(label="OCR Test Region")
+        page.to_image(path=os.path.join(output_dir, "region_ocr_highlighted.png"), show_labels=True)
+        
+        # Extract text from the region with and without OCR
+        text_no_ocr = region.extract_text()
+        text_with_ocr = region.extract_text(ocr=True)
+        
+        # Print results
+        print("\nRegion Text WITHOUT OCR:")
+        print("-" * 40)
+        print(text_no_ocr)
+        
+        print("\nRegion Text WITH OCR:")
+        print("-" * 40)
+        print(text_with_ocr)
+        
+        # Apply OCR to the region and visualize the results
+        ocr_elements = region.apply_ocr(enabled=True)
+        
+        print(f"\nFound {len(ocr_elements)} OCR elements in the region")
+        
+        # Get the region image
+        page_img = page.to_image()
+        region_img = page_img.crop((region.x0, region.top, region.x1, region.bottom))
+        
+        # Save region image for reference
+        region_img.save(os.path.join(output_dir, "region_ocr_cropped.png"))
+        
+        # Create debug image showing OCR bounding boxes
+        debug_img = page.to_image()
+        draw = ImageDraw.Draw(debug_img)
+        
+        # Draw region rectangle
+        draw.rectangle(
+            (region.x0, region.top, region.x1, region.bottom),
+            outline=(255, 0, 0),
+            width=3
+        )
+        
+        # Draw OCR element bounding boxes
+        for elem in ocr_elements:
+            draw.rectangle(
+                (elem.x0, elem.top, elem.x1, elem.bottom),
+                outline=(0, 255, 0),
+                width=2
+            )
+            
+            # Draw text label
+            draw.text(
+                (elem.x0, elem.top - 10),
+                elem.text[:10],
+                fill=(0, 0, 255)
+            )
+        
+        # Save debug image
+        debug_img.save(os.path.join(output_dir, "region_ocr_debug.png"))
+        
+        print(f"\nCreated debug images in: {output_dir}")
+        print("- region_ocr_full_page.png: Original page")
+        print("- region_ocr_highlighted.png: Page with region highlighted")
+        print("- region_ocr_cropped.png: Cropped region image")
+        print("- region_ocr_debug.png: Page with OCR text bounding boxes")
+
+if __name__ == "__main__":
+    test_region_ocr()

examples/region_sections_example.py

@@ -0,0 +1,115 @@
+"""
+Example demonstrating the get_sections() method on regions in Natural PDF.
+
+This example shows how to extract logical sections from regions 
+using various types of boundary elements.
+"""
+import os
+import sys
+
+# Add the parent directory to the path so we can import natural_pdf module
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from natural_pdf import PDF
+
+def main():
+    # If a PDF path is provided, use it; otherwise use the default example
+    if len(sys.argv) > 1:
+        pdf_path = sys.argv[1]
+    else:
+        # Use a default PDF path - you'll need to replace this with an actual PDF path
+        pdf_path = "examples/sample.pdf"
+        if not os.path.exists(pdf_path):
+            print(f"Default PDF not found at {pdf_path}")
+            print("Please provide a PDF path as an argument")
+            return
+    
+    print(f"Processing PDF: {pdf_path}")
+    pdf = PDF(pdf_path)
+    page = pdf.pages[0]
+    
+    # Example 1: Get sections within a region using separators
+    print("\n1. Get sections within a region using separators")
+    
+    # First, create a region from the top half of the page
+    top_half = page.create_region(0, 0, page.width, page.height / 2)
+    print(f"Created region: {top_half.bbox}")
+    
+    # Method 1: Find elements first, then pass them to get_sections
+    lines = top_half.find_all('line')
+    print(f"Found {len(lines)} line elements in the region")
+    
+    # Extract sections using lines as start elements
+    sections1 = top_half.get_sections(start_elements=lines)
+    print(f"Found {len(sections1)} sections using explicit elements")
+    
+    # Method 2: Pass selector directly to start_elements
+    sections2 = top_half.get_sections(start_elements='line')
+    print(f"Found {len(sections2)} sections using selector string")
+    
+    # Display section details
+    for i, section in enumerate(sections2):
+        text = section.extract_text()
+        text_snippet = text[:50] + "..." if len(text) > 50 else text
+        print(f"  Section {i+1}: {section.bbox}, Text: {text_snippet}")
+    
+    # Example 2: Get sections within a region using start/end elements
+    print("\n2. Get sections within a region using start/end elements")
+    
+    # Create a region from the bottom half of the page
+    bottom_half = page.create_region(0, page.height / 2, page.width, page.height)
+    print(f"Created region: {bottom_half.bbox}")
+    
+    # Method 1: Find heading elements first, then pass them to get_sections (old way)
+    headings = bottom_half.find_all('text[size>=12]')
+    print(f"Found {len(headings)} potential headings in the region")
+    
+    # Use headings as start elements and extract sections (old way)
+    sections1 = bottom_half.get_sections(start_elements=headings)
+    print(f"Found {len(sections1)} sections using explicit elements")
+    
+    # Method 2: Pass selector directly to start_elements (new way)
+    sections2 = bottom_half.get_sections(start_elements='text[size>=12]')
+    print(f"Found {len(sections2)} sections using selector string")
+    
+    # Display section details
+    for i, section in enumerate(sections2):
+        start_element = section.start_element
+        start_text = start_element.text if start_element else "None"
+        
+        text = section.extract_text()
+        text_snippet = text[:50] + "..." if len(text) > 50 else text
+        
+        print(f"  Section {i+1} (starts with '{start_text}'): {text_snippet}")
+    
+    # Example 3: Use selectors within a region
+    print("\n3. Get sections using selectors within a region")
+    
+    # Create a region from the center of the page
+    center = page.create_region(50, 50, page.width - 50, page.height - 50)
+    
+    # Get sections with start elements
+    sections1 = center.get_sections(
+        start_elements='text[size>=12]'  # Large text as section starts
+    )
+    
+    # Get sections with both start and end elements
+    sections2 = center.get_sections(
+        start_elements='text[size>=12]',  # Large text as section starts
+        end_elements='line[width>=1]'  # Thick lines as section ends
+    )
+    
+    print(f"Found {len(sections1)} sections using traditional selectors")
+    print(f"Found {len(sections2)} sections using direct selector strings")
+    
+    # Compare the results - they should be identical
+    print(f"Both approaches match: {len(sections1) == len(sections2)}")
+    
+    # Display section details for the new approach
+    for i, section in enumerate(sections2):
+        text = section.extract_text()
+        text_snippet = text[:50] + "..." if len(text) > 50 else text
+        print(f"  Section {i+1}: {text_snippet}")
+
+if __name__ == "__main__":
+    main()

examples/school_books.py

@@ -0,0 +1,49 @@
+"""
+Example demonstrating section extraction with the get_sections method.
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+
+pdf = PDF("./pdfs/Atlanta_Public_Schools_GA_sample.pdf")
+
+page = pdf.pages[0]
+day_sections = page.get_sections(start_elements='line[width>=2]')
+
+for day in day_sections:
+    date = day.find('text').text
+    book_sections = day.get_sections(start_elements='text:contains("(Removed:")')
+    for j, book in enumerate(book_sections):
+        print("-----")
+        if book.height < 30:
+            print("Not a book, skipping")
+            continue
+        book.highlight(label=f"Day {date} section {j}")
+
+        title = book.find_all('text[font_variant="AAAAAB"][size>=10]')
+        title.highlight(label='Title')
+        
+        price = book.find('text:contains("Price")').below(height=15, width="element").expand(right=30)
+        price.highlight(label='Price')
+
+        acquired = book.find('text:contains("Acquired")').below(height=15, width="element").expand(right=30)
+        acquired.highlight(label='Acquired')
+
+        removed_by = book.find('text[size<10]:contains("Removed")').below(height=17, width="element").expand(right=60)
+        removed_by.highlight(label='Removed By')
+
+        data = {
+            'Title': title.extract_text(),
+            'Price': price.extract_text(),
+            'Acquired': acquired.extract_text(),
+            'Removed By': removed_by.extract_text()
+        }
+        print(data)
+
+page.save("highlight.png", show_labels=True)