natural-pdf 25.3.16__py3-none-any.whl

examples/school_books_all.py

@@ -0,0 +1,52 @@
+"""
+Example demonstrating section extraction with the get_sections method.
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+
+pdf = PDF("./pdfs/Atlanta_Public_Schools_GA_sample.pdf")
+
+day_sections = pdf.pages.get_sections(start_elements='line[width>=2]')
+
+rows = []
+for day in day_sections:
+    date = day.find('text').text
+    book_sections = day.get_sections(start_elements='text:contains("(Removed:")')
+    
+    for book in book_sections:
+        if book.height < 30:
+            print("Not a book, skipping")
+            continue
+
+        # Bold big text is the title
+        title = book.find_all('text[font_variant="AAAAAB"][size>=10]')
+        price = book.find('text:contains("Price")').below(height=15, width="element").expand(right=30)
+        acquired = book.find('text:contains("Acquired")').below(height=15, width="element").expand(right=30)
+        removed_by = book.find('text[size<10]:contains("Removed")').below(height=17, width="element").expand(right=60)
+
+        # Highlight them
+        book.highlight(label=title.extract_text())
+        title.highlight(label='title')
+        price.highlight(label='price')
+        acquired.highlight(label='acquired')
+        removed_by.highlight(label='removed')
+
+        # Save them
+        data = {
+            'Title': title.extract_text(),
+            'Price': price.extract_text(),
+            'Acquired': acquired.extract_text(),
+            'Removed By': removed_by.extract_text()
+        }
+        rows.append(data)
+
+pdf.pages[0].save("highlight-1.png", show_labels=True)
+pdf.pages[1].save("highlight-2.png", show_labels=True)
+pdf.pages[2].save("highlight-3.png", show_labels=True)

examples/scouring.py

@@ -0,0 +1,36 @@
+"""
+Example demonstrating section extraction with the get_sections method.
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+
+pdf = PDF("./pdfs/Nigeria 2021_MICS_SFR_English.pdf")
+
+# Exclude "Page | 123" footer from all queries
+pdf.add_exclusion(lambda page: page.find_all('text').lowest().below(include_element=True))
+
+# There's a bold header for 'EQ.4.1W' on a few of these pages
+header = pdf.pages[460:470].find('text:contains("EQ.4.1W"):bold')
+
+header.highlight(label='table header')
+
+(
+    header
+        .below()
+        .find('text:contains("Total"):bold')
+        .below(
+            until='text:contains("MICS")',
+            include_element=True,
+            include_until=False
+        )
+        .highlight(label='table area')
+)
+
+header.page.to_image(path="output.png", show_labels=True)

examples/section_extraction_example.py

@@ -0,0 +1,232 @@
+"""
+Example demonstrating section extraction with the get_sections method.
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from natural_pdf import PDF
+
+
+def example_sections_between_headings(pdf_path):
+    """
+    Example showing how to extract sections between headings.
+    """
+    print("\n=== SECTIONS BETWEEN HEADINGS ===")
+    pdf = PDF(pdf_path)
+    page = pdf.pages[0]
+    page.add_exclusion(page.find('text:contains("November")').above(include_element=True))
+    
+    # Create an output directory
+    output_dir = Path(__file__).parent / "section_output"
+    output_dir.mkdir(exist_ok=True)
+    
+    # Find all the headings on the page
+    headings = page.find_all('text[size>=12]')
+    print(f"Found {len(headings)} heading elements")
+    
+    # Create a highlighted visualization to see what we found
+    page.clear_highlights()
+    headings.highlight(label="Headings", color=(255, 100, 0, 100))
+    page.to_image(path=str(output_dir / "headings.png"), show_labels=True)
+    
+    # First try without line grouping 
+    print("Extracting sections WITHOUT line grouping:")
+    # Set y_threshold to 0 to disable line grouping
+    sections_no_grouping = page.get_sections(
+        start_elements=headings,
+        boundary_inclusion='start',
+        y_threshold=0  # Disable line grouping
+    )
+    print(f"Found {len(sections_no_grouping)} sections without line grouping")
+    
+    # Now with line grouping
+    print("\nExtracting sections WITH line grouping:")
+    sections = page.get_sections(
+        start_elements=headings,
+        boundary_inclusion='start',  # Include heading with its section
+        y_threshold=5  # Group elements within 5 points vertically
+    )
+    print(f"Found {len(sections)} sections with line grouping")
+    
+    # Create visualizations showing the difference
+    # Highlight sections without grouping
+    page.clear_highlights()
+    for i, section in enumerate(sections_no_grouping):
+        section.highlight(label=f"Section {i+1}", use_color_cycling=True)
+    page.to_image(path=str(output_dir / "sections_no_grouping.png"), show_labels=True)
+    
+    # Highlight sections with grouping
+    page.clear_highlights()
+    for i, section in enumerate(sections):
+        section.highlight(label=f"Section {i+1}", use_color_cycling=True)
+    page.to_image(path=str(output_dir / "sections_with_grouping.png"), show_labels=True)
+    
+    # Process each section from the grouped version
+    for i, section in enumerate(sections):
+        # Get the heading text
+        heading_text = section.start_element.extract_text() if hasattr(section, 'start_element') else "No heading"
+        
+        # Get section content (limited to first 50 chars for display)
+        content = section.extract_text()
+        content_preview = content[:50] + "..." if len(content) > 50 else content
+        
+        print(f"Section {i+1}: '{heading_text}'")
+        print(f"  Content: {content_preview}")
+        
+        # Create visualization 
+        page.clear_highlights()
+        section.highlight(label=f"Section {i+1}")
+        if hasattr(section, 'start_element') and section.start_element:
+            section.start_element.highlight(label="Heading", color=(255, 0, 0, 100))
+        if hasattr(section, 'end_element') and section.end_element:
+            section.end_element.highlight(label="End", color=(0, 0, 255, 100))
+        
+        page.to_image(path=str(output_dir / f"section_{i+1}.png"), show_labels=True)
+
+
+def example_sections_with_separators(pdf_path):
+    """
+    Example showing how to extract sections with separators.
+    """
+    print("\n=== SECTIONS WITH SEPARATORS ===")
+    pdf = PDF(pdf_path)
+    page = pdf.pages[0]
+    
+    # Create an output directory
+    output_dir = Path(__file__).parent / "separator_output"
+    output_dir.mkdir(exist_ok=True)
+    
+    # Find all horizontal lines that could be separators
+    separators = page.find_all('line[width>=2]')
+    print(f"Found {len(separators)} separator lines")
+    
+    # Create a highlighted visualization to see what we found
+    page.clear_highlights()
+    separators.highlight(label="Separators", color=(0, 0, 255, 100))
+    page.to_image(path=str(output_dir / "separators.png"), show_labels=True)
+    
+    # Try different boundary inclusions
+    inclusion_options = ['none', 'start', 'end', 'both']
+    
+    for inclusion in inclusion_options:
+        print(f"\nSections with boundary_inclusion='{inclusion}':")
+        sections = page.get_sections(
+            start_elements=separators,
+            boundary_inclusion=inclusion
+        )
+        
+        print(f"Found {len(sections)} sections")
+        
+        # Create visualization for all sections
+        page.clear_highlights()
+        
+        for i, section in enumerate(sections):
+            # Use different color for each section
+            color = None  # Let it cycle through colors
+            section.highlight(label=f"Section {i+1}", use_color_cycling=True)
+            
+            # Section info
+            content = section.extract_text()
+            content_preview = content[:30] + "..." if len(content) > 30 else content
+            print(f"  Section {i+1}: {content_preview}")
+        
+        # Save the visualization
+        page.to_image(path=str(output_dir / f"sections_{inclusion}.png"), show_labels=True)
+        page.clear_highlights()
+
+
+def example_start_end_sections(pdf_path):
+    """
+    Example showing how to extract sections between start and end elements.
+    """
+    print("\n=== SECTIONS BETWEEN START AND END ELEMENTS ===")
+    pdf = PDF(pdf_path)
+    page = pdf.pages[0]
+    
+    # Create an output directory
+    output_dir = Path(__file__).parent / "start_end_output"
+    output_dir.mkdir(exist_ok=True)
+    
+    # Find headings and lines
+    headings = page.find_all('text[size>=14]')
+    lines = page.find_all('line[width>=2]')
+    
+    print(f"Found {len(headings)} headings and {len(lines)} lines")
+    
+    # Create a visualization to show both
+    page.clear_highlights()
+    headings.highlight(label="Headings", color=(255, 100, 0, 100))
+    lines.highlight(label="Lines", color=(0, 0, 255, 100))
+    page.to_image(path=str(output_dir / "elements.png"), show_labels=True)
+    
+    # Get sections from heading to next line
+    sections = page.get_sections(
+        start_elements=headings,
+        end_elements=lines,
+        boundary_inclusion='start'  # Include heading but not the line
+    )
+    
+    print(f"Found {len(sections)} sections from headings to lines")
+    
+    # Process each section
+    for i, section in enumerate(sections):
+        # Get the heading text
+        if hasattr(section, 'start_element') and section.start_element:
+            heading_text = section.start_element.extract_text()
+        else:
+            heading_text = "No heading"
+        
+        # Get section content
+        content = section.extract_text()
+        content_preview = content[:50] + "..." if len(content) > 50 else content
+        
+        print(f"Section {i+1}: '{heading_text}'")
+        print(f"  Content: {content_preview}")
+        
+        # Create visualization
+        page.clear_highlights()
+        section.highlight(label=f"Section {i+1}")
+        if hasattr(section, 'start_element') and section.start_element:
+            section.start_element.highlight(label="Heading", color=(255, 0, 0, 100))
+        if hasattr(section, 'end_element') and section.end_element:
+            section.end_element.highlight(label="Line", color=(0, 0, 255, 100))
+        
+        page.to_image(path=str(output_dir / f"section_{i+1}.png"), show_labels=True)
+
+
+def main():
+    """Main entry point."""
+    # Get the PDF path from command line or use a default
+    if len(sys.argv) > 1:
+        pdf_path = sys.argv[1]
+    else:
+        # Look for any PDF in the examples directory or pdfs directory
+        example_dir = Path(__file__).parent
+        pdf_files = list(example_dir.glob("*.pdf"))
+        
+        if not pdf_files:
+            pdfs_dir = example_dir.parent / "pdfs"
+            if pdfs_dir.exists():
+                pdf_files = list(pdfs_dir.glob("*.pdf"))
+        
+        if pdf_files:
+            pdf_path = str(pdf_files[0])
+        else:
+            print("No PDF file found. Please provide a path to a PDF file.")
+            sys.exit(1)
+    
+    print(f"Using PDF: {pdf_path}")
+    
+    # Run the examples
+    example_sections_between_headings(pdf_path)
+    example_sections_with_separators(pdf_path)
+    example_start_end_sections(pdf_path)
+
+
+if __name__ == "__main__":
+    main()

examples/simple_document_qa.py

@@ -0,0 +1,97 @@
+"""
+Simple demonstration of document QA functionality in Natural PDF.
+"""
+
+import os
+import sys
+import argparse
+
+# Add parent directory to path to run without installing
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from natural_pdf import PDF, configure_logging
+import logging
+
+def main():
+    # Set up logging
+    configure_logging(level=logging.INFO)
+    
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(description="Simple Document QA Example")
+    parser.add_argument("pdf_path", nargs="?", default="../pdfs/0500000US42001.pdf", 
+                      help="Path to PDF document")
+    parser.add_argument("question", nargs="?", default="How many votes for Harris and Walz?",
+                      help="Question to ask about the document")
+    parser.add_argument("--debug", action="store_true",
+                      help="Save debug information for troubleshooting")
+    args = parser.parse_args()
+    
+    # Open the PDF
+    pdf = PDF(args.pdf_path)
+    print(f"Loaded PDF: {args.pdf_path} ({len(pdf.pages)} pages)")
+    
+    # Get the first page
+    page = pdf.pages[0]
+    
+    # Ask a question to the document
+    print(f"\nAsking document: {args.question}")
+    doc_result = pdf.ask(args.question, debug=args.debug)
+    
+    if doc_result.get("found", False):
+        print(f"Document answer: {doc_result['answer']}")
+        print(f"Confidence: {doc_result['confidence']:.2f}")
+        print(f"Page: {doc_result.get('page_num', 0)}")
+    else:
+        print(f"No answer found in document: {doc_result.get('message', '')}")
+    
+    # Ask the same question to the page
+    print(f"\nAsking page 0: {args.question}")
+    page_result = page.ask(args.question, debug=args.debug)
+    
+    if page_result.get("found", False):
+        print(f"Page answer: {page_result['answer']}")
+        print(f"Confidence: {page_result['confidence']:.2f}")
+        
+        # Highlight the answer elements if available
+        if page_result.get("source_elements"):
+            for element in page_result["source_elements"]:
+                element.highlight(color=(1, 0.5, 0, 0.5))  # Orange highlight
+            
+            # Save the highlighted image
+            os.makedirs("output", exist_ok=True)
+            page.save_image("output/simple_qa_answer.png")
+            print("Saved highlighted answer to output/simple_qa_answer.png")
+    else:
+        print(f"No answer found on page: {page_result.get('message', '')}")
+    
+    # Optional: Analyze layout and ask questions to specific regions
+    print("\nDetecting document layout...")
+    page.analyze_layout(confidence=0.3)
+    regions = page.find_all('region[type=title], region[type=plain-text], region[type=table]')
+    print(f"Found {len(regions)} relevant regions")
+    
+    # Save layout visualization
+    page.highlight_layout()
+    page.save_image("output/simple_qa_regions.png")
+    print("Saved layout visualization to output/simple_qa_regions.png")
+    
+    # Ask questions to each region
+    best_region_result = None
+    best_confidence = 0
+    
+    for i, region in enumerate(regions):
+        region_result = region.ask(args.question, debug=args.debug)
+        
+        if region_result.get("found", False) and region_result.get("confidence", 0) > best_confidence:
+            best_region_result = region_result
+            best_confidence = region_result["confidence"]
+    
+    if best_region_result:
+        region_type = best_region_result["region"].region_type
+        print(f"\nBest region answer ({region_type}): {best_region_result['answer']}")
+        print(f"Confidence: {best_region_result['confidence']:.2f}")
+    else:
+        print("\nNo answer found in any region")
+
+if __name__ == "__main__":
+    main()

examples/spatial_navigation_example.py

@@ -0,0 +1,108 @@
+"""
+Spatial navigation example.
+
+This example demonstrates how to navigate between elements using
+spatial navigation methods: next(), prev(), and nearest().
+"""
+import os
+import sys
+
+# Add the parent directory to the Python path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from natural_pdf import PDF
+
+# Get the current directory of this script
+script_dir = os.path.dirname(os.path.realpath(__file__))
+# Get the parent directory (project root)
+root_dir = os.path.dirname(script_dir)
+# Default PDF path
+default_pdf_path = os.path.join(root_dir, "pdfs", "01-practice.pdf")
+# Output directory
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+# Get PDF path from command line or use default
+pdf_path = sys.argv[1] if len(sys.argv) > 1 else default_pdf_path
+print(f"Using PDF: {pdf_path}")
+
+# Load the PDF
+pdf = PDF(pdf_path)
+page = pdf.pages[0]
+
+print("\n=== Spatial Navigation Examples ===")
+
+# First, find a heading or title to start with
+title = page.find('text[size>=12]')
+if title:
+    print(f"\nStarting with: '{title.text}'")
+    
+    # 1. Find the next element in reading order
+    print("\n--- Next Element ---")
+    next_element = title.next()
+    if next_element:
+        print(f"Next element: '{next_element.text if hasattr(next_element, 'text') else next_element.type}'")
+    
+    # 2. Find the next element matching a selector
+    print("\n--- Next Matching Element ---")
+    next_bold = title.next('text:bold', limit=20)
+    if next_bold:
+        print(f"Next bold text: '{next_bold.text}'")
+    
+    # 3. Find the previous element in reading order
+    print("\n--- Previous Element ---")
+    prev_element = title.prev()
+    if prev_element:
+        print(f"Previous element: '{prev_element.text if hasattr(prev_element, 'text') else prev_element.type}'")
+    
+    # 4. Find the previous element matching a selector
+    print("\n--- Previous Matching Element ---")
+    # Find a element further down first 
+    middle_element = page.find_all('text')[len(page.find_all('text'))//2]
+    if middle_element:
+        print(f"Middle element: '{middle_element.text}'")
+        prev_large = middle_element.prev('text[size>=12]')
+        if prev_large:
+            print(f"Previous large text: '{prev_large.text}'")
+    
+    # 5. Find the nearest element matching a selector
+    print("\n--- Nearest Element ---")
+    nearest_rect = title.nearest('rect')
+    if nearest_rect:
+        print(f"Nearest rectangle: {nearest_rect.bbox}")
+    
+    # 6. Find the nearest element with max distance
+    print("\n--- Nearest Element with Max Distance ---")
+    nearest_small = title.nearest('text[size<10]', max_distance=100)
+    if nearest_small:
+        print(f"Nearest small text within 100 points: '{nearest_small.text}'")
+    else:
+        print("No small text within 100 points")
+    
+    # Visualize the navigation
+    print("\n--- Visualizing Navigation ---")
+    page.clear_highlights()
+    
+    # Highlight the starting element
+    title.highlight(label="Starting Element")
+    
+    # Find and highlight the next few elements
+    current = title
+    for i in range(5):
+        next_elem = current.next()
+        if next_elem:
+            next_elem.highlight(label=f"Next {i+1}")
+            current = next_elem
+        else:
+            break
+    
+    # Find and highlight the nearest elements
+    title.nearest('rect').highlight(label="Nearest Rectangle")
+    title.nearest('line').highlight(label="Nearest Line")
+    
+    # Save the visualization
+    output_path = os.path.join(output_dir, "spatial_navigation.png")
+    page.to_image(path=output_path, show_labels=True)
+    print(f"Saved visualization to {output_path}")
+    
+else:
+    print("Could not find a title to start with.")

examples/table_extraction_example.py

@@ -0,0 +1,135 @@
+"""
+Table extraction example using both TATR and pdfplumber methods.
+
+This example demonstrates how to extract tables from PDF documents
+using both the Table Transformer (TATR) structure detection and
+pdfplumber's table extraction methods.
+
+Note: This example requires additional dependencies:
+    - torch
+    - torchvision
+    - transformers
+    
+These will be automatically installed when you install natural-pdf.
+"""
+import os
+from natural_pdf import PDF
+import pprint
+
+# Get the current directory of this script
+script_dir = os.path.dirname(os.path.realpath(__file__))
+# Get the parent directory (project root)
+root_dir = os.path.dirname(script_dir)
+# Setup paths
+pdf_path = os.path.join(root_dir, "pdfs", "01-practice.pdf")
+output_dir = os.path.join(root_dir, "output")
+os.makedirs(output_dir, exist_ok=True)
+
+print(f"Extracting tables from: {pdf_path}")
+
+# Load the PDF
+pdf = PDF(pdf_path)
+page = pdf.pages[0]  # This PDF has a single page with a table
+
+# First, let's try the traditional pdfplumber method
+print("\n== TRADITIONAL TABLE EXTRACTION ==")
+table_plumber = page.extract_table()  # Uses pdfplumber's table extraction
+print("PDFPlumber extracted table:")
+pprint.pprint(table_plumber)
+
+# Now, let's detect and extract using TATR
+print("\n== TABLE TRANSFORMER (TATR) EXTRACTION ==")
+
+# Run table structure detection
+print("Running Table Transformer detection...")
+tatr_regions = page.analyze_layout(
+    model="tatr",
+    confidence=0.4  # Table detection confidence threshold
+)
+
+# Find the detected table
+tables = page.find_all('region[type=table][model=tatr]')
+
+if tables:
+    print(f"Found {len(tables)} tables")
+    
+    # Get the first table
+    table = tables[0]
+    
+    # Now extract the table using TATR structure (auto-detected)
+    tatr_table_data = table.extract_table()  # Automatically uses TATR because it's a TATR region
+    print("\nExtracted table data (TATR auto-detection):")
+    pprint.pprint(tatr_table_data)
+    
+    # You can also explicitly specify which method to use
+    plumber_table_data = table.extract_table(method='plumber')
+    print("\nExtracted table data (explicit pdfplumber method):")
+    pprint.pprint(plumber_table_data)
+    
+    # Compare the results
+    print("\n== EXTRACTION METHOD COMPARISON ==")
+    print(f"TATR rows: {len(tatr_table_data)}, cols in first row: {len(tatr_table_data[0]) if tatr_table_data else 0}")
+    print(f"Plumber rows: {len(plumber_table_data)}, cols in first row: {len(plumber_table_data[0]) if plumber_table_data else 0}")
+    
+    # Visualize the table structure
+    page.clear_highlights()
+    
+    # First highlight the table
+    table.highlight(label="Table", color=(1, 0, 0, 0.3))
+    
+    # Then highlight the structure elements
+    rows = page.find_all('region[type=table-row][model=tatr]')
+    columns = page.find_all('region[type=table-column][model=tatr]')
+    headers = page.find_all('region[type=table-column-header][model=tatr]')
+    
+    for row in rows:
+        row.highlight(label="Row", color=(0, 1, 0, 0.3))
+    for column in columns:
+        column.highlight(label="Column", color=(0, 0, 1, 0.3))
+    for header in headers:
+        header.highlight(label="Header", color=(0, 1, 1, 0.3))
+    
+    # Save the highlighted table structure
+    output_path = os.path.join(output_dir, "table_extraction.png")
+    page.to_image(path=output_path, show_labels=True)
+    print(f"\nSaved table structure visualization to {output_path}")
+    
+    # Demonstrate working with individual cells
+    if rows and columns:
+        print("\n== EXTRACTING INDIVIDUAL CELLS ==")
+        # Create a cell at the intersection of first row and first column
+        from natural_pdf.elements.region import Region
+        
+        row = rows[0]
+        col = columns[0]
+        
+        cell_bbox = (col.x0, row.top, col.x1, row.bottom)
+        cell = Region(page, cell_bbox)
+        
+        cell_text = cell.extract_text().strip()
+        print(f"Text in first cell: '{cell_text}'")
+    
+    # When working with tables with headers, you might want to create a dictionary
+    if headers and rows and columns:
+        print("\n== CREATING A DICTIONARY FROM TABLE ==")
+        header_texts = [header.extract_text().strip() for header in headers]
+        
+        table_dict = []
+        for row in rows:
+            row_dict = {}
+            for i, col in enumerate(columns):
+                if i < len(header_texts):
+                    # Create cell region
+                    cell_bbox = (col.x0, row.top, col.x1, row.bottom)
+                    cell = Region(page, cell_bbox)
+                    
+                    # Extract text and add to dictionary
+                    row_dict[header_texts[i]] = cell.extract_text().strip()
+            
+            if row_dict:
+                table_dict.append(row_dict)
+        
+        print("Table as dictionary:")
+        pprint.pprint(table_dict)
+else:
+    print("No tables detected with TATR.")