natural-pdf 0.2.18__py3-none-any.whl → 0.2.20__py3-none-any.whl

temp/demo_text_closest.py

@@ -0,0 +1,66 @@
+"""
+Demo of the text:closest() selector for fuzzy text matching in Natural PDF.
+
+This selector is designed to handle OCR errors and text variations by:
+1. First finding exact substring matches
+2. Then ranking other elements by similarity score
+"""
+
+from natural_pdf import PDF
+
+# Load a PDF
+pdf = PDF("pdfs/01-practice.pdf")
+page = pdf.pages[0]
+
+print("=== text:closest() Selector Demo ===\n")
+
+# Example 1: Basic fuzzy matching (default threshold 0.0 - all elements)
+print("1. Find elements closest to 'Durham' (no threshold):")
+results = page.find_all('text:closest("Durham")')
+print(f"   Found {len(results)} elements (all text elements)")
+print(f"   First 3: {[r.text.strip() for r in results[:3]]}\n")
+
+# Example 2: With similarity threshold
+print("2. Find elements with at least 40% similarity to 'Durham':")
+results = page.find_all('text:closest("Durham@0.4")')
+print(f"   Found {len(results)} elements")
+for r in results:
+    print(f"   - '{r.text.strip()}'")
+print()
+
+# Example 3: OCR error simulation
+print("3. Simulate OCR errors:")
+print("   Searching for 'Durharn' (OCR error: 'rn' instead of 'm'):")
+results = page.find_all('text:closest("Durharn@0.4")')
+print(f"   Found: {[r.text.strip() for r in results if 'Durham' in r.text]}\n")
+
+# Example 4: Case insensitive matching (default)
+print("4. Case insensitive search for 'chicago':")
+results = page.find_all('text:closest("chicago@0.6")')
+print(f"   Found: {[r.text.strip() for r in results if 'Chicago' in r.text]}\n")
+
+# Example 5: Combining with other selectors
+print("5. Find fuzzy matches with size constraints:")
+results = page.find_all('text:closest("Violation@0.6")[size>10]')
+print(f"   Found {len(results)} elements with size > 10")
+if results:
+    print(f"   Example: '{results[0].text.strip()}' (size={results[0].size})\n")
+
+# Example 6: Practical use case - finding labels
+print("6. Practical OCR use case - finding form labels:")
+print("   Looking for 'Date:' even if OCR missed the colon:")
+results = page.find_all('text:closest("Date@0.8")')
+date_labels = [r for r in results if "Date" in r.text]
+if date_labels:
+    print(f"   Found: '{date_labels[0].text.strip()}'")
+    # Now find the value to the right
+    value = date_labels[0].right(until='text')
+    print(f"   Value: '{value.extract_text().strip()}'")
+
+print("\n=== Key Features ===")
+print("- Default threshold is 0.0 (matches all elements, sorted by similarity)")
+print("- Exact substring matches always come first")
+print("- Case insensitive by default (use case=True for case sensitive)")
+print("- Threshold specified with @ separator: 'search@0.8'")
+print("- Uses Python's difflib.SequenceMatcher for similarity calculation")
+print("- Empty search string returns no results")

temp/inspect_model.py

@@ -0,0 +1,43 @@
+import torch
+
+model_path = "/Users/soma/Development/natural-pdf/model-weights/checkbox-nano.pt"
+
+# Load checkpoint
+checkpoint = torch.load(model_path, map_location='cpu', weights_only=False)
+
+print("Checkpoint keys:", checkpoint.keys())
+print("\nArgs:", checkpoint['args'])
+
+# Look at model structure
+model = checkpoint['model']
+print(f"\nModel type: {type(model)}")
+
+# If it's an OrderedDict, it's just the state dict
+if isinstance(model, dict):
+    print(f"Model has {len(model)} weight tensors")
+    # Look at first few keys to understand architecture
+    for i, key in enumerate(list(model.keys())[:10]):
+        print(f"  {key}")
+        
+# Check the args to understand what model this is
+args = checkpoint['args']
+print(f"\nModel configuration from args:")
+for attr in ['model', 'task', 'mode', 'imgsz', 'batch', 'device']:
+    if hasattr(args, attr):
+        print(f"  {attr}: {getattr(args, attr)}")
+        
+# Try to determine RT-DETR variant
+if hasattr(args, 'model'):
+    model_name = getattr(args, 'model')
+    print(f"\nModel name: {model_name}")
+    
+    # RT-DETR variants mapping
+    if 'rtdetr' in str(model_name).lower():
+        if '18' in str(model_name) or 'r18' in str(model_name):
+            print("This appears to be RT-DETR with ResNet-18 backbone")
+        elif '34' in str(model_name) or 'r34' in str(model_name):
+            print("This appears to be RT-DETR with ResNet-34 backbone")
+        elif '50' in str(model_name) or 'r50' in str(model_name):
+            print("This appears to be RT-DETR with ResNet-50 backbone")
+        elif '101' in str(model_name) or 'r101' in str(model_name):
+            print("This appears to be RT-DETR with ResNet-101 backbone")

temp/rtdetr_dinov2_test.py

@@ -0,0 +1,49 @@
+"""Test using RT-DETR with DINOv2 backbone in transformers"""
+from transformers import RTDetrConfig, RTDetrForObjectDetection
+import torch
+
+# Create config with DINOv2 backbone
+config = RTDetrConfig(
+    # Model architecture
+    backbone="facebook/dinov2-small",  # DINOv2 small variant
+    use_pretrained_backbone=False,  # We'll load our weights
+    backbone_kwargs={
+        "out_indices": [3, 6, 9, 12],  # Match the indices from your model
+    },
+    
+    # Detection head config (from your checkpoint)
+    num_labels=3,  # checked, unchecked, (background?)
+    hidden_dim=256,
+    num_queries=300,
+    decoder_layers=2,
+    d_model=256,
+    dim_feedforward=2048,
+    dropout=0.0,
+    nheads=8,  # sa_nheads from your model
+    
+    # Loss coefficients
+    bbox_loss_coefficient=5.0,
+    giou_loss_coefficient=2.0,
+    cls_loss_coefficient=1.0,
+)
+
+# Initialize model
+model = RTDetrForObjectDetection(config)
+
+print(f"Model initialized with {sum(p.numel() for p in model.parameters())} parameters")
+
+# Your checkpoint has these keys in the state dict
+# We'd need to map them to the RT-DETR expected keys
+checkpoint = torch.load("model-weights/checkbox-nano.pt", map_location='cpu', weights_only=False)
+state_dict = checkpoint['model']
+
+# Show some key mappings needed
+print("\nYour model keys (first 5):")
+for key in list(state_dict.keys())[:5]:
+    print(f"  {key}")
+    
+print("\nRT-DETR expects keys like:")
+for key in list(model.state_dict().keys())[:5]:
+    print(f"  {key}")
+    
+print("\nWould need to create a mapping between these formats")

temp/test_closest_debug.py

@@ -0,0 +1,26 @@
+from natural_pdf import PDF
+from natural_pdf.selectors.parser import parse_selector, selector_to_filter_func
+
+# Test parsing
+selector_str = 'text:closest("Durham@0.7")'
+parsed = parse_selector(selector_str)
+print("Parsed selector:", parsed)
+
+# Test with actual PDF
+pdf = PDF('pdfs/01-practice.pdf')
+page = pdf.pages[0]
+
+# Get all text elements
+all_text = page.find_all('text')
+print(f"\nTotal text elements: {len(all_text)}")
+
+# Test the selector
+results = page.find_all('text:closest("Durham@0.7")')
+print(f"Results with :closest selector: {len(results)}")
+
+# Let's manually test the filter function
+filter_func = selector_to_filter_func(parsed)
+print("\nTesting filter function manually:")
+for i, el in enumerate(all_text[:5]):
+    match = filter_func(el)
+    print(f"  Element {i}: '{el.text}' -> {match}")

temp/test_closest_debug2.py

@@ -0,0 +1,22 @@
+from natural_pdf import PDF
+from natural_pdf.selectors.parser import parse_selector, _build_filter_list
+import difflib
+
+# Test parsing
+selector_str = 'text:closest("Durham@0.7")'
+parsed = parse_selector(selector_str)
+print("Parsed selector:", parsed)
+print("Args:", repr(parsed['pseudo_classes'][0]['args']))
+
+# Build filters
+filters = _build_filter_list(parsed)
+print("\nFilters built:", filters)
+
+# Test similarity calculation manually
+search_text = "Durham"
+test_texts = ["Durham's Meatpacking  ", "Chicago, Ill.", "Violations"]
+
+print(f"\nTesting similarity for '{search_text}':")
+for text in test_texts:
+    ratio = difflib.SequenceMatcher(None, search_text.lower(), text.lower()).ratio()
+    print(f"  '{text}' -> ratio: {ratio:.3f}")

temp/test_context_exploration.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""
+Test to explore how context managers might work with Natural PDF
+"""
+
+import natural_pdf as npdf
+from contextlib import contextmanager
+import threading
+
+# Test 1: Check current global option system
+print("=== Current Global Options ===")
+print(f"Default directional_offset: {npdf.options.layout.directional_offset}")
+print(f"Default auto_multipage: {npdf.options.layout.auto_multipage}")
+
+# Test 2: See how options are used in directional methods
+print("\n=== How Options Are Used ===")
+# Let's trace through the code to see how offset is used
+
+# Test 3: Prototype a context manager approach
+@contextmanager
+def temporary_options(**kwargs):
+    """Context manager to temporarily change options."""
+    # Store original values
+    original_values = {}
+    
+    for key, value in kwargs.items():
+        parts = key.split('.')
+        obj = npdf.options
+        
+        # Navigate to the right section
+        for part in parts[:-1]:
+            obj = getattr(obj, part)
+        
+        # Store original and set new value
+        final_key = parts[-1]
+        original_values[key] = getattr(obj, final_key)
+        setattr(obj, final_key, value)
+    
+    try:
+        yield
+    finally:
+        # Restore original values
+        for key, original_value in original_values.items():
+            parts = key.split('.')
+            obj = npdf.options
+            
+            for part in parts[:-1]:
+                obj = getattr(obj, part)
+            
+            final_key = parts[-1]
+            setattr(obj, final_key, original_value)
+
+# Test the context manager
+print("\n=== Testing Context Manager ===")
+print(f"Before: offset={npdf.options.layout.directional_offset}")
+
+with temporary_options(**{'layout.directional_offset': 5.0}):
+    print(f"Inside context: offset={npdf.options.layout.directional_offset}")
+
+print(f"After: offset={npdf.options.layout.directional_offset}")
+
+# Test 4: Check thread safety
+print("\n=== Thread Safety Test ===")
+results = []
+
+def thread_func(thread_id, offset_value):
+    with temporary_options(**{'layout.directional_offset': offset_value}):
+        import time
+        time.sleep(0.1)  # Simulate some work
+        results.append((thread_id, npdf.options.layout.directional_offset))
+
+threads = []
+for i in range(3):
+    t = threading.Thread(target=thread_func, args=(i, float(i * 10)))
+    threads.append(t)
+    t.start()
+
+for t in threads:
+    t.join()
+
+print("Thread results (thread_id, offset):")
+for result in sorted(results):
+    print(f"  Thread {result[0]}: offset={result[1]}")
+
+print("\nConclusion: Global options are NOT thread-safe!")

temp/test_durham.py

@@ -0,0 +1,30 @@
+from natural_pdf import PDF
+import difflib
+
+pdf = PDF('pdfs/01-practice.pdf')
+page = pdf.pages[0]
+
+# Find all text elements
+all_text = page.find_all('text')
+print(f"Total text elements: {len(all_text)}")
+
+# Look for Durham-related text
+print("\nLooking for Durham-related text:")
+for el in all_text:
+    if "Durham" in el.text:
+        print(f"  Found: '{el.text}'")
+
+# Test similarity
+search_text = "Durharn"  # OCR error: rn instead of m
+print(f"\nTesting similarity for '{search_text}':")
+for el in all_text[:20]:  # Just check first 20
+    if el.text:
+        ratio = difflib.SequenceMatcher(None, search_text.lower(), el.text.lower()).ratio()
+        if ratio > 0.3:
+            print(f"  '{el.text}' -> ratio: {ratio:.3f}")
+
+# Test the actual selector
+results = page.find_all('text:closest("Durharn@0.4")')
+print(f"\nResults with threshold 0.4: {len(results)}")
+for r in results[:5]:
+    print(f"  - {r.text}")

temp/test_empty_string.py

@@ -0,0 +1,16 @@
+from natural_pdf import PDF
+from natural_pdf.selectors.parser import parse_selector
+
+# Test parsing empty string
+selector_str = 'text:closest("")'
+parsed = parse_selector(selector_str)
+print("Parsed selector:", parsed)
+print("Args:", repr(parsed['pseudo_classes'][0]['args']))
+
+# Test with actual PDF
+pdf = PDF('pdfs/01-practice.pdf')
+page = pdf.pages[0]
+
+# Try the selector
+results = page.find_all('text:closest("")')
+print(f"Results with empty string: {len(results)}")

temp/test_similarity.py

@@ -0,0 +1,15 @@
+import difflib
+
+# Test similarity calculation
+pairs = [
+    ("Durham", "Durham's Meatpacking  "),
+    ("durham", "durham's meatpacking  "),  # lowercase
+    ("Chicgo", "Chicago, Ill."),
+    ("chicgo", "chicago, ill."),  # lowercase
+    ("Chicago", "Chicago, Ill."),
+]
+
+print("Similarity ratios:")
+for search, text in pairs:
+    ratio = difflib.SequenceMatcher(None, search, text).ratio()
+    print(f"  '{search}' vs '{text}' -> {ratio:.3f}")