natural-pdf 0.1.32__tar.gz → 0.1.33__tar.gz

{natural_pdf-0.1.32 → natural_pdf-0.1.33}/.gitignore

@@ -1,3 +1,4 @@
+Untitled*.ipynb
 importtime_output.txt
 .notebook_cache.json
 .venv

{natural_pdf-0.1.32/natural_pdf.egg-info → natural_pdf-0.1.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: natural-pdf
-Version: 0.1.32
+Version: 0.1.33
 Summary: A more intuitive interface for working with PDFs
 Author-email: Jonathan Soma <jonathan.soma@gmail.com>
 License-Expression: MIT

{natural_pdf-0.1.32 → natural_pdf-0.1.33}/natural_pdf/core/element_manager.py

@@ -146,7 +146,7 @@ class ElementManager:
     contained in the Page class, providing better separation of concerns.
     """
 
-    def __init__(self, page, font_attrs=None):
+    def __init__(self, page, font_attrs=None, load_text: bool = True):
         """
         Initialize the ElementManager.
 
@@ -156,9 +156,11 @@ class ElementManager:
                        Default: ['fontname', 'size', 'bold', 'italic']
                        None: Only consider spatial relationships
                        List: Custom attributes to consider
+            load_text: Whether to load text elements from the PDF (default: True).
         """
         self._page = page
         self._elements = None  # Lazy-loaded
+        self._load_text = load_text
         # Default to splitting by fontname, size, bold, italic if not specified
         # Renamed internal variable for clarity
         self._word_split_attributes = (
@@ -175,11 +177,15 @@ class ElementManager:
 
         logger.debug(f"Page {self._page.number}: Loading elements...")
 
-        # 1. Prepare character dictionaries (native + OCR) with necessary attributes
-        prepared_char_dicts = self._prepare_char_dicts()
-        logger.debug(
-            f"Page {self._page.number}: Prepared {len(prepared_char_dicts)} character dictionaries."
-        )
+        # 1. Prepare character dictionaries only if loading text
+        if self._load_text:
+            prepared_char_dicts = self._prepare_char_dicts()
+            logger.debug(
+                f"Page {self._page.number}: Prepared {len(prepared_char_dicts)} character dictionaries."
+            )
+        else:
+            prepared_char_dicts = []
+            logger.debug(f"Page {self._page.number}: Skipping text loading (load_text=False)")
 
         # -------------------------------------------------------------
         # Detect strikethrough (horizontal strike-out lines) on raw
@@ -189,61 +195,75 @@ class ElementManager:
         # belong to the same word.
         # -------------------------------------------------------------
 
-        try:
-            self._mark_strikethrough_chars(prepared_char_dicts)
-        except Exception as strike_err:  # pragma: no cover – strike detection must never crash loading
-            logger.warning(
-                f"Page {self._page.number}: Strikethrough detection failed – {strike_err}",
-                exc_info=True,
-            )
+        if self._load_text and prepared_char_dicts:
+            try:
+                self._mark_strikethrough_chars(prepared_char_dicts)
+            except Exception as strike_err:  # pragma: no cover – strike detection must never crash loading
+                logger.warning(
+                    f"Page {self._page.number}: Strikethrough detection failed – {strike_err}",
+                    exc_info=True,
+                )
 
         # -------------------------------------------------------------
         # Detect underlines on raw characters (must come after strike so
         # both attributes are present before word grouping).
         # -------------------------------------------------------------
 
-        try:
-            self._mark_underline_chars(prepared_char_dicts)
-        except Exception as u_err:  # pragma: no cover
-            logger.warning(
-                f"Page {self._page.number}: Underline detection failed – {u_err}",
-                exc_info=True,
-            )
+        if self._load_text and prepared_char_dicts:
+            try:
+                self._mark_underline_chars(prepared_char_dicts)
+            except Exception as u_err:  # pragma: no cover
+                logger.warning(
+                    f"Page {self._page.number}: Underline detection failed – {u_err}",
+                    exc_info=True,
+                )
 
         # Detect highlights
-        try:
-            self._mark_highlight_chars(prepared_char_dicts)
-        except Exception as h_err:
-            logger.warning(
-                f"Page {self._page.number}: Highlight detection failed – {h_err}",
-                exc_info=True,
-            )
+        if self._load_text and prepared_char_dicts:
+            try:
+                self._mark_highlight_chars(prepared_char_dicts)
+            except Exception as h_err:
+                logger.warning(
+                    f"Page {self._page.number}: Highlight detection failed – {h_err}",
+                    exc_info=True,
+                )
 
         # Create a mapping from character dict to index for efficient lookup
-        char_to_index = {}
-        for idx, char_dict in enumerate(prepared_char_dicts):
-            key = (
-                char_dict.get("x0", 0),
-                char_dict.get("top", 0),
-                char_dict.get("text", ""),
-            )
-            char_to_index[key] = idx
+        if self._load_text:
+            char_to_index = {}
+            for idx, char_dict in enumerate(prepared_char_dicts):
+                key = (
+                    char_dict.get("x0", 0),
+                    char_dict.get("top", 0),
+                    char_dict.get("text", ""),
+                )
+                char_to_index[key] = idx
+        else:
+            char_to_index = {}
 
         # 2. Instantiate the custom word extractor
         # Prefer page-level config over PDF-level for tolerance lookup
+        word_elements: List[TextElement] = []
+        
+        # Get config objects (needed for auto_text_tolerance check)
         page_config = getattr(self._page, "_config", {})
         pdf_config = getattr(self._page._parent, "_config", {})
-
-        # Start with any explicitly supplied tolerances (may be None)
-        xt = page_config.get("x_tolerance", pdf_config.get("x_tolerance"))
-        yt = page_config.get("y_tolerance", pdf_config.get("y_tolerance"))
+        
+        # Initialize tolerance variables
+        xt = None
+        yt = None
         use_flow = pdf_config.get("use_text_flow", False)
+        
+        if self._load_text and prepared_char_dicts:
+            # Start with any explicitly supplied tolerances (may be None)
+            xt = page_config.get("x_tolerance", pdf_config.get("x_tolerance"))
+            yt = page_config.get("y_tolerance", pdf_config.get("y_tolerance"))
 
         # ------------------------------------------------------------------
         # Auto-adaptive tolerance: scale based on median character size when
         # requested and explicit values are absent.
         # ------------------------------------------------------------------
-        if pdf_config.get("auto_text_tolerance", True):
+        if self._load_text and pdf_config.get("auto_text_tolerance", True):
             import statistics
 
             sizes = [c.get("size", 0) for c in prepared_char_dicts if c.get("size")]
@@ -323,7 +343,6 @@ class ElementManager:
                 current_line_key = line_key
             lines[-1].append(char_dict)
 
-        word_elements: List[TextElement] = []
         # Process each line separately with direction detection
         for line_chars in lines:
             if not line_chars:
@@ -480,7 +499,8 @@ class ElementManager:
                         except Exception:
                             w._obj["highlight_color"] = dominant_color
 
-        generated_words = word_elements
+        # generated_words defaults to empty list if text loading is disabled
+        generated_words = word_elements if self._load_text else []
         logger.debug(
             f"Page {self._page.number}: Generated {len(generated_words)} words using NaturalWordExtractor."
         )

{natural_pdf-0.1.32 → natural_pdf-0.1.33}/natural_pdf/core/page.py

@@ -101,7 +101,7 @@ class Page(ClassificationMixin, ExtractionMixin, ShapeDetectionMixin, DescribeMi
     with improved selection, navigation, extraction, and question-answering capabilities.
     """
 
-    def __init__(self, page: "pdfplumber.page.Page", parent: "PDF", index: int, font_attrs=None):
+    def __init__(self, page: "pdfplumber.page.Page", parent: "PDF", index: int, font_attrs=None, load_text: bool = True):
         """
         Initialize a page wrapper.
 
@@ -110,10 +110,12 @@ class Page(ClassificationMixin, ExtractionMixin, ShapeDetectionMixin, DescribeMi
             parent: Parent PDF object
             index: Index of this page in the PDF (0-based)
             font_attrs: Font attributes to consider when grouping characters into words.
+            load_text: Whether to load text elements from the PDF (default: True).
         """
         self._page = page
         self._parent = parent
         self._index = index
+        self._load_text = load_text
         self._text_styles = None  # Lazy-loaded text style analyzer results
         self._exclusions = []  # List to store exclusion functions/regions
         self._skew_angle: Optional[float] = None  # Stores detected skew angle
@@ -136,7 +138,7 @@ class Page(ClassificationMixin, ExtractionMixin, ShapeDetectionMixin, DescribeMi
         self._config = dict(getattr(self._parent, "_config", {}))
 
         # Initialize ElementManager, passing font_attrs
-        self._element_mgr = ElementManager(self, font_attrs=font_attrs)
+        self._element_mgr = ElementManager(self, font_attrs=font_attrs, load_text=self._load_text)
         # self._highlighter = HighlightingService(self) # REMOVED - Use property accessor
         # --- NEW --- Central registry for analysis results
         self.analyses: Dict[str, Any] = {}
@@ -2998,6 +3000,29 @@ class Page(ClassificationMixin, ExtractionMixin, ShapeDetectionMixin, DescribeMi
         """
         return self.find_all('*').inspect(limit=limit)
 
+    def remove_text_layer(self) -> "Page":
+        """
+        Remove all text elements from this page.
+        
+        This removes all text elements (words and characters) from the page,
+        effectively clearing the text layer.
+        
+        Returns:
+            Self for method chaining
+        """
+        logger.info(f"Page {self.number}: Removing all text elements...")
+        
+        # Remove all words and chars from the element manager
+        removed_words = len(self._element_mgr.words)
+        removed_chars = len(self._element_mgr.chars)
+        
+        # Clear the lists
+        self._element_mgr._elements["words"] = []
+        self._element_mgr._elements["chars"] = []
+        
+        logger.info(f"Page {self.number}: Removed {removed_words} words and {removed_chars} characters")
+        return self
+
     @property
     def lines(self) -> List[Any]:
         """Get all line elements on this page."""

{natural_pdf-0.1.32 → natural_pdf-0.1.33}/natural_pdf/core/pdf.py

@@ -108,12 +108,13 @@ class _LazyPageList(Sequence):
     also supported and will materialise pages on demand.
     """
 
-    def __init__(self, parent_pdf: "PDF", plumber_pdf: "pdfplumber.PDF", font_attrs=None):
+    def __init__(self, parent_pdf: "PDF", plumber_pdf: "pdfplumber.PDF", font_attrs=None, load_text=True):
         self._parent_pdf = parent_pdf
         self._plumber_pdf = plumber_pdf
         self._font_attrs = font_attrs
         # One slot per pdfplumber page – initially all None
         self._cache: List[Optional["Page"]] = [None] * len(self._plumber_pdf.pages)
+        self._load_text = load_text
 
     # Internal helper -----------------------------------------------------
     def _create_page(self, index: int) -> "Page":
@@ -123,7 +124,7 @@ class _LazyPageList(Sequence):
             from natural_pdf.core.page import Page
 
             plumber_page = self._plumber_pdf.pages[index]
-            cached = Page(plumber_page, parent=self._parent_pdf, index=index, font_attrs=self._font_attrs)
+            cached = Page(plumber_page, parent=self._parent_pdf, index=index, font_attrs=self._font_attrs, load_text=self._load_text)
             self._cache[index] = cached
         return cached
 
@@ -170,6 +171,7 @@ class PDF(ExtractionMixin, ExportMixin, ClassificationMixin):
         keep_spaces: bool = True,
         text_tolerance: Optional[dict] = None,
         auto_text_tolerance: bool = True,
+        text_layer: bool = True,
     ):
         """
         Initialize the enhanced PDF object.
@@ -181,11 +183,14 @@ class PDF(ExtractionMixin, ExportMixin, ClassificationMixin):
             keep_spaces: Whether to include spaces in word elements
             text_tolerance: PDFplumber-style tolerance settings
             auto_text_tolerance: Whether to automatically scale text tolerance
+            text_layer: Whether to keep the existing text layer from the PDF (default: True).
+                       If False, removes all existing text elements during initialization.
         """
         self._original_path_or_stream = path_or_url_or_stream
         self._temp_file = None
         self._resolved_path = None
         self._is_stream = False
+        self._text_layer = text_layer
         stream_to_open = None
 
         if hasattr(path_or_url_or_stream, "read"):  # Check if it's file-like
@@ -257,7 +262,7 @@ class PDF(ExtractionMixin, ExportMixin, ClassificationMixin):
         self._manager_registry = {}
 
         # Lazily instantiate pages only when accessed
-        self._pages = _LazyPageList(self, self._pdf, font_attrs=font_attrs)
+        self._pages = _LazyPageList(self, self._pdf, font_attrs=font_attrs, load_text=self._text_layer)
 
         self._element_cache = {}
         self._exclusions = []
@@ -267,6 +272,13 @@ class PDF(ExtractionMixin, ExportMixin, ClassificationMixin):
 
         self._initialize_managers()
         self._initialize_highlighter()
+        
+        # Remove text layer if requested
+        if not self._text_layer:
+            logger.info("Removing text layer as requested (text_layer=False)")
+            # Text layer is not loaded when text_layer=False, so no need to remove
+            pass
+        
         # Analysis results accessed via self.analyses property (see below)
 
         # --- Automatic cleanup when object is garbage-collected ---
@@ -1463,6 +1475,7 @@ class PDF(ExtractionMixin, ExportMixin, ClassificationMixin):
                 reading_order=self._reading_order,
                 font_attrs=self._font_attrs,
                 keep_spaces=self._config.get("keep_spaces", True),
+                text_layer=self._text_layer,
             )
             return new_pdf
         except Exception as e:

{natural_pdf-0.1.32 → natural_pdf-0.1.33}/natural_pdf/elements/region.py

@@ -2282,15 +2282,28 @@ class Region(DirectionalMixin, ClassificationMixin, ExtractionMixin, ShapeDetect
                 if success:
                     removed_count += 1
             
-            # Remove OCR elements overlapping this region
+            # Remove ALL OCR elements overlapping this region
+            # Remove elements with source=="ocr" (built-in OCR) or matching the source_label (previous custom OCR)
             for word in list(self.page._element_mgr.words):
-                if getattr(word, "source", "").startswith("ocr") and self.intersects(word):
+                word_source = getattr(word, "source", "")
+                # Match built-in OCR behavior: remove elements with source "ocr" exactly
+                # Also remove elements with the same source_label to avoid duplicates
+                if (word_source == "ocr" or word_source == source_label) and self.intersects(word):
                     _safe_remove(word)
             
-            # Also check custom-ocr sources
-            for word in list(self.page._element_mgr.words):
-                if getattr(word, "source", "") == source_label and self.intersects(word):
-                    _safe_remove(word)
+            # Also remove char dicts if needed (matching built-in OCR)
+            for char in list(self.page._element_mgr.chars):
+                # char can be dict or TextElement; normalize
+                char_src = char.get("source") if isinstance(char, dict) else getattr(char, "source", None)
+                if char_src == "ocr" or char_src == source_label:
+                    # Rough bbox for dicts
+                    if isinstance(char, dict):
+                        cx0, ctop, cx1, cbottom = char.get("x0", 0), char.get("top", 0), char.get("x1", 0), char.get("bottom", 0)
+                    else:
+                        cx0, ctop, cx1, cbottom = char.x0, char.top, char.x1, char.bottom
+                    # Quick overlap check
+                    if not (cx1 < self.x0 or cx0 > self.x1 or cbottom < self.top or ctop > self.bottom):
+                        _safe_remove(char)
             
             if removed_count > 0:
                 logger.info(

{natural_pdf-0.1.32 → natural_pdf-0.1.33/natural_pdf.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: natural-pdf
-Version: 0.1.32
+Version: 0.1.33
 Summary: A more intuitive interface for working with PDFs
 Author-email: Jonathan Soma <jonathan.soma@gmail.com>
 License-Expression: MIT

{natural_pdf-0.1.32 → natural_pdf-0.1.33}/natural_pdf.egg-info/SOURCES.txt

@@ -824,6 +824,7 @@ tests/test_core/test_elements.py
 tests/test_core/test_loading.py
 tests/test_core/test_spatial.py
 tests/test_core/test_text_extraction.py
+tests/test_core/test_text_layer.py
 todo/bad_pdf_analysis.md
 todo/evaluation.md
 tools/bad_pdf_eval/LLM_NaturalPDF_CheatSheet.md

natural_pdf-0.1.33/tests/test_core/test_text_layer.py

@@ -0,0 +1,125 @@
+"""
+Test text layer functionality including text_layer=False parameter and remove_text_layer() method.
+"""
+
+import pytest
+from natural_pdf import PDF
+
+
+def test_pdf_has_text_layer():
+    """Test that pdfs/01-practice.pdf has a text layer by default."""
+    pdf = PDF("pdfs/01-practice.pdf")
+    page = pdf.pages[0]
+    
+    # Check that text elements exist
+    assert len(page.words) > 0, "PDF should have word elements"
+    assert len(page.chars) > 0, "PDF should have character elements"
+    
+    # Check that we can extract text
+    text = page.extract_text()
+    assert text.strip() != "", "Should be able to extract text from PDF"
+    assert "Jungle Health and Safety" in text, "Expected text should be in the extracted text"
+    
+    pdf.close()
+
+
+def test_text_layer_false_parameter():
+    """Test that text_layer=False prevents loading text elements."""
+    pdf = PDF("pdfs/01-practice.pdf", text_layer=False)
+    page = pdf.pages[0]
+    
+    # Check that no text elements exist
+    assert len(page.words) == 0, "PDF should have no word elements when text_layer=False"
+    assert len(page.chars) == 0, "PDF should have no character elements when text_layer=False"
+    
+    # Check that extract_text returns empty
+    text = page.extract_text()
+    assert text.strip() == "", "extract_text should return empty string when text_layer=False"
+    
+    # But other elements should still exist
+    assert len(page.rects) > 0, "Rectangle elements should still be loaded"
+    assert len(page.lines) >= 0, "Line elements should still be accessible"
+    
+    pdf.close()
+
+
+def test_remove_text_layer_method():
+    """Test that remove_text_layer() removes existing text elements."""
+    pdf = PDF("pdfs/01-practice.pdf")
+    page = pdf.pages[0]
+    
+    # First verify text exists
+    initial_words = len(page.words)
+    initial_chars = len(page.chars)
+    assert initial_words > 0, "Should have words initially"
+    assert initial_chars > 0, "Should have chars initially"
+    
+    # Extract text before removal
+    text_before = page.extract_text()
+    assert text_before.strip() != "", "Should have text before removal"
+    
+    # Remove text layer
+    page.remove_text_layer()
+    
+    # Check that text elements are gone
+    assert len(page.words) == 0, "Should have no words after remove_text_layer()"
+    assert len(page.chars) == 0, "Should have no chars after remove_text_layer()"
+    
+    # Check that extract_text returns empty
+    text_after = page.extract_text()
+    assert text_after.strip() == "", "extract_text should return empty after remove_text_layer()"
+    
+    # But other elements should still exist
+    assert len(page.rects) > 0, "Rectangle elements should still exist after text removal"
+    
+    pdf.close()
+
+
+def test_text_layer_false_then_ocr():
+    """Test that we can apply OCR to a PDF loaded with text_layer=False."""
+    # Skip if OCR dependencies are not available
+    pytest.importorskip("easyocr")
+    
+    pdf = PDF("pdfs/01-practice.pdf", text_layer=False)
+    page = pdf.pages[0]
+    
+    # Verify no text initially
+    assert len(page.words) == 0, "Should have no words with text_layer=False"
+    
+    # Apply OCR
+    page.apply_ocr(engine="easyocr", languages=["en"])
+    
+    # Now we should have OCR text
+    assert len(page.words) > 0, "Should have words after OCR"
+    
+    # Check that all words are from OCR
+    for word in page.words:
+        assert word.source == "ocr", "All words should have source='ocr'"
+    
+    pdf.close()
+
+
+def test_multiple_pages_text_layer():
+    """Test text_layer=False works across multiple pages."""
+    pdf = PDF("pdfs/01-practice.pdf", text_layer=False)
+    
+    # Check all pages have no text
+    for page in pdf.pages:
+        assert len(page.words) == 0, f"Page {page.number} should have no words"
+        assert len(page.chars) == 0, f"Page {page.number} should have no chars"
+        assert page.extract_text().strip() == "", f"Page {page.number} should extract empty text"
+    
+    pdf.close()
+
+
+def test_text_layer_parameter_types():
+    """Test that text_layer parameter accepts proper boolean values."""
+    # Test with explicit True (default behavior)
+    pdf_true = PDF("pdfs/01-practice.pdf", text_layer=True)
+    assert len(pdf_true.pages[0].words) > 0, "text_layer=True should load text"
+    pdf_true.close()
+    
+    # Test with explicit False
+    pdf_false = PDF("pdfs/01-practice.pdf", text_layer=False)
+    assert len(pdf_false.pages[0].words) == 0, "text_layer=False should not load text"
+    pdf_false.close()