natural-pdf 0.1.32__py3-none-any.whl → 0.1.34__py3-none-any.whl

natural_pdf/core/element_manager.py

@@ -1,8 +1,16 @@
-"""
-Element Manager for natural-pdf.
-
-This class handles the loading, creation, and management of PDF elements like
-characters, words, rectangles, and lines extracted from a page.
+"""Element Manager for natural-pdf.
+
+This module handles the loading, creation, and management of PDF elements like
+characters, words, rectangles, lines, and images extracted from a page. The
+ElementManager class serves as the central coordinator for element lifecycle
+management and provides enhanced word extraction capabilities.
+
+The module includes:
+- Element creation and caching for performance
+- Custom word extraction that respects font boundaries
+- OCR coordinate transformation and integration
+- Text decoration detection (underline, strikethrough, highlights)
+- Performance optimizations for bulk text processing
 """
 
 import logging
@@ -13,10 +21,10 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 
 from pdfplumber.utils.text import WordExtractor
 
+from natural_pdf.elements.image import ImageElement
 from natural_pdf.elements.line import LineElement
 from natural_pdf.elements.rect import RectangleElement
 from natural_pdf.elements.text import TextElement
-from natural_pdf.elements.image import ImageElement
 
 logger = logging.getLogger(__name__)
 
@@ -25,8 +33,8 @@ logger = logging.getLogger(__name__)
 # ------------------------------------------------------------------
 
 STRIKE_DEFAULTS = {
-    "thickness_tol": 1.5,   # pt ; max height of line/rect to be considered strike
-    "horiz_tol": 1.0,       # pt ; vertical tolerance for horizontality
+    "thickness_tol": 1.5,  # pt ; max height of line/rect to be considered strike
+    "horiz_tol": 1.0,  # pt ; vertical tolerance for horizontality
     "coverage_ratio": 0.7,  # proportion of glyph width to be overlapped
     "band_top_frac": 0.35,  # fraction of glyph height above top baseline band
     "band_bottom_frac": 0.65,  # fraction below top (same used internally)
@@ -36,48 +44,90 @@ UNDERLINE_DEFAULTS = {
     "thickness_tol": 1.5,
     "horiz_tol": 1.0,
     "coverage_ratio": 0.8,
-    "band_frac": 0.25,   # height fraction above baseline
-    "below_pad": 0.7,    # pt ; pad below baseline
+    "band_frac": 0.25,  # height fraction above baseline
+    "below_pad": 0.7,  # pt ; pad below baseline
 }
 
 HIGHLIGHT_DEFAULTS = {
     "height_min_ratio": 0.6,  # rect height relative to char height lower bound
     "height_max_ratio": 2.0,  # upper bound
-    "coverage_ratio": 0.6,    # horizontal overlap with glyph
+    "coverage_ratio": 0.6,  # horizontal overlap with glyph
     "color_saturation_min": 0.4,  # HSV S >
-    "color_value_min": 0.4,        # HSV V >
+    "color_value_min": 0.4,  # HSV V >
 }
 
 
 @contextmanager
 def disable_text_sync():
-    """
-    Temporarily disable text synchronization for performance.
-    
-    This is used when bulk-updating text content where character-level
-    synchronization is not needed, such as during bidi processing.
-    Fixes exponential recursion issue with Arabic/RTL text processing.
+    """Temporarily disable text synchronization for performance.
+
+    This context manager is used when bulk-updating text content where character-level
+    synchronization is not needed, such as during bidi processing or large-scale
+    text transformations. It prevents exponential recursion issues with Arabic/RTL
+    text processing by bypassing the normal text property setter.
+
+    Yields:
+        None: The context where text synchronization is disabled.
+
+    Example:
+        ```python
+        with disable_text_sync():
+            for element in text_elements:
+                element.text = process_arabic_text(element.text)
+        # Text sync automatically restored after the block
+        ```
+
+    Note:
+        This optimization is critical for performance when processing documents
+        with complex text layouts or right-to-left scripts that would otherwise
+        trigger expensive character synchronization operations.
     """
     # Save original setter
     original_setter = TextElement.text.fset
-    
+
     # Create a fast setter that skips sync
     def fast_setter(self, value):
         self._obj["text"] = value
         # Skip character synchronization for performance
-    
+
     # Apply fast setter
     TextElement.text = property(TextElement.text.fget, fast_setter)
-    
+
     try:
         yield
     finally:
         # Restore original setter
         TextElement.text = property(TextElement.text.fget, original_setter)
 
+
 class NaturalWordExtractor(WordExtractor):
-    """
-    Custom WordExtractor that splits words based on specified character attributes
+    """Custom WordExtractor that splits words based on specified character attributes.
+
+    This class extends pdfplumber's WordExtractor to provide more intelligent word
+    segmentation that respects font boundaries and other character attributes.
+    It prevents words from spanning across different fonts, sizes, or styles,
+    which is essential for maintaining semantic meaning in document analysis.
+
+    The extractor considers multiple character attributes when determining word
+    boundaries, ensuring that visually distinct text elements (like bold headers
+    mixed with regular text) are properly separated into distinct words.
+
+    Attributes:
+        font_attrs: List of character attributes to consider for word boundaries.
+            Common attributes include 'fontname', 'size', 'flags', etc.
+
+    Example:
+        ```python
+        # Create extractor that splits on font and size changes
+        extractor = NaturalWordExtractor(['fontname', 'size'])
+
+        # Extract words with font-aware boundaries
+        words = extractor.extract_words(page_chars)
+
+        # Each word will have consistent font properties
+        for word in words:
+            print(f"'{word['text']}' in {word['fontname']} size {word['size']}")
+        ```
     in addition to pdfplumber's default spatial logic.
     """
 
@@ -146,7 +196,7 @@ class ElementManager:
     contained in the Page class, providing better separation of concerns.
     """
 
-    def __init__(self, page, font_attrs=None):
+    def __init__(self, page, font_attrs=None, load_text: bool = True):
         """
         Initialize the ElementManager.
 
@@ -156,9 +206,11 @@ class ElementManager:
                        Default: ['fontname', 'size', 'bold', 'italic']
                        None: Only consider spatial relationships
                        List: Custom attributes to consider
+            load_text: Whether to load text elements from the PDF (default: True).
         """
         self._page = page
         self._elements = None  # Lazy-loaded
+        self._load_text = load_text
         # Default to splitting by fontname, size, bold, italic if not specified
         # Renamed internal variable for clarity
         self._word_split_attributes = (
@@ -175,11 +227,15 @@ class ElementManager:
 
         logger.debug(f"Page {self._page.number}: Loading elements...")
 
-        # 1. Prepare character dictionaries (native + OCR) with necessary attributes
-        prepared_char_dicts = self._prepare_char_dicts()
-        logger.debug(
-            f"Page {self._page.number}: Prepared {len(prepared_char_dicts)} character dictionaries."
-        )
+        # 1. Prepare character dictionaries only if loading text
+        if self._load_text:
+            prepared_char_dicts = self._prepare_char_dicts()
+            logger.debug(
+                f"Page {self._page.number}: Prepared {len(prepared_char_dicts)} character dictionaries."
+            )
+        else:
+            prepared_char_dicts = []
+            logger.debug(f"Page {self._page.number}: Skipping text loading (load_text=False)")
 
         # -------------------------------------------------------------
         # Detect strikethrough (horizontal strike-out lines) on raw
@@ -189,61 +245,77 @@ class ElementManager:
         # belong to the same word.
         # -------------------------------------------------------------
 
-        try:
-            self._mark_strikethrough_chars(prepared_char_dicts)
-        except Exception as strike_err:  # pragma: no cover – strike detection must never crash loading
-            logger.warning(
-                f"Page {self._page.number}: Strikethrough detection failed – {strike_err}",
-                exc_info=True,
-            )
+        if self._load_text and prepared_char_dicts:
+            try:
+                self._mark_strikethrough_chars(prepared_char_dicts)
+            except (
+                Exception
+            ) as strike_err:  # pragma: no cover – strike detection must never crash loading
+                logger.warning(
+                    f"Page {self._page.number}: Strikethrough detection failed – {strike_err}",
+                    exc_info=True,
+                )
 
         # -------------------------------------------------------------
         # Detect underlines on raw characters (must come after strike so
         # both attributes are present before word grouping).
         # -------------------------------------------------------------
 
-        try:
-            self._mark_underline_chars(prepared_char_dicts)
-        except Exception as u_err:  # pragma: no cover
-            logger.warning(
-                f"Page {self._page.number}: Underline detection failed – {u_err}",
-                exc_info=True,
-            )
+        if self._load_text and prepared_char_dicts:
+            try:
+                self._mark_underline_chars(prepared_char_dicts)
+            except Exception as u_err:  # pragma: no cover
+                logger.warning(
+                    f"Page {self._page.number}: Underline detection failed – {u_err}",
+                    exc_info=True,
+                )
 
         # Detect highlights
-        try:
-            self._mark_highlight_chars(prepared_char_dicts)
-        except Exception as h_err:
-            logger.warning(
-                f"Page {self._page.number}: Highlight detection failed – {h_err}",
-                exc_info=True,
-            )
+        if self._load_text and prepared_char_dicts:
+            try:
+                self._mark_highlight_chars(prepared_char_dicts)
+            except Exception as h_err:
+                logger.warning(
+                    f"Page {self._page.number}: Highlight detection failed – {h_err}",
+                    exc_info=True,
+                )
 
         # Create a mapping from character dict to index for efficient lookup
-        char_to_index = {}
-        for idx, char_dict in enumerate(prepared_char_dicts):
-            key = (
-                char_dict.get("x0", 0),
-                char_dict.get("top", 0),
-                char_dict.get("text", ""),
-            )
-            char_to_index[key] = idx
+        if self._load_text:
+            char_to_index = {}
+            for idx, char_dict in enumerate(prepared_char_dicts):
+                key = (
+                    char_dict.get("x0", 0),
+                    char_dict.get("top", 0),
+                    char_dict.get("text", ""),
+                )
+                char_to_index[key] = idx
+        else:
+            char_to_index = {}
 
         # 2. Instantiate the custom word extractor
         # Prefer page-level config over PDF-level for tolerance lookup
+        word_elements: List[TextElement] = []
+
+        # Get config objects (needed for auto_text_tolerance check)
         page_config = getattr(self._page, "_config", {})
         pdf_config = getattr(self._page._parent, "_config", {})
 
-        # Start with any explicitly supplied tolerances (may be None)
-        xt = page_config.get("x_tolerance", pdf_config.get("x_tolerance"))
-        yt = page_config.get("y_tolerance", pdf_config.get("y_tolerance"))
+        # Initialize tolerance variables
+        xt = None
+        yt = None
         use_flow = pdf_config.get("use_text_flow", False)
 
+        if self._load_text and prepared_char_dicts:
+            # Start with any explicitly supplied tolerances (may be None)
+            xt = page_config.get("x_tolerance", pdf_config.get("x_tolerance"))
+            yt = page_config.get("y_tolerance", pdf_config.get("y_tolerance"))
+
         # ------------------------------------------------------------------
         # Auto-adaptive tolerance: scale based on median character size when
         # requested and explicit values are absent.
         # ------------------------------------------------------------------
-        if pdf_config.get("auto_text_tolerance", True):
+        if self._load_text and pdf_config.get("auto_text_tolerance", True):
             import statistics
 
             sizes = [c.get("size", 0) for c in prepared_char_dicts if c.get("size")]
@@ -255,7 +327,7 @@ class ElementManager:
                     # Record back to page config for downstream users
                     page_config["x_tolerance"] = xt
                 if yt is None:
-                    yt = 0.6 * median_size   # ~line spacing fraction
+                    yt = 0.6 * median_size  # ~line spacing fraction
                     page_config["y_tolerance"] = yt
 
             # Warn users when the page's font size is extremely small –
@@ -323,7 +395,6 @@ class ElementManager:
                 current_line_key = line_key
             lines[-1].append(char_dict)
 
-        word_elements: List[TextElement] = []
         # Process each line separately with direction detection
         for line_chars in lines:
             if not line_chars:
@@ -345,7 +416,8 @@ class ElementManager:
                 char_dir = "ltr"
 
             extractor = NaturalWordExtractor(
-                word_split_attributes=self._word_split_attributes + ["strike", "underline", "highlight"],
+                word_split_attributes=self._word_split_attributes
+                + ["strike", "underline", "highlight"],
                 extra_attrs=attributes_to_preserve,
                 x_tolerance=xt,
                 y_tolerance=yt,
@@ -394,12 +466,13 @@ class ElementManager:
                     # Convert from visual order (from PDF) to logical order using bidi
                     try:
                         from bidi.algorithm import get_display  # type: ignore
+
                         from natural_pdf.utils.bidi_mirror import mirror_brackets
 
                         with disable_text_sync():
                             # word_element.text is currently in visual order (from PDF)
                             # Convert to logical order using bidi with auto direction detection
-                            logical_text = get_display(word_element.text, base_dir='L')
+                            logical_text = get_display(word_element.text, base_dir="L")
                             # Apply bracket mirroring for logical order
                             word_element.text = mirror_brackets(logical_text)
                     except Exception:
@@ -476,11 +549,16 @@ class ElementManager:
                     if color_counts:
                         dominant_color = max(color_counts.items(), key=lambda t: t[1])[0]
                         try:
-                            w._obj["highlight_color"] = tuple(dominant_color) if isinstance(dominant_color, (list, tuple)) else dominant_color
+                            w._obj["highlight_color"] = (
+                                tuple(dominant_color)
+                                if isinstance(dominant_color, (list, tuple))
+                                else dominant_color
+                            )
                         except Exception:
                             w._obj["highlight_color"] = dominant_color
 
-        generated_words = word_elements
+        # generated_words defaults to empty list if text loading is disabled
+        generated_words = word_elements if self._load_text else []
         logger.debug(
             f"Page {self._page.number}: Generated {len(generated_words)} words using NaturalWordExtractor."
         )
@@ -978,12 +1056,16 @@ class ElementManager:
     #  Strikethrough detection (horizontal strike-out lines)
     # ------------------------------------------------------------------
 
-    def _mark_strikethrough_chars(self, char_dicts: List[Dict[str, Any]], *,
-                                  thickness_tol: float = 1.5,
-                                  horiz_tol: float = 1.0,
-                                  coverage_ratio: float = 0.7,
-                                  band_top: float = 0.35,
-                                  band_bottom: float = 0.65) -> None:
+    def _mark_strikethrough_chars(
+        self,
+        char_dicts: List[Dict[str, Any]],
+        *,
+        thickness_tol: float = 1.5,
+        horiz_tol: float = 1.0,
+        coverage_ratio: float = 0.7,
+        band_top: float = 0.35,
+        band_bottom: float = 0.65,
+    ) -> None:
         """Annotate character dictionaries with a boolean ``strike`` flag.
 
         Args
@@ -1082,11 +1164,31 @@ class ElementManager:
         # Allow user overrides via PDF._config["underline_detection"]
         pdf_cfg = getattr(self._page._parent, "_config", {}).get("underline_detection", {})
 
-        thickness_tol = thickness_tol if thickness_tol is not None else pdf_cfg.get("thickness_tol", UNDERLINE_DEFAULTS["thickness_tol"])
-        horiz_tol     = horiz_tol     if horiz_tol     is not None else pdf_cfg.get("horiz_tol", UNDERLINE_DEFAULTS["horiz_tol"])
-        coverage_ratio= coverage_ratio if coverage_ratio is not None else pdf_cfg.get("coverage_ratio", UNDERLINE_DEFAULTS["coverage_ratio"])
-        band_frac     = band_frac     if band_frac     is not None else pdf_cfg.get("band_frac", UNDERLINE_DEFAULTS["band_frac"])
-        below_pad     = below_pad     if below_pad     is not None else pdf_cfg.get("below_pad", UNDERLINE_DEFAULTS["below_pad"])
+        thickness_tol = (
+            thickness_tol
+            if thickness_tol is not None
+            else pdf_cfg.get("thickness_tol", UNDERLINE_DEFAULTS["thickness_tol"])
+        )
+        horiz_tol = (
+            horiz_tol
+            if horiz_tol is not None
+            else pdf_cfg.get("horiz_tol", UNDERLINE_DEFAULTS["horiz_tol"])
+        )
+        coverage_ratio = (
+            coverage_ratio
+            if coverage_ratio is not None
+            else pdf_cfg.get("coverage_ratio", UNDERLINE_DEFAULTS["coverage_ratio"])
+        )
+        band_frac = (
+            band_frac
+            if band_frac is not None
+            else pdf_cfg.get("band_frac", UNDERLINE_DEFAULTS["band_frac"])
+        )
+        below_pad = (
+            below_pad
+            if below_pad is not None
+            else pdf_cfg.get("below_pad", UNDERLINE_DEFAULTS["below_pad"])
+        )
 
         raw_lines = list(getattr(self._page._page, "lines", []))
         raw_rects = list(getattr(self._page._page, "rects", []))
@@ -1128,7 +1230,7 @@ class ElementManager:
         table_y = {k for k, v in y_groups.items() if v >= 3}
 
         # filter out candidates on those y values
-        filtered_candidates = [c for c in candidates if int((c[1]+c[3])/2) not in table_y]
+        filtered_candidates = [c for c in candidates if int((c[1] + c[3]) / 2) not in table_y]
 
         # annotate chars
         for ch in char_dicts:
@@ -1185,7 +1287,9 @@ class ElementManager:
             y0_rect = min(rc.get("y0", 0), rc.get("y1", 0))
             y1_rect = max(rc.get("y0", 0), rc.get("y1", 0))
             rheight = y1_rect - y0_rect
-            highlight_rects.append((rc.get("x0", 0), y0_rect, rc.get("x1", 0), y1_rect, rheight, fill_col))
+            highlight_rects.append(
+                (rc.get("x0", 0), y0_rect, rc.get("x1", 0), y1_rect, rheight, fill_col)
+            )
 
         if not highlight_rects:
             for ch in char_dicts:
@@ -1218,7 +1322,9 @@ class ElementManager:
                 if overlap > 0 and (overlap / width) >= coverage_ratio:
                     ch["highlight"] = True
                     try:
-                        ch["highlight_color"] = tuple(rcolor) if isinstance(rcolor, (list, tuple)) else rcolor
+                        ch["highlight_color"] = (
+                            tuple(rcolor) if isinstance(rcolor, (list, tuple)) else rcolor
+                        )
                     except Exception:
                         ch["highlight_color"] = rcolor
                     break

natural_pdf/core/highlighting_service.py

@@ -98,7 +98,9 @@ class HighlightRenderer:
             scaled_bbox = None
 
             if highlight.is_polygon:
-                scaled_polygon = [(p[0] * self.scale_factor, p[1] * self.scale_factor) for p in highlight.polygon]
+                scaled_polygon = [
+                    (p[0] * self.scale_factor, p[1] * self.scale_factor) for p in highlight.polygon
+                ]
                 # Draw polygon fill and border
                 draw.polygon(
                     scaled_polygon, fill=highlight.color, outline=highlight.border_color, width=2
@@ -597,7 +599,7 @@ class HighlightingService:
         if page_index in self._highlights_by_page:
             del self._highlights_by_page[page_index]
             logger.debug(f"Cleared highlights for page {page_index}.")
-        
+
         # Also clear any cached rendered images for this page so the next render
         # reflects the removal of highlights.
         try:
@@ -683,7 +685,6 @@ class HighlightingService:
             )
 
         try:
-            # base_image = render_plain_page(page_obj, actual_scale_x * 72 if actual_scale_x else scale * 72) # Old call
             img_object = page_obj._page.to_image(**to_image_args)
             base_image_pil = (
                 img_object.annotated
@@ -929,9 +930,7 @@ class HighlightingService:
                 right_px = max(left_px + 1, min(right_px, rendered_image.width))
                 bottom_px = max(top_px + 1, min(bottom_px, rendered_image.height))
 
-                rendered_image = rendered_image.crop(
-                    (left_px, top_px, right_px, bottom_px)
-                )
+                rendered_image = rendered_image.crop((left_px, top_px, right_px, bottom_px))
 
             legend = None
             if labels: