natural-pdf 0.1.11__py3-none-any.whl → 0.1.12__py3-none-any.whl

natural_pdf/exporters/hocr.py

@@ -66,28 +66,28 @@ class HocrTransform:
     """
 
     box_pattern = re.compile(
-        r'''
+        r"""
         bbox \s+
         (\d+) \s+   # left: uint
         (\d+) \s+   # top: uint
         (\d+) \s+   # right: uint
         (\d+)       # bottom: uint
-        ''',
+        """,
         re.VERBOSE,
     )
     baseline_pattern = re.compile(
-        r'''
+        r"""
         baseline \s+
         ([\-\+]?\d*\.?\d*) \s+  # +/- decimal float
         ([\-\+]?\d+)            # +/- int
-        ''',
+        """,
         re.VERBOSE,
     )
     textangle_pattern = re.compile(
-        r'''
+        r"""
         textangle \s+
         ([\-\+]?\d*\.?\d*)  # +/- decimal float
-        ''',
+        """,
         re.VERBOSE,
     )
 
@@ -121,12 +121,12 @@ class HocrTransform:
 
         # if the hOCR file has a namespace, ElementTree requires its use to
         # find elements
-        matches = re.match(r'({.*})html', self.hocr.getroot().tag)
-        self.xmlns = ''
+        matches = re.match(r"({.*})html", self.hocr.getroot().tag)
+        self.xmlns = ""
         if matches:
             self.xmlns = matches.group(1)
 
-        for div in self.hocr.findall(self._child_xpath('div', 'ocr_page')):
+        for div in self.hocr.findall(self._child_xpath("div", "ocr_page")):
             coords = self.element_coordinates(div)
             if not coords:
                 raise HocrTransformError("hocr file is missing page dimensions")
@@ -137,16 +137,16 @@ class HocrTransform:
 
     def _get_element_text(self, element: Element) -> str:
         """Return the textual content of the element and its children."""
-        text = element.text if element.text is not None else ''
+        text = element.text if element.text is not None else ""
         for child in element:
             text += self._get_element_text(child)
-        text += element.tail if element.tail is not None else ''
+        text += element.tail if element.tail is not None else ""
         return text
 
     @classmethod
     def element_coordinates(cls, element: Element) -> Rectangle | None:
         """Get coordinates of the bounding box around an element."""
-        matches = cls.box_pattern.search(element.attrib.get('title', ''))
+        matches = cls.box_pattern.search(element.attrib.get("title", ""))
         if not matches:
             return None
         return Rectangle(
@@ -159,7 +159,7 @@ class HocrTransform:
     @classmethod
     def baseline(cls, element: Element) -> tuple[float, float]:
         """Get baseline's slope and intercept."""
-        matches = cls.baseline_pattern.search(element.attrib.get('title', ''))
+        matches = cls.baseline_pattern.search(element.attrib.get("title", ""))
         if not matches:
             return (0.0, 0.0)
         return float(matches.group(1)), int(matches.group(2))
@@ -167,7 +167,7 @@ class HocrTransform:
     @classmethod
     def textangle(cls, element: Element) -> float:
         """Get text angle of an element."""
-        matches = cls.textangle_pattern.search(element.attrib.get('title', ''))
+        matches = cls.textangle_pattern.search(element.attrib.get("title", ""))
         if not matches:
             return 0.0
         return float(matches.group(1))
@@ -220,13 +220,13 @@ class HocrTransform:
         with canvas.do.save_state(cm=page_matrix):
             self._debug_draw_paragraph_boxes(canvas)
             found_lines = False
-            for par in self.hocr.iterfind(self._child_xpath('p', 'ocr_par')):
+            for par in self.hocr.iterfind(self._child_xpath("p", "ocr_par")):
                 for line in (
                     element
-                    for element in par.iterfind(self._child_xpath('span'))
-                    if 'class' in element.attrib
-                    and element.attrib['class']
-                    in {'ocr_header', 'ocr_line', 'ocr_textfloat', 'ocr_caption'}
+                    for element in par.iterfind(self._child_xpath("span"))
+                    if "class" in element.attrib
+                    and element.attrib["class"]
+                    in {"ocr_header", "ocr_line", "ocr_textfloat", "ocr_caption"}
                 ):
                     found_lines = True
                     direction = self._get_text_direction(par)
@@ -242,7 +242,7 @@ class HocrTransform:
 
             if not found_lines:
                 # Tesseract did not report any lines (just words)
-                root = self.hocr.find(self._child_xpath('div', 'ocr_page'))
+                root = self.hocr.find(self._child_xpath("div", "ocr_page"))
                 direction = self._get_text_direction(root)
                 self._do_line(
                     canvas,
@@ -254,27 +254,21 @@ class HocrTransform:
                 )
         # put the image on the page, scaled to fill the page
         if image_filename is not None:
-            canvas.do.draw_image(
-                image_filename, 0, 0, width=self.width, height=self.height
-            )
+            canvas.do.draw_image(image_filename, 0, 0, width=self.width, height=self.height)
 
         # finish up the page and save it
         canvas.to_pdf().save(out_filename)
 
     def _get_text_direction(self, par):
         """Get the text direction of the paragraph.
-    
+
         Arabic, Hebrew, Persian, are right-to-left languages.
         When the paragraph element is None, defaults to left-to-right.
         """
         if par is None:
             return TextDirection.LTR
-            
-        return (
-            TextDirection.RTL
-            if par.attrib.get('dir', 'ltr') == 'rtl'
-            else TextDirection.LTR
-        )
+
+        return TextDirection.RTL if par.attrib.get("dir", "ltr") == "rtl" else TextDirection.LTR
 
     def _get_inject_word_breaks(self, par):
         """Determine whether word breaks should be injected.
@@ -283,9 +277,9 @@ class HocrTransform:
         words are usually one or two characters and separators are usually explicit.
         In all other languages, we inject word breaks to help word segmentation.
         """
-        lang = par.attrib.get('lang', '')
+        lang = par.attrib.get("lang", "")
         log.debug(lang)
-        if lang in {'chi_sim', 'chi_tra', 'jpn', 'kor'}:
+        if lang in {"chi_sim", "chi_tra", "jpn", "kor"}:
             return False
         return True
 
@@ -339,8 +333,7 @@ class HocrTransform:
         # size as the true bounding box of the line.
         top_left_corner = (line_min_aabb.llx, line_min_aabb.lly)
         line_size_aabb_matrix = (
-            Matrix()
-            .translated(*top_left_corner)
+            Matrix().translated(*top_left_corner)
             # Note: negative sign (textangle is counter-clockwise, see hOCR spec)
             .rotated(-self.textangle(line))
         )
@@ -371,12 +364,10 @@ class HocrTransform:
             text.font(self._fontname, fontsize)
             text.render_mode(3 if invisible_text else 0)
 
-            self._debug_draw_baseline(
-                canvas, baseline_matrix.inverse().transform(line_min_aabb), 0
-            )
+            self._debug_draw_baseline(canvas, baseline_matrix.inverse().transform(line_min_aabb), 0)
 
             canvas.do.fill_color(BLACK)  # text in black
-            elements = line.findall(self._child_xpath('span', elemclass))
+            elements = line.findall(self._child_xpath("span", elemclass))
             for elem, next_elem in pairwise(elements + [None]):
                 self._do_line_word(
                     canvas,
@@ -405,7 +396,7 @@ class HocrTransform:
         if elem is None:
             return
         elemtxt = self.normalize_text(self._get_element_text(elem).strip())
-        if elemtxt == '':
+        if elemtxt == "":
             return
 
         hocr_box = self.element_coordinates(elem)
@@ -430,9 +421,7 @@ class HocrTransform:
             text.show(self._font.text_encode(elemtxt))
 
         # Get coordinates of the next word (if there is one)
-        hocr_next_box = (
-            self.element_coordinates(next_elem) if next_elem is not None else None
-        )
+        hocr_next_box = self.element_coordinates(next_elem) if next_elem is not None else None
         if hocr_next_box is None:
             return
         # Render a space between this word and the next word. The explicit space helps
@@ -447,16 +436,14 @@ class HocrTransform:
         elif text_direction == TextDirection.RTL:
             space_box = Rectangle(next_box.urx, box.lly, box.llx, next_box.ury)
         self._debug_draw_space_bbox(canvas, space_box)
-        space_width = self._font.text_width(' ', fontsize)
+        space_width = self._font.text_width(" ", fontsize)
         if space_width > 0 and space_box.width > 0:
             if text_direction == TextDirection.LTR:
                 text.text_transform(Matrix(1, 0, 0, -1, space_box.llx, 0))
             elif text_direction == TextDirection.RTL:
-                text.text_transform(
-                    Matrix(-1, 0, 0, -1, space_box.llx + space_box.width, 0)
-                )
+                text.text_transform(Matrix(-1, 0, 0, -1, space_box.llx + space_box.width, 0))
             text.horiz_scale(100 * space_box.width / space_width)
-            text.show(self._font.text_encode(' '))
+            text.show(self._font.text_encode(" "))
 
     def _debug_draw_paragraph_boxes(self, canvas: Canvas, color=CYAN):
         """Draw boxes around paragraphs in the document."""
@@ -465,16 +452,14 @@ class HocrTransform:
         with canvas.do.save_state():
             # draw box around paragraph
             canvas.do.stroke_color(color).line_width(0.1)
-            for elem in self.hocr.iterfind(self._child_xpath('p', 'ocr_par')):
+            for elem in self.hocr.iterfind(self._child_xpath("p", "ocr_par")):
                 elemtxt = self._get_element_text(elem).strip()
                 if len(elemtxt) == 0:
                     continue
                 ocr_par = self.element_coordinates(elem)
                 if ocr_par is None:
                     continue
-                canvas.do.rect(
-                    ocr_par.llx, ocr_par.lly, ocr_par.width, ocr_par.height, fill=False
-                )
+                canvas.do.rect(ocr_par.llx, ocr_par.lly, ocr_par.width, ocr_par.height, fill=False)
 
     def _debug_draw_line_bbox(self, canvas: Canvas, line_box: Rectangle, color=BLUE):
         """Render the bounding box of a text line."""
@@ -485,22 +470,16 @@ class HocrTransform:
                 line_box.llx, line_box.lly, line_box.width, line_box.height, fill=False
             )
 
-    def _debug_draw_word_triangle(
-        self, canvas: Canvas, box: Rectangle, color=RED, line_width=0.1
-    ):
+    def _debug_draw_word_triangle(self, canvas: Canvas, box: Rectangle, color=RED, line_width=0.1):
         """Render a triangle that conveys word height and drawing direction."""
         if not self.render_options.render_triangle:  # pragma: no cover
             return
         with canvas.do.save_state():
             canvas.do.stroke_color(color).line_width(line_width).line(
                 box.llx, box.lly, box.urx, box.lly
-            ).line(box.urx, box.lly, box.llx, box.ury).line(
-                box.llx, box.lly, box.llx, box.ury
-            )
+            ).line(box.urx, box.lly, box.llx, box.ury).line(box.llx, box.lly, box.llx, box.ury)
 
-    def _debug_draw_word_bbox(
-        self, canvas: Canvas, box: Rectangle, color=GREEN, line_width=0.1
-    ):
+    def _debug_draw_word_bbox(self, canvas: Canvas, box: Rectangle, color=GREEN, line_width=0.1):
         """Render a box depicting the word."""
         if not self.render_options.render_word_bbox:  # pragma: no cover
             return
@@ -537,4 +516,4 @@ class HocrTransform:
                 baseline_lly,
                 line_box.urx,
                 baseline_lly,
-            )
+            )

natural_pdf/exporters/hocr_font.py

@@ -8,11 +8,7 @@ import unicodedata
 import zlib
 from importlib.resources import files as package_files
 
-from pikepdf import (
-    Dictionary,
-    Name,
-    Pdf,
-)
+from pikepdf import Dictionary, Name, Pdf
 from pikepdf.canvas import Font
 
 log = logging.getLogger(__name__)
@@ -25,8 +21,8 @@ class EncodableFont(Font):
 
 class GlyphlessFont(EncodableFont):
     CID_TO_GID_DATA = zlib.compress(b"\x00\x01" * 65536)
-    GLYPHLESS_FONT_NAME = 'pdf.ttf'
-    GLYPHLESS_FONT_PACKAGE_PATH = 'natural_pdf.exporters.data'
+    GLYPHLESS_FONT_NAME = "pdf.ttf"
+    GLYPHLESS_FONT_PACKAGE_PATH = "natural_pdf.exporters.data"
     GLYPHLESS_FONT = (package_files(GLYPHLESS_FONT_PACKAGE_PATH) / GLYPHLESS_FONT_NAME).read_bytes()
     CHAR_ASPECT = 2
 
@@ -39,7 +35,7 @@ class GlyphlessFont(EncodableFont):
         return len(unicodedata.normalize("NFKC", text)) * (fontsize / self.CHAR_ASPECT)
 
     def text_encode(self, text: str) -> bytes:
-        return text.encode('utf-16be')
+        return text.encode("utf-16be")
 
     def register(self, pdf: Pdf):
         """Register the glyphless font.
@@ -76,9 +72,7 @@ class GlyphlessFont(EncodableFont):
             )
         )
         basefont.DescendantFonts = [cid_font_type2]
-        cid_font_type2.CIDToGIDMap = pdf.make_stream(
-            self.CID_TO_GID_DATA, Filter=Name.FlateDecode
-        )
+        cid_font_type2.CIDToGIDMap = pdf.make_stream(self.CID_TO_GID_DATA, Filter=Name.FlateDecode)
         basefont.ToUnicode = pdf.make_stream(
             b"/CIDInit /ProcSet findresource begin\n"
             b"12 dict begin\n"
@@ -129,7 +123,7 @@ class Courier(EncodableFont):
         return len(text) * fontsize
 
     def text_encode(self, text: str) -> bytes:
-        return text.encode('pdfdoc', errors='ignore')
+        return text.encode("pdfdoc", errors="ignore")
 
     def register(self, pdf: Pdf) -> Dictionary:
         """Register the font."""
@@ -139,4 +133,4 @@ class Courier(EncodableFont):
                 Type=Name.Font,
                 Subtype=Name.Type1,
             )
-        )
+        )

natural_pdf/exporters/original_pdf.py

@@ -44,7 +44,7 @@ def create_original_pdf(
     if pikepdf is None:
         raise ImportError(
             "Saving original PDF pages requires 'pikepdf'. "
-            "Install with: pip install \"natural-pdf[ocr-export]\""
+            'Install with: pip install "natural-pdf[ocr-export]"'
         )
 
     output_path_str = str(output_path)
@@ -55,18 +55,17 @@ def create_original_pdf(
         if not source.pages:
             raise ValueError("Cannot save an empty collection/PDF.")
         pages_to_extract = source.pages
-    elif hasattr(source, "page") and hasattr(source, "number"): # Single Page object
-         # Check if it's a natural_pdf.core.page.Page or similar duck-typed object
-         if hasattr(source, 'pdf') and source.pdf and hasattr(source.pdf, 'path'):
-             pages_to_extract = [source]
-         else:
+    elif hasattr(source, "page") and hasattr(source, "number"):  # Single Page object
+        # Check if it's a natural_pdf.core.page.Page or similar duck-typed object
+        if hasattr(source, "pdf") and source.pdf and hasattr(source.pdf, "path"):
+            pages_to_extract = [source]
+        else:
             raise ValueError("Input Page object does not have a valid PDF reference with a path.")
     else:
         raise TypeError(f"Unsupported source type for create_original_pdf: {type(source)}")
 
-
     if not pages_to_extract:
-         raise ValueError("No valid pages found in the source object.")
+        raise ValueError("No valid pages found in the source object.")
 
     # Verify all pages come from the same PDF and get path
     first_page_pdf_path = None
@@ -115,16 +114,14 @@ def create_original_pdf(
             )
 
     except pikepdf.PasswordError:
-        logger.error(
-            f"Failed to open password-protected source PDF: {first_page_pdf_path}"
-        )
+        logger.error(f"Failed to open password-protected source PDF: {first_page_pdf_path}")
         raise RuntimeError(
             f"Source PDF '{first_page_pdf_path}' is password-protected."
-        ) from None # Raise specific error without chaining the generic Exception
+        ) from None  # Raise specific error without chaining the generic Exception
     except Exception as e:
         logger.error(
             f"Failed to save original pages PDF to '{output_path_str}': {e}",
             exc_info=True,
         )
         # Re-raise as RuntimeError for consistent API error handling
-        raise RuntimeError(f"Failed to save original pages PDF: {e}") from e 
+        raise RuntimeError(f"Failed to save original pages PDF: {e}") from e

natural_pdf/exporters/searchable_pdf.py

@@ -318,16 +318,6 @@ def create_searchable_pdf(
         dpi: The resolution (dots per inch) for rendering page images and hOCR.
     """
 
-    # --- Ensure dependencies are loaded (they should be if installed) ---
-    if Image is None or pikepdf is None or HocrTransform is None:
-        # This should ideally not happen if dependencies are in main install,
-        # but serves as a safeguard during development or if install is broken.
-        raise ImportError(
-            "Required dependencies (Pillow, pikepdf) are missing. "
-            "Please ensure natural-pdf is installed correctly with all dependencies."
-        )
-    # --- End Safeguard Check ---
-
     # duck type to see if source has .pages, to populate pages =
     if hasattr(source, "pages"):
         pages = source.pages

natural_pdf/search/__init__.py

@@ -3,29 +3,46 @@
 import logging
 from typing import Optional
 
-# --- Service Implementation Import ---
-# Import the concrete implementation
-from .haystack_search_service import HaystackSearchService
-
-# --- Utils Import ---
-from .haystack_utils import (  # Re-export flag and helper
-    HAS_HAYSTACK_EXTRAS,
-    check_haystack_availability,
-)
-
-# --- Option Imports (for convenience) ---
-# Make options easily available via `from natural_pdf.search import ...`
-from .search_options import SearchOptions  # Alias for TextSearchOptions for simplicity?
+# Import constants
+from .search_options import SearchOptions 
 from .search_options import BaseSearchOptions, MultiModalSearchOptions, TextSearchOptions
-
-# --- Protocol Import ---
-# Import the protocol for type hinting
 from .search_service_protocol import Indexable, IndexConfigurationError, SearchServiceProtocol
 
+# Check search extras availability
+LANCEDB_AVAILABLE = False
+SEARCH_DEPENDENCIES_AVAILABLE = False
+
+try:
+    import sentence_transformers
+    import numpy as np
+    # Basic search dependencies are available
+    SEARCH_DEPENDENCIES_AVAILABLE = True
+    
+    # Check if LanceDB is available
+    try:
+        import lancedb
+        import pyarrow
+        LANCEDB_AVAILABLE = True
+        from .lancedb_search_service import LanceDBSearchService, DEFAULT_LANCEDB_PERSIST_PATH, DEFAULT_EMBEDDING_MODEL
+    except ImportError:
+        # LanceDB not available, we'll use NumPy fallback
+        LANCEDB_AVAILABLE = False
+        from .numpy_search_service import NumpySearchService, DEFAULT_EMBEDDING_MODEL
+except ImportError:
+    # Basic dependencies missing
+    SEARCH_DEPENDENCIES_AVAILABLE = False
+    LANCEDB_AVAILABLE = False
+
 logger = logging.getLogger(__name__)
 
+def check_search_availability():
+    """Check if required search dependencies are available."""
+    if not SEARCH_DEPENDENCIES_AVAILABLE:
+        raise ImportError(
+            "Search functionality requires 'sentence-transformers' and NumPy. "
+            "Install with: pip install natural-pdf[search] (or pip install sentence-transformers numpy)"
+        )
 
-# Factory Function
 def get_search_service(
     collection_name: str,
     persist: bool = False,
@@ -34,53 +51,49 @@ def get_search_service(
 ) -> SearchServiceProtocol:
     """
     Factory function to get an instance of the configured search service.
-
-    A service instance is tied to a specific index name (collection/table).
-
-    Currently, only returns HaystackSearchService but is structured for future extension.
+    
+    Automatically selects the best available implementation:
+    - LanceDB if installed (recommended for both in-memory and persistent)
+    - Numpy fallback for in-memory only
 
     Args:
-        collection_name: The logical name for the index this service instance manages
-                         (used as table_name for LanceDB).
+        collection_name: The logical name for the index/table this service instance manages.
         persist: If True, creates a service instance configured for persistent
-                 storage (currently LanceDB). If False (default), uses InMemory.
-        uri: Override the default path/URI for persistent storage.
+                 storage. If False (default), uses InMemory (via temp dir for LanceDB).
+        uri: Override the default path for persistent storage.
         default_embedding_model: Override the default embedding model used by the service.
-        **kwargs: Reserved for future configuration options.
 
     Returns:
-        An instance conforming to the SearchServiceProtocol for the specified collection/table.
+        An instance conforming to the SearchServiceProtocol.
     """
     logger.debug(
-        f"Calling get_search_service factory for index '{collection_name}' (persist={persist}, uri={uri})..."
+        f"Calling get_search_service factory for collection '{collection_name}' (persist={persist}, uri={uri})..."
     )
+    check_search_availability()
 
-    # Collect arguments relevant to HaystackSearchService.__init__
-    service_args = {}
-    service_args["table_name"] = collection_name
-    service_args["persist"] = persist
+    service_args = {
+        "collection_name": collection_name,
+        "persist": persist,
+    }
     if uri is not None:
         service_args["uri"] = uri
-    if default_embedding_model is not None:
-        service_args["embedding_model"] = default_embedding_model
 
-    # Cache logic commented out as before
+    if default_embedding_model is not None:
+        service_args["embedding_model_name"] = default_embedding_model
 
-    try:
-        service_instance = HaystackSearchService(**service_args)
-        logger.info(f"Created new HaystackSearchService instance for index '{collection_name}'.")
-        return service_instance
-    except ImportError as e:
-        # Error message remains valid
-        logger.error(
-            f"Failed to instantiate Search Service due to missing dependencies: {e}", exc_info=True
+    # If persistence is requested, LanceDB is required
+    if persist and not LANCEDB_AVAILABLE:
+        raise RuntimeError(
+            "Persistent vector search requires LanceDB. "
+            "Please install: pip install lancedb"
         )
-        raise ImportError(
-            "Search Service could not be created. Ensure Haystack extras are installed: pip install natural-pdf[haystack]"
-        ) from e
-    except Exception as e:
-        logger.error(f"Failed to instantiate Search Service: {e}", exc_info=True)
-        raise RuntimeError("Could not create Search Service instance.") from e
-
-
-# Default instance commented out as before
+    
+    # Select the appropriate implementation
+    if LANCEDB_AVAILABLE:
+        logger.info(f"Using LanceDB for vector search (collection: {collection_name})")
+        service_instance = LanceDBSearchService(**service_args)
+    else:
+        logger.info(f"Using NumPy fallback for in-memory vector search (collection: {collection_name})")
+        service_instance = NumpySearchService(**service_args)
+        
+    return service_instance