ducpy 3.4.0__tar.gz → 3.5.0__tar.gz

{ducpy-3.4.0 → ducpy-3.5.0}/Cargo.lock

@@ -412,7 +412,7 @@ dependencies = [
 
 [[package]]
 name = "ducpy-native"
-version = "3.4.0"
+version = "3.5.0"
 dependencies = [
  "duc",
  "pyo3",

{ducpy-3.4.0 → ducpy-3.5.0}/PKG-INFO

@@ -1,8 +1,12 @@
 Metadata-Version: 2.4
 Name: ducpy
-Version: 3.4.0
+Version: 3.5.0
 Classifier: License :: OSI Approved :: MIT License
 Requires-Dist: nanoid>=2.0.0
+Requires-Dist: pypdf>=4.2.0
+Requires-Dist: rapidocr-onnxruntime>=1.3.24 ; extra == 'ocr'
+Requires-Dist: numpy>=1.20.0 ; extra == 'ocr'
+Provides-Extra: ocr
 License-File: LICENSE
 Summary: The library for the Duc 2D CAD file format
 Author: Jorge Soares

{ducpy-3.4.0 → ducpy-3.5.0}/packages/ducpy/crate/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ducpy-native"
-version = "3.4.0"
+version = "3.5.0"
 edition = "2021"
 description = "Python native extension for the duc file format (used by ducpy)"
 publish = false

{ducpy-3.4.0 → ducpy-3.5.0}/packages/ducrs/schema/duc.sql

@@ -1,7 +1,7 @@
 -- "DUC_" in ASCII
 -- Apply in order: duc.sql → version_control.sql → search.sql
 PRAGMA application_id = 1146569567;
-PRAGMA user_version = 3000004;
+PRAGMA user_version = 3000005;
 PRAGMA journal_mode = WAL;
 PRAGMA foreign_keys = ON;
 PRAGMA synchronous = NORMAL;

ducpy-3.5.0/packages/ducrs/schema/migrations/3000004_to_3000005.sql

@@ -0,0 +1,51 @@
+-- Migration: 3000003 → 3000004
+-- Add searchable index for extracted external-file text (PDF content search).
+
+CREATE TABLE IF NOT EXISTS external_file_text_index (
+    id             INTEGER PRIMARY KEY AUTOINCREMENT,
+    file_id        TEXT    NOT NULL,
+    revision_id    TEXT    NOT NULL,
+    mime_type      TEXT    NOT NULL,
+    extracted_text TEXT    NOT NULL DEFAULT '',
+    has_ocr        INTEGER NOT NULL DEFAULT 0,
+    updated        INTEGER NOT NULL,
+    UNIQUE (file_id, revision_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_external_file_text_index_file_id
+    ON external_file_text_index(file_id);
+
+CREATE INDEX IF NOT EXISTS idx_external_file_text_index_revision_id
+    ON external_file_text_index(revision_id);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS search_external_file_text USING fts5(
+    extracted_text,
+    content='external_file_text_index',
+    content_rowid='id',
+    tokenize='unicode61 remove_diacritics 2',
+    prefix='2 3 4 5 6 7 8 9 10'
+);
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_ai
+AFTER INSERT ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(rowid, extracted_text)
+    VALUES (NEW.id, NEW.extracted_text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_ad
+AFTER DELETE ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(search_external_file_text, rowid, extracted_text)
+    VALUES ('delete', OLD.id, OLD.extracted_text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_au
+AFTER UPDATE OF extracted_text ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(search_external_file_text, rowid, extracted_text)
+    VALUES ('delete', OLD.id, OLD.extracted_text);
+    INSERT INTO search_external_file_text(rowid, extracted_text)
+    VALUES (NEW.id, NEW.extracted_text);
+END;
+
+INSERT INTO search_external_file_text(search_external_file_text) VALUES ('rebuild');
+
+PRAGMA user_version = 3000005;

{ducpy-3.4.0 → ducpy-3.5.0}/packages/ducrs/schema/search.sql

@@ -5,7 +5,8 @@
 -- Triggers keep the FTS index automatically in sync on every INSERT/UPDATE/DELETE.
 -- No application-level sync needed.
 --
--- Depends on: duc.sql (elements, element_text, element_doc, element_model, blocks)
+-- Depends on: duc.sql (elements, element_text, element_doc, element_model, blocks,
+--   external_files, external_file_revisions, external_file_revision_data, document_grid_config)
 --
 -- Query examples:
 --   SELECT rowid, rank FROM search_elements WHERE search_elements MATCH 'motor';
@@ -129,6 +130,52 @@ CREATE TRIGGER IF NOT EXISTS trg_blocks_ai AFTER INSERT ON blocks BEGIN
     INSERT INTO search_blocks(rowid, label, description)
     VALUES (NEW.rowid, NEW.label, NEW.description);
 END;
+
+-- Extracted text index for external file revisions (used for PDF content search).
+CREATE TABLE IF NOT EXISTS external_file_text_index (
+    id           INTEGER PRIMARY KEY AUTOINCREMENT,
+    file_id      TEXT    NOT NULL,
+    revision_id  TEXT    NOT NULL,
+    mime_type    TEXT    NOT NULL,
+    extracted_text TEXT  NOT NULL DEFAULT '',
+    has_ocr      INTEGER NOT NULL DEFAULT 0,
+    updated      INTEGER NOT NULL,
+    UNIQUE (file_id, revision_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_external_file_text_index_file_id
+    ON external_file_text_index(file_id);
+
+CREATE INDEX IF NOT EXISTS idx_external_file_text_index_revision_id
+    ON external_file_text_index(revision_id);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS search_external_file_text USING fts5(
+    extracted_text,
+    content='external_file_text_index',
+    content_rowid='id',
+    tokenize='unicode61 remove_diacritics 2',
+    prefix='2 3 4 5 6 7 8 9 10'
+);
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_ai
+AFTER INSERT ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(rowid, extracted_text)
+    VALUES (NEW.id, NEW.extracted_text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_ad
+AFTER DELETE ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(search_external_file_text, rowid, extracted_text)
+    VALUES ('delete', OLD.id, OLD.extracted_text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_au
+AFTER UPDATE OF extracted_text ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(search_external_file_text, rowid, extracted_text)
+    VALUES ('delete', OLD.id, OLD.extracted_text);
+    INSERT INTO search_external_file_text(rowid, extracted_text)
+    VALUES (NEW.id, NEW.extracted_text);
+END;
 CREATE TRIGGER IF NOT EXISTS trg_blocks_ad AFTER DELETE ON blocks BEGIN
     INSERT INTO search_blocks(search_blocks, rowid, label, description)
     VALUES ('delete', OLD.rowid, OLD.label, OLD.description);
@@ -146,3 +193,4 @@ INSERT INTO search_element_text(search_element_text) VALUES ('rebuild');
 INSERT INTO search_element_doc(search_element_doc) VALUES ('rebuild');
 INSERT INTO search_element_model(search_element_model) VALUES ('rebuild');
 INSERT INTO search_blocks(search_blocks) VALUES ('rebuild');
+INSERT INTO search_external_file_text(search_external_file_text) VALUES ('rebuild');

{ducpy-3.4.0 → ducpy-3.5.0}/pyproject.toml

@@ -6,6 +6,7 @@ readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
     "nanoid>=2.0.0",
+    "pypdf>=4.2.0"
 ]
 
 license = { text = "MIT" }
@@ -19,6 +20,12 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
 ]
 
+[project.optional-dependencies]
+ocr = [
+    "rapidocr-onnxruntime>=1.3.24",
+    "numpy>=1.20.0",
+]
+
 [project.urls]
 "Homepage" = "https://duc.ducflair.com"
 "Source" = "https://github.com/ducflair/duc/tree/main/packages/ducpy"
@@ -44,11 +51,14 @@ where = ["src"]
 
 [dependency-groups]
 dev = [
+    "ducpy[ocr]",
+    "maturin>=1.0,<2.0",
     "furo>=2024.8.6",
     "sphinx>=8.1.3",
     "sphinx-autoapi>=3.6.0",
     "pytest>=8.3.5",
     "rich>=13.0.0",
+    "numpy>=1.20.0",
     "typst>=0.14.9",
     "build123d>=0.10.0",
     "ocp-vscode>=3.4.0",
@@ -60,3 +70,6 @@ dev = [
 testpaths = ["src/tests"]
 python_files = "test_*.py"
 addopts = "-vv"
+markers = [
+    "slow: marks tests as slow (OCR-heavy, etc.)",
+]

ducpy-3.5.0/src/ducpy/search/__init__.py

@@ -0,0 +1,19 @@
+"""Search helpers for DUC SQLite databases."""
+
+from .search_elements import (
+    DucElementSearchResult,
+    DucFileSearchResult,
+    DucSearchResponse,
+    DucSearchResult,
+    ExternalFileSearchTarget,
+    search_duc_elements,
+)
+
+__all__ = [
+    "DucElementSearchResult",
+    "DucFileSearchResult",
+    "DucSearchResponse",
+    "DucSearchResult",
+    "ExternalFileSearchTarget",
+    "search_duc_elements",
+]

ducpy-3.5.0/src/ducpy/search/image_ocr.py

@@ -0,0 +1,79 @@
+"""Server-side image OCR helpers for DUC search.
+
+Uses ``rapidocr-onnxruntime`` when the ``ocr`` extra is installed, otherwise
+OCR is gracefully skipped.
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+from functools import lru_cache
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_RAPID_OCR_ENGINE: Any | None = None
+
+
+@lru_cache(maxsize=1)
+def server_side_ocr_available() -> bool:
+    """Return whether the ``ocr`` extra (RapidOCR) is installed and usable."""
+
+    try:
+        from rapidocr_onnxruntime import RapidOCR  # type: ignore
+        from PIL import Image  # type: ignore  # noqa: F401
+    except Exception as exc:
+        logger.debug("OCR extra (rapidocr-onnxruntime) unavailable: %s", exc)
+        return False
+
+    return True
+
+
+def _get_rapid_engine() -> Any:
+    global _RAPID_OCR_ENGINE
+    if _RAPID_OCR_ENGINE is None:
+        from rapidocr_onnxruntime import RapidOCR  # type: ignore
+
+        _RAPID_OCR_ENGINE = RapidOCR()
+    return _RAPID_OCR_ENGINE
+
+
+def extract_image_text_with_ocr(image_bytes: bytes, *, ocr_language: str) -> tuple[str, bool]:
+    """OCR an image using the ``ocr`` extra, or skip when unavailable."""
+
+    if not server_side_ocr_available():
+        return "", False
+
+    try:
+        import numpy as np  # type: ignore
+        from PIL import Image  # type: ignore
+    except Exception as exc:
+        logger.debug("Image OCR dependencies unavailable: %s", exc)
+        return "", False
+
+    try:
+        with Image.open(io.BytesIO(image_bytes)) as image_obj:
+            image_input = np.array(image_obj.convert("RGB"))
+
+        engine = _get_rapid_engine()
+        result, _elapsed = engine(image_input)
+        if not result:
+            return "", False
+
+        lines: list[str] = []
+        for row in result:
+            if isinstance(row, (tuple, list)) and len(row) >= 2:
+                value = str(row[1] or "").strip()
+                if value:
+                    lines.append(value)
+        text = "\n".join(lines)
+    except Exception as exc:
+        logger.debug("Failed to OCR image bytes: %s", exc)
+        return "", False
+
+    return text, bool(text)
+
+
+_server_side_ocr_available = server_side_ocr_available
+_extract_image_text_with_ocr = extract_image_text_with_ocr