ducpy 3.4.0__tar.gz → 3.5.1__tar.gz

{ducpy-3.4.0 → ducpy-3.5.1}/Cargo.lock

@@ -412,7 +412,7 @@ dependencies = [
 
 [[package]]
 name = "ducpy-native"
-version = "3.4.0"
+version = "3.5.1"
 dependencies = [
  "duc",
  "pyo3",

{ducpy-3.4.0 → ducpy-3.5.1}/PKG-INFO

@@ -1,8 +1,14 @@
 Metadata-Version: 2.4
 Name: ducpy
-Version: 3.4.0
+Version: 3.5.1
 Classifier: License :: OSI Approved :: MIT License
 Requires-Dist: nanoid>=2.0.0
+Requires-Dist: pypdf>=4.2.0
+Requires-Dist: rapidocr==3.8.1 ; extra == 'ocr'
+Requires-Dist: onnxruntime ; extra == 'ocr'
+Requires-Dist: huggingface-hub>=0.20.0 ; extra == 'ocr'
+Requires-Dist: numpy>=1.20.0 ; extra == 'ocr'
+Provides-Extra: ocr
 License-File: LICENSE
 Summary: The library for the Duc 2D CAD file format
 Author: Jorge Soares

{ducpy-3.4.0 → ducpy-3.5.1}/packages/ducpy/crate/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ducpy-native"
-version = "3.4.0"
+version = "3.5.1"
 edition = "2021"
 description = "Python native extension for the duc file format (used by ducpy)"
 publish = false

{ducpy-3.4.0 → ducpy-3.5.1}/packages/ducrs/schema/duc.sql

@@ -1,7 +1,7 @@
 -- "DUC_" in ASCII
 -- Apply in order: duc.sql → version_control.sql → search.sql
 PRAGMA application_id = 1146569567;
-PRAGMA user_version = 3000004;
+PRAGMA user_version = 3000005;
 PRAGMA journal_mode = WAL;
 PRAGMA foreign_keys = ON;
 PRAGMA synchronous = NORMAL;

ducpy-3.5.1/packages/ducrs/schema/migrations/3000004_to_3000005.sql

@@ -0,0 +1,51 @@
+-- Migration: 3000003 → 3000004
+-- Add searchable index for extracted external-file text (PDF content search).
+
+CREATE TABLE IF NOT EXISTS external_file_text_index (
+    id             INTEGER PRIMARY KEY AUTOINCREMENT,
+    file_id        TEXT    NOT NULL,
+    revision_id    TEXT    NOT NULL,
+    mime_type      TEXT    NOT NULL,
+    extracted_text TEXT    NOT NULL DEFAULT '',
+    has_ocr        INTEGER NOT NULL DEFAULT 0,
+    updated        INTEGER NOT NULL,
+    UNIQUE (file_id, revision_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_external_file_text_index_file_id
+    ON external_file_text_index(file_id);
+
+CREATE INDEX IF NOT EXISTS idx_external_file_text_index_revision_id
+    ON external_file_text_index(revision_id);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS search_external_file_text USING fts5(
+    extracted_text,
+    content='external_file_text_index',
+    content_rowid='id',
+    tokenize='unicode61 remove_diacritics 2',
+    prefix='2 3 4 5 6 7 8 9 10'
+);
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_ai
+AFTER INSERT ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(rowid, extracted_text)
+    VALUES (NEW.id, NEW.extracted_text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_ad
+AFTER DELETE ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(search_external_file_text, rowid, extracted_text)
+    VALUES ('delete', OLD.id, OLD.extracted_text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_au
+AFTER UPDATE OF extracted_text ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(search_external_file_text, rowid, extracted_text)
+    VALUES ('delete', OLD.id, OLD.extracted_text);
+    INSERT INTO search_external_file_text(rowid, extracted_text)
+    VALUES (NEW.id, NEW.extracted_text);
+END;
+
+INSERT INTO search_external_file_text(search_external_file_text) VALUES ('rebuild');
+
+PRAGMA user_version = 3000005;

{ducpy-3.4.0 → ducpy-3.5.1}/packages/ducrs/schema/search.sql

@@ -5,7 +5,8 @@
 -- Triggers keep the FTS index automatically in sync on every INSERT/UPDATE/DELETE.
 -- No application-level sync needed.
 --
--- Depends on: duc.sql (elements, element_text, element_doc, element_model, blocks)
+-- Depends on: duc.sql (elements, element_text, element_doc, element_model, blocks,
+--   external_files, external_file_revisions, external_file_revision_data, document_grid_config)
 --
 -- Query examples:
 --   SELECT rowid, rank FROM search_elements WHERE search_elements MATCH 'motor';
@@ -129,6 +130,52 @@ CREATE TRIGGER IF NOT EXISTS trg_blocks_ai AFTER INSERT ON blocks BEGIN
     INSERT INTO search_blocks(rowid, label, description)
     VALUES (NEW.rowid, NEW.label, NEW.description);
 END;
+
+-- Extracted text index for external file revisions (used for PDF content search).
+CREATE TABLE IF NOT EXISTS external_file_text_index (
+    id           INTEGER PRIMARY KEY AUTOINCREMENT,
+    file_id      TEXT    NOT NULL,
+    revision_id  TEXT    NOT NULL,
+    mime_type    TEXT    NOT NULL,
+    extracted_text TEXT  NOT NULL DEFAULT '',
+    has_ocr      INTEGER NOT NULL DEFAULT 0,
+    updated      INTEGER NOT NULL,
+    UNIQUE (file_id, revision_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_external_file_text_index_file_id
+    ON external_file_text_index(file_id);
+
+CREATE INDEX IF NOT EXISTS idx_external_file_text_index_revision_id
+    ON external_file_text_index(revision_id);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS search_external_file_text USING fts5(
+    extracted_text,
+    content='external_file_text_index',
+    content_rowid='id',
+    tokenize='unicode61 remove_diacritics 2',
+    prefix='2 3 4 5 6 7 8 9 10'
+);
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_ai
+AFTER INSERT ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(rowid, extracted_text)
+    VALUES (NEW.id, NEW.extracted_text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_ad
+AFTER DELETE ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(search_external_file_text, rowid, extracted_text)
+    VALUES ('delete', OLD.id, OLD.extracted_text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_external_file_text_index_au
+AFTER UPDATE OF extracted_text ON external_file_text_index BEGIN
+    INSERT INTO search_external_file_text(search_external_file_text, rowid, extracted_text)
+    VALUES ('delete', OLD.id, OLD.extracted_text);
+    INSERT INTO search_external_file_text(rowid, extracted_text)
+    VALUES (NEW.id, NEW.extracted_text);
+END;
 CREATE TRIGGER IF NOT EXISTS trg_blocks_ad AFTER DELETE ON blocks BEGIN
     INSERT INTO search_blocks(search_blocks, rowid, label, description)
     VALUES ('delete', OLD.rowid, OLD.label, OLD.description);
@@ -146,3 +193,4 @@ INSERT INTO search_element_text(search_element_text) VALUES ('rebuild');
 INSERT INTO search_element_doc(search_element_doc) VALUES ('rebuild');
 INSERT INTO search_element_model(search_element_model) VALUES ('rebuild');
 INSERT INTO search_blocks(search_blocks) VALUES ('rebuild');
+INSERT INTO search_external_file_text(search_external_file_text) VALUES ('rebuild');

{ducpy-3.4.0 → ducpy-3.5.1}/pyproject.toml

@@ -6,6 +6,7 @@ readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
     "nanoid>=2.0.0",
+    "pypdf>=4.2.0"
 ]
 
 license = { text = "MIT" }
@@ -19,6 +20,14 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
 ]
 
+[project.optional-dependencies]
+ocr = [
+    "rapidocr==3.8.1",
+    "onnxruntime",
+    "huggingface-hub>=0.20.0",
+    "numpy>=1.20.0",
+]
+
 [project.urls]
 "Homepage" = "https://duc.ducflair.com"
 "Source" = "https://github.com/ducflair/duc/tree/main/packages/ducpy"
@@ -44,11 +53,14 @@ where = ["src"]
 
 [dependency-groups]
 dev = [
+    "ducpy[ocr]",
+    "maturin>=1.0,<2.0",
     "furo>=2024.8.6",
     "sphinx>=8.1.3",
     "sphinx-autoapi>=3.6.0",
     "pytest>=8.3.5",
     "rich>=13.0.0",
+    "numpy>=1.20.0",
     "typst>=0.14.9",
     "build123d>=0.10.0",
     "ocp-vscode>=3.4.0",
@@ -60,3 +72,6 @@ dev = [
 testpaths = ["src/tests"]
 python_files = "test_*.py"
 addopts = "-vv"
+markers = [
+    "slow: marks tests as slow (OCR-heavy, etc.)",
+]

ducpy-3.5.1/src/ducpy/search/__init__.py

@@ -0,0 +1,19 @@
+"""Search helpers for DUC SQLite databases."""
+
+from .search_elements import (
+    DucElementSearchResult,
+    DucFileSearchResult,
+    DucSearchResponse,
+    DucSearchResult,
+    ExternalFileSearchTarget,
+    search_duc_elements,
+)
+
+__all__ = [
+    "DucElementSearchResult",
+    "DucFileSearchResult",
+    "DucSearchResponse",
+    "DucSearchResult",
+    "ExternalFileSearchTarget",
+    "search_duc_elements",
+]

ducpy-3.5.1/src/ducpy/search/image_ocr.py

@@ -0,0 +1,189 @@
+"""Server-side image OCR helpers for DUC search.
+
+Uses ``rapidocr-onnxruntime`` with custom PP-OCRv6_tiny ONNX models when the ``ocr`` extra is installed, otherwise
+OCR is gracefully skipped.
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+from functools import lru_cache
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_RAPID_OCR_ENGINE: Any | None = None
+
+
+@lru_cache(maxsize=1)
+def server_side_ocr_available() -> bool:
+    """Return whether the ``ocr`` extra (RapidOCR) is installed and usable."""
+
+    try:
+        from rapidocr import RapidOCR  # type: ignore
+        from huggingface_hub import snapshot_download  # type: ignore # noqa: F401
+        from PIL import Image  # type: ignore  # noqa: F401
+    except Exception as exc:
+        logger.debug("OCR extra (rapidocr/huggingface-hub) unavailable: %s", exc)
+        return False
+
+    return True
+
+
+def _get_local_cache_dir(repo_id: str) -> str | None:
+    import os
+    folder_name = "models--" + repo_id.replace("/", "--")
+    cache_dir = os.environ.get("HF_HUB_CACHE")
+    if not cache_dir:
+        hf_home = os.environ.get("HF_HOME")
+        if hf_home:
+            cache_dir = os.path.join(hf_home, "hub")
+        else:
+            cache_dir = os.path.expanduser("~/.cache/huggingface/hub")
+    
+    repo_dir = os.path.join(cache_dir, folder_name)
+    if not os.path.isdir(repo_dir):
+        return None
+        
+    # Read refs/main if it exists to get the current commit snapshot
+    refs_main = os.path.join(repo_dir, "refs", "main")
+    if os.path.isfile(refs_main):
+        try:
+            with open(refs_main, "r", encoding="utf-8") as f:
+                commit_hash = f.read().strip()
+            snapshot_dir = os.path.join(repo_dir, "snapshots", commit_hash)
+            if os.path.isdir(snapshot_dir):
+                return snapshot_dir
+        except Exception:
+            pass
+            
+    # Fallback to the latest folder in snapshots/
+    snapshots_dir = os.path.join(repo_dir, "snapshots")
+    if os.path.isdir(snapshots_dir):
+        try:
+            subdirs = [os.path.join(snapshots_dir, d) for d in os.listdir(snapshots_dir)]
+            subdirs = [d for d in subdirs if os.path.isdir(d)]
+            if subdirs:
+                return max(subdirs, key=os.path.getmtime)
+        except Exception:
+            pass
+            
+    return None
+
+
+def _get_rapid_engine() -> Any:
+    global _RAPID_OCR_ENGINE
+    if _RAPID_OCR_ENGINE is None:
+        from rapidocr import RapidOCR  # type: ignore
+        from rapidocr.utils.typings import EngineType  # type: ignore
+        import os
+        import yaml
+
+        # Try to resolve cache locally first to avoid snapshot_download overhead (0.1ms vs 180ms)
+        det_dir = _get_local_cache_dir("PaddlePaddle/PP-OCRv6_tiny_det_onnx")
+        rec_dir = _get_local_cache_dir("PaddlePaddle/PP-OCRv6_tiny_rec_onnx")
+
+        # Fallback to snapshot_download if local files aren't fully resolved
+        if (
+            not det_dir
+            or not rec_dir
+            or not os.path.exists(os.path.join(det_dir, "inference.onnx"))
+            or not os.path.exists(os.path.join(rec_dir, "inference.onnx"))
+        ):
+            from huggingface_hub import snapshot_download  # type: ignore
+            try:
+                det_dir = snapshot_download("PaddlePaddle/PP-OCRv6_tiny_det_onnx", local_files_only=True)
+                rec_dir = snapshot_download("PaddlePaddle/PP-OCRv6_tiny_rec_onnx", local_files_only=True)
+            except Exception:
+                det_dir = snapshot_download("PaddlePaddle/PP-OCRv6_tiny_det_onnx")
+                rec_dir = snapshot_download("PaddlePaddle/PP-OCRv6_tiny_rec_onnx")
+
+        det_model = os.path.join(det_dir, "inference.onnx")
+        rec_model = os.path.join(rec_dir, "inference.onnx")
+
+        # Load the dictionary from the model config to prevent IndexError
+        dict_path = os.path.join(rec_dir, "ppocrv6_keys.txt")
+        if not os.path.exists(dict_path):
+            yml_path = os.path.join(rec_dir, "inference.yml")
+            try:
+                with open(yml_path, "r", encoding="utf-8") as f:
+                    cfg = yaml.safe_load(f)
+                char_list = cfg["PostProcess"]["character_dict"]
+                with open(dict_path, "w", encoding="utf-8") as f:
+                    for char in char_list:
+                        f.write(char + "\n")
+            except Exception as e:
+                logger.debug("Failed to extract character_dict from inference.yml: %s", e)
+                # Fallback to the en_dict.txt shipped in the repository if yaml parsing fails
+                dict_path = os.path.join(rec_dir, "en_dict.txt")
+
+        _RAPID_OCR_ENGINE = RapidOCR(
+            params={
+                "Global.use_cls": False,
+
+                "Det.model_path": det_model,
+                "Det.engine_type": EngineType.ONNXRUNTIME,
+
+                "Rec.model_path": rec_model,
+                "Rec.rec_keys_path": dict_path,
+                "Rec.engine_type": EngineType.ONNXRUNTIME,
+            }
+        )
+    return _RAPID_OCR_ENGINE
+
+
+
+def extract_image_text_with_ocr(image_bytes: bytes, *, ocr_language: str) -> tuple[str, bool]:
+    """OCR an image using the ``ocr`` extra, or skip when unavailable."""
+
+    if not server_side_ocr_available():
+        return "", False
+
+    try:
+        import numpy as np  # type: ignore
+        from PIL import Image  # type: ignore
+    except Exception as exc:
+        logger.debug("Image OCR dependencies unavailable: %s", exc)
+        return "", False
+
+    try:
+        with Image.open(io.BytesIO(image_bytes)) as image_obj:
+            image_input = np.array(image_obj.convert("RGB"))
+
+        engine = _get_rapid_engine()
+        raw_res = engine(image_input)
+
+        if hasattr(raw_res, "txts"):
+            txts = raw_res.txts
+        elif isinstance(raw_res, tuple) and len(raw_res) == 2:
+            result, _ = raw_res
+            if hasattr(result, "txts"):
+                txts = result.txts
+            elif isinstance(result, (list, tuple)):
+                txts = [row[1] for row in result if isinstance(row, (tuple, list)) and len(row) >= 2]
+            else:
+                txts = None
+        elif isinstance(raw_res, (list, tuple)):
+            txts = [row[1] for row in raw_res if isinstance(row, (tuple, list)) and len(row) >= 2]
+        else:
+            txts = None
+
+        if not txts:
+            return "", False
+
+        lines: list[str] = []
+        for txt in txts:
+            value = str(txt or "").strip()
+            if value:
+                lines.append(value)
+        text = "\n".join(lines)
+    except Exception as exc:
+        logger.debug("Failed to OCR image bytes: %s", exc)
+        return "", False
+
+    return text, bool(text)
+
+
+_server_side_ocr_available = server_side_ocr_available
+_extract_image_text_with_ocr = extract_image_text_with_ocr