marvisx-cli 0.1.0__py3-none-any.whl

core/api/services/ingest/embedding_router.py

@@ -0,0 +1,272 @@
+"""Embedding routing for Universal Ingestion E2.1."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from functools import lru_cache
+from pathlib import Path
+from typing import Literal
+
+import aiosqlite
+
+from core.api.config import settings
+from core.api.db import acquire_write_db
+from core.api.services import embedding_service
+from core.api.services.pii_redactor import redact
+
+logger = logging.getLogger(__name__)
+
+EmbedStatus = Literal["voyage", "nomic-local", "skipped"]
+ProjectType = Literal["work", "code", "system"]
+NOMIC_MODEL_PATH = Path(
+    os.environ.get("NOMIC_MODEL_PATH", "/data/pir/models/nomic-embed-text-v1.5")
+)
+_LOCAL_MODEL_MISSING_LOGGED = False
+
+
+async def embed_and_index(
+    *,
+    ingest_id: str,
+    slug: str,
+    target_path: Path,
+    extracted_text: str | None,
+    document_type: str = "file",
+    title: str | None = None,
+    project_type: ProjectType = "work",
+) -> EmbedStatus:
+    """Redact, embed, and index one ingested file.
+
+    Work projects skip embeddings unless external embedding is explicitly
+    enabled. Code/system projects use local Nomic by default and route to Voyage
+    only through migration 095's project opt-in table.
+    """
+    if not extracted_text or len(extracted_text.strip()) < 20:
+        logger.info(
+            "embedding backend: skipped ingest_id=%s reason=text-too-short",
+            ingest_id,
+        )
+        return "skipped"
+
+    allow_external = await _project_allows_external_embedding(slug)
+    if project_type == "work" and not allow_external:
+        logger.info(
+            "embedding backend: skipped ingest_id=%s reason=work-no-opt-in",
+            ingest_id,
+        )
+        return "skipped"
+
+    redacted_text = redact(extracted_text)
+    backend: Literal["voyage", "nomic-local"]
+    try:
+        if allow_external:
+            vector = await _embed_voyage_text(redacted_text)
+            backend = "voyage"
+        else:
+            if not _local_embedding_model_available():
+                _log_local_embedding_missing_once("local-model-missing")
+                logger.info(
+                    "embedding backend: skipped ingest_id=%s reason=local-model-missing",
+                    ingest_id,
+                )
+                return "skipped"
+            vector = await asyncio.to_thread(_embed_nomic_text_sync, redacted_text)
+            backend = "nomic-local"
+    except Exception:
+        if allow_external:
+            logger.warning(
+                "Voyage embedding failed; falling back to nomic-local", exc_info=True
+            )
+            if not _local_embedding_model_available():
+                _log_local_embedding_missing_once("local-fallback-model-missing")
+                logger.info(
+                    "embedding backend: skipped ingest_id=%s reason=local-fallback-model-missing",
+                    ingest_id,
+                )
+                return "skipped"
+            try:
+                vector = await asyncio.to_thread(_embed_nomic_text_sync, redacted_text)
+                backend = "nomic-local"
+            except Exception:
+                logger.warning(
+                    "Local embedding fallback failed; skipping index", exc_info=True
+                )
+                return "skipped"
+        else:
+            logger.warning("Local embedding unavailable; skipping index", exc_info=True)
+            return "skipped"
+
+    await _persist_ingest_embedding(
+        slug=slug,
+        target_path=target_path,
+        document_type=document_type,
+        title=title or target_path.stem,
+        content=extracted_text,
+        vector=_coerce_dimensions(vector),
+    )
+    logger.info(
+        "embedding backend: %s ingest_id=%s project=%s", backend, ingest_id, slug
+    )
+    return backend
+
+
+async def _project_allows_external_embedding(slug: str) -> bool:
+    try:
+        async with acquire_write_db() as db:
+            async with db.execute(
+                """
+                SELECT allow_external_embed
+                  FROM project_external_embedding_policy
+                 WHERE project_slug = ?
+                """,
+                (slug,),
+            ) as cursor:
+                row = await cursor.fetchone()
+    except Exception:
+        logger.warning(
+            "external embedding policy unavailable; defaulting to local", exc_info=True
+        )
+        return False
+    return bool(row and row["allow_external_embed"])
+
+
+async def _embed_voyage_text(text: str) -> list[float]:
+    if not embedding_service.is_available():
+        embedding_service.init_voyage_client()
+    if not embedding_service.is_available():
+        raise RuntimeError("Voyage AI client not initialized")
+
+    last_error: Exception | None = None
+    for attempt, delay in enumerate((0.0, 0.4, 1.5, 4.0), start=1):
+        if delay:
+            await asyncio.sleep(delay)
+        try:
+            vectors = await embedding_service.embed_texts([text], input_type="document")
+            return vectors[0]
+        except Exception as exc:
+            last_error = exc
+            message = str(exc)
+            if any(code in message for code in ("400", "401", "403", "422")):
+                raise
+            logger.warning("Voyage attempt %d failed: %s", attempt, exc)
+    assert last_error is not None
+    raise last_error
+
+
+def _embed_nomic_text_sync(text: str) -> list[float]:
+    model = _nomic_model()
+    try:
+        vector = model.encode(
+            text,
+            normalize_embeddings=True,
+            truncate_dim=embedding_service.DIMENSIONS,
+        )
+    except TypeError:
+        vector = model.encode(text, normalize_embeddings=True)
+    return vector.tolist()
+
+
+def _local_embedding_model_available() -> bool:
+    return NOMIC_MODEL_PATH.exists()
+
+
+def _log_local_embedding_missing_once(reason: str) -> None:
+    global _LOCAL_MODEL_MISSING_LOGGED
+    if _LOCAL_MODEL_MISSING_LOGGED:
+        return
+    _LOCAL_MODEL_MISSING_LOGGED = True
+    logger.warning(
+        "embedding backend: local embedding disabled reason=%s path=%s",
+        reason,
+        NOMIC_MODEL_PATH,
+    )
+
+
+@lru_cache(maxsize=1)
+def _nomic_model():
+    from sentence_transformers import SentenceTransformer
+
+    return SentenceTransformer(str(NOMIC_MODEL_PATH), trust_remote_code=True)
+
+
+def _coerce_dimensions(vector: list[float]) -> list[float]:
+    if len(vector) == embedding_service.DIMENSIONS:
+        return vector
+    if len(vector) > embedding_service.DIMENSIONS:
+        return vector[: embedding_service.DIMENSIONS]
+    padding = [0.0] * (embedding_service.DIMENSIONS - len(vector))
+    return [*vector, *padding]
+
+
+async def _persist_ingest_embedding(
+    *,
+    slug: str,
+    target_path: Path,
+    document_type: str,
+    title: str,
+    content: str,
+    vector: list[float],
+) -> None:
+    file_path = str(target_path)
+    async with acquire_write_db() as db:
+        await db.execute(
+            """INSERT INTO documents (file_path, project, workspace_id, doc_type, doc_title, content_hash)
+               VALUES (?, ?, 'ws_default', ?, ?, ?)
+               ON CONFLICT(file_path) DO UPDATE SET
+                 content_hash = excluded.content_hash,
+                 project = excluded.project,
+                 workspace_id = excluded.workspace_id,
+                 doc_type = excluded.doc_type,
+                 doc_title = excluded.doc_title""",
+            [
+                file_path,
+                slug,
+                document_type,
+                title,
+                embedding_service.content_hash(content),
+            ],
+        )
+        async with db.execute(
+            "SELECT id FROM documents WHERE file_path = ?", [file_path]
+        ) as cursor:
+            row = await cursor.fetchone()
+        if row is None:
+            raise RuntimeError(f"document upsert failed for {file_path}")
+
+        if await _ensure_vec_documents(db):
+            await db.execute("DELETE FROM vec_documents WHERE doc_id = ?", [row["id"]])
+            await db.execute(
+                "INSERT INTO vec_documents (doc_id, embedding) VALUES (?, ?)",
+                [row["id"], embedding_service.serialize_f32(vector)],
+            )
+        await db.commit()
+
+
+async def _ensure_vec_documents(db: aiosqlite.Connection) -> bool:
+    vec_path = Path(settings.vec0_path)
+    vec_so = vec_path.with_suffix(".so") if not vec_path.suffix else vec_path
+    if not vec_so.exists():
+        logger.warning(
+            "sqlite-vec extension missing at %s; document metadata stored without vector",
+            vec_so,
+        )
+        return False
+    try:
+        await db._execute(db._conn.enable_load_extension, True)
+        await db.execute("SELECT load_extension(?)", [str(vec_path)])
+        await db.execute(
+            """
+            CREATE VIRTUAL TABLE IF NOT EXISTS vec_documents USING vec0(
+                doc_id INTEGER PRIMARY KEY,
+                embedding float[512]
+            )
+            """
+        )
+        return True
+    except Exception:
+        logger.warning(
+            "sqlite-vec setup failed; document metadata stored without vector",
+            exc_info=True,
+        )
+        return False

core/api/services/ingest/events.py

@@ -0,0 +1,33 @@
+"""Best-effort websocket notifications for ingest state changes."""
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+async def broadcast_ingest_changed(
+    event: str,
+    *,
+    ingest_id: str | None = None,
+    project_slug: str | None = None,
+    status: str | None = None,
+    extra: dict[str, Any] | None = None,
+) -> None:
+    payload: dict[str, Any] = {"type": "ingest_changed", "event": event}
+    if ingest_id is not None:
+        payload["ingest_id"] = ingest_id
+    if project_slug is not None:
+        payload["project_slug"] = project_slug
+    if status is not None:
+        payload["status"] = status
+    if extra:
+        payload.update(extra)
+
+    try:
+        from core.api.terminal import session_manager
+
+        await session_manager.broadcast_control_message(payload)
+    except Exception:
+        logger.exception("failed to broadcast ingest_changed event")

core/api/services/ingest/ignore_patterns.py

@@ -0,0 +1,79 @@
+"""Shared ignore rules for container ingest inputs."""
+from __future__ import annotations
+
+from pathlib import PurePosixPath
+
+MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024
+
+IGNORED_DIRECTORY_NAMES = {
+    ".git",
+    ".hg",
+    ".svn",
+    ".venv",
+    "__pycache__",
+    "build",
+    "coverage",
+    "dist",
+    "node_modules",
+    "tmp",
+}
+
+IGNORED_FILE_NAMES = {
+    ".DS_Store",
+    "Thumbs.db",
+    "desktop.ini",
+}
+
+IGNORED_SUFFIXES = {
+    ".app",
+    ".bat",
+    ".bin",
+    ".cmd",
+    ".com",
+    ".dll",
+    ".dmg",
+    ".exe",
+    ".iso",
+    ".lnk",
+    ".msi",
+    ".o",
+    ".out",
+    ".part",
+    ".pyc",
+    ".so",
+    ".swp",
+    ".tmp",
+}
+
+
+def normalize_relative_path(path: str | PurePosixPath) -> PurePosixPath:
+    normalized = str(path).replace("\\", "/")
+    return PurePosixPath(normalized)
+
+
+def is_ignored_directory_name(name: str) -> bool:
+    return name in IGNORED_DIRECTORY_NAMES
+
+
+def should_ignore(
+    relative_path: str | PurePosixPath,
+    *,
+    file_size_bytes: int | None = None,
+) -> str | None:
+    path = normalize_relative_path(relative_path)
+    parts = [part for part in path.parts if part not in {"", "."}]
+    if not parts:
+        return "empty-path"
+    if any(part == ".." for part in parts):
+        return "path-traversal"
+    if any(is_ignored_directory_name(part) for part in parts[:-1]):
+        return "ignored-directory"
+
+    filename = parts[-1]
+    if filename in IGNORED_FILE_NAMES or filename.startswith("."):
+        return "ignored-file"
+    if PurePosixPath(filename).suffix.lower() in IGNORED_SUFFIXES:
+        return "ignored-suffix"
+    if file_size_bytes is not None and file_size_bytes > MAX_FILE_SIZE_BYTES:
+        return "file-too-large"
+    return None

core/api/services/ingest/image_probe.py

@@ -0,0 +1,218 @@
+"""Lightweight image evidence for ingest routing decisions."""
+from __future__ import annotations
+
+from pathlib import Path
+from statistics import mean, pstdev
+from typing import Any
+
+from PIL import Image
+
+BILL_TERMS = {"bolletta", "fattura", "invoice", "pod", "pdr", "kwh", "energia", "gas"}
+IDENTITY_TERMS = {
+    "carta-identita",
+    "carta_identita",
+    "identity",
+    "passport",
+    "passaporto",
+    "codice-fiscale",
+}
+DOCUMENT_TERMS = {"contratto", "contract", "report", "modulo", "form", "documento"}
+
+
+def probe_image(path: Path) -> dict[str, Any]:
+    """Return cheap, auditable image signals without OCR or vision calls."""
+    try:
+        with Image.open(path) as image:
+            exif = image.getexif()
+            width = int(image.width)
+            height = int(image.height)
+            orientation = exif.get(0x0112) if exif else None
+            camera_hint = bool(exif.get(0x010F) or exif.get(0x0110)) if exif else False
+            stats = _pixel_stats(image)
+            aspect_ratio = round(width / height, 4) if height else None
+            suffix = path.suffix.lower()
+            name_score = _filename_document_score(path)
+            screenshot_hint = _screenshot_hint(
+                suffix=suffix,
+                width=width,
+                height=height,
+                camera_hint=camera_hint,
+            )
+            document_boundary_hint = _document_boundary_hint(width, height)
+            poor_capture_hint = _poor_capture_hint(width, height, orientation)
+
+            document_likelihood = _clamp(
+                0.12
+                + (0.35 if document_boundary_hint else 0.0)
+                + (0.18 if stats["white_ratio"] >= 0.45 else 0.0)
+                + (0.16 if stats["edge_density"] >= 0.08 else 0.0)
+                + name_score
+                - (0.18 if camera_hint and stats["white_ratio"] < 0.35 else 0.0)
+            )
+            screenshot_likelihood = _clamp(
+                0.10
+                + (0.45 if screenshot_hint else 0.0)
+                + (0.18 if width > height else 0.0)
+                + (0.14 if stats["edge_density"] >= 0.10 else 0.0)
+                - (0.28 if camera_hint else 0.0)
+            )
+            photo_likelihood = _clamp(
+                0.18
+                + (0.42 if camera_hint else 0.0)
+                + (0.18 if stats["white_ratio"] < 0.25 else 0.0)
+                + (0.10 if stats["contrast"] > 55 else 0.0)
+                - (0.20 if screenshot_hint else 0.0)
+            )
+            text_likelihood = _clamp(
+                0.20
+                + (0.35 if stats["edge_density"] >= 0.10 else 0.0)
+                + (0.18 if stats["contrast"] >= 35 else 0.0)
+                + (0.15 if document_boundary_hint else 0.0)
+            )
+
+            return {
+                "kind": "image",
+                "format": image.format,
+                "width": width,
+                "height": height,
+                "aspect_ratio": aspect_ratio,
+                "orientation": orientation,
+                "exif_camera_hint": camera_hint,
+                "exif_tag_count": len(exif or {}),
+                "screenshot_hint": screenshot_hint,
+                "document_boundary_hint": document_boundary_hint,
+                "poor_capture_hint": poor_capture_hint,
+                "white_background_ratio": stats["white_ratio"],
+                "edge_density": stats["edge_density"],
+                "brightness": stats["brightness"],
+                "contrast": stats["contrast"],
+                "document_likelihood": round(document_likelihood, 4),
+                "screenshot_likelihood": round(screenshot_likelihood, 4),
+                "photo_likelihood": round(photo_likelihood, 4),
+                "text_likelihood": round(text_likelihood, 4),
+                "image_kind": _image_kind(
+                    document_likelihood=document_likelihood,
+                    screenshot_likelihood=screenshot_likelihood,
+                    photo_likelihood=photo_likelihood,
+                ),
+                "signals": _signals(
+                    screenshot_hint=screenshot_hint,
+                    document_boundary_hint=document_boundary_hint,
+                    poor_capture_hint=poor_capture_hint,
+                    camera_hint=camera_hint,
+                    name_score=name_score,
+                    stats=stats,
+                ),
+            }
+    except Exception:
+        return {"kind": "image", "preflight_error": "image_probe_failed"}
+
+
+def _pixel_stats(image: Image.Image) -> dict[str, float]:
+    sample = image.convert("L")
+    sample.thumbnail((160, 160))
+    width, height = sample.size
+    pixels = list(sample.tobytes())
+    if not pixels:
+        return {"white_ratio": 0.0, "edge_density": 0.0, "brightness": 0.0, "contrast": 0.0}
+
+    white_ratio = sum(1 for value in pixels if value >= 235) / len(pixels)
+    brightness = mean(pixels)
+    contrast = pstdev(pixels) if len(pixels) > 1 else 0.0
+    edge_hits = 0
+    comparisons = 0
+    for y in range(height):
+        row = y * width
+        for x in range(width):
+            value = pixels[row + x]
+            if x + 1 < width:
+                comparisons += 1
+                edge_hits += abs(value - pixels[row + x + 1]) >= 32
+            if y + 1 < height:
+                comparisons += 1
+                edge_hits += abs(value - pixels[row + width + x]) >= 32
+    edge_density = edge_hits / comparisons if comparisons else 0.0
+    return {
+        "white_ratio": round(white_ratio, 4),
+        "edge_density": round(edge_density, 4),
+        "brightness": round(brightness, 2),
+        "contrast": round(contrast, 2),
+    }
+
+
+def _filename_document_score(path: Path) -> float:
+    stem = path.stem.lower()
+    if any(term in stem for term in BILL_TERMS | IDENTITY_TERMS):
+        return 0.28
+    if any(term in stem for term in DOCUMENT_TERMS):
+        return 0.18
+    return 0.0
+
+
+def _screenshot_hint(*, suffix: str, width: int, height: int, camera_hint: bool) -> bool:
+    if camera_hint:
+        return False
+    if suffix != ".png":
+        return False
+    return width >= 800 and height >= 500
+
+
+def _document_boundary_hint(width: int, height: int) -> bool:
+    if not width or not height:
+        return False
+    aspect = width / height
+    return 0.55 <= aspect <= 1.9
+
+
+def _poor_capture_hint(width: int, height: int, orientation: Any) -> bool:
+    return width * height < 600_000 or orientation in {3, 6, 8}
+
+
+def _image_kind(
+    *,
+    document_likelihood: float,
+    screenshot_likelihood: float,
+    photo_likelihood: float,
+) -> str:
+    ranked = sorted(
+        [
+            ("document", document_likelihood),
+            ("screenshot", screenshot_likelihood),
+            ("photo", photo_likelihood),
+        ],
+        key=lambda item: item[1],
+        reverse=True,
+    )
+    kind, score = ranked[0]
+    return kind if score >= 0.62 else "ambiguous"
+
+
+def _signals(
+    *,
+    screenshot_hint: bool,
+    document_boundary_hint: bool,
+    poor_capture_hint: bool,
+    camera_hint: bool,
+    name_score: float,
+    stats: dict[str, float],
+) -> list[str]:
+    signals: list[str] = []
+    if screenshot_hint:
+        signals.append("screenshot_hint")
+    if document_boundary_hint:
+        signals.append("document_boundary_hint")
+    if poor_capture_hint:
+        signals.append("poor_capture_hint")
+    if camera_hint:
+        signals.append("exif_camera_hint")
+    if name_score:
+        signals.append("filename_document_terms")
+    if stats["white_ratio"] >= 0.45:
+        signals.append("white_background")
+    if stats["edge_density"] >= 0.10:
+        signals.append("edge_dense")
+    return signals
+
+
+def _clamp(value: float) -> float:
+    return max(0.0, min(1.0, value))