code-atlas-forge 1.1.0__py3-none-any.whl

code_atlas/search/vector/embeddings.py

@@ -0,0 +1,96 @@
+"""EmbeddingCache + build de embeddings por snapshot (lf-0039).
+
+Cache por ``content_hash`` (sha256 do texto de indexação): rebuild só
+quando conteúdo OU modelo mudam — PK ``(model_id, model_version,
+content_hash)`` torna o versionamento estrutural, não convenção.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import sqlite3
+
+from code_atlas.search.vector.contracts import (
+    EmbeddingModelInfo,
+    EmbeddingProvider,
+)
+from code_atlas.search.vector.index import pack_vector, unpack_vector
+
+
+def content_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+class SqliteEmbeddingCache:
+    """Cache persistente por conteúdo+modelo (tabela embedding_cache)."""
+
+    _GET = (
+        "SELECT vector FROM embedding_cache "
+        "WHERE model_id = ? AND model_version = ? AND content_hash = ?"
+    )
+    _PUT = (
+        "INSERT OR REPLACE INTO embedding_cache "
+        "(model_id, model_version, content_hash, dim, vector) "
+        "VALUES (?, ?, ?, ?, ?)"
+    )
+
+    def __init__(self, conn: sqlite3.Connection) -> None:
+        self._conn = conn
+
+    def get(
+        self, model: EmbeddingModelInfo, chash: str
+    ) -> list[float] | None:
+        row = self._conn.execute(
+            self._GET, (model.model_id, model.version, chash),
+        ).fetchone()
+        return unpack_vector(row["vector"]) if row else None
+
+    def put(
+        self, model: EmbeddingModelInfo, chash: str, vector: list[float]
+    ) -> None:
+        self._conn.execute(
+            self._PUT,
+            (model.model_id, model.version, chash, len(vector),
+             pack_vector(vector)),
+        )
+
+    def commit(self) -> None:
+        """Persiste as writes pendentes do cache (lote, não por linha)."""
+        self._conn.commit()
+
+
+def embed_texts_cached(
+    provider: EmbeddingProvider,
+    cache: SqliteEmbeddingCache,
+    texts: list[str],
+) -> tuple[list[list[float]], int, int]:
+    """Embed com cache: retorna (vetores, hits, misses) — misses sobem
+    em lote único ao provider."""
+    model = provider.info()
+    out: list[list[float] | None] = [None] * len(texts)
+    miss_idx: list[int] = []
+    hits = 0
+    for i, text in enumerate(texts):
+        vec = cache.get(model, content_hash(text))
+        if vec is None:
+            miss_idx.append(i)
+        else:
+            out[i] = vec
+            hits += 1
+    if miss_idx:
+        fresh = provider.embed([texts[i] for i in miss_idx])
+        for i, vec in zip(miss_idx, fresh, strict=True):
+            cache.put(model, content_hash(texts[i]), vec)
+            out[i] = vec
+        # Cache de embeddings é append-only e semanticamente
+        # independente do estado transacional do snapshot — commita o
+        # lote para que CLI/processos one-shot não percam o cache no
+        # rollback do close (lf-0142 finding).
+        cache.commit()
+    return [v if v is not None else [] for v in out], hits, len(miss_idx)
+
+
+def index_text_for(symbol_key: str, signature: str, docstring: str) -> str:
+    """Texto de indexação: fronteiras sintáticas (nome+signature+doc),
+    nunca chunk fixo."""
+    return "\n".join(p for p in (symbol_key, signature, docstring) if p)

code_atlas/search/vector/index.py

@@ -0,0 +1,169 @@
+"""VectorIndex persistido — sqlite-vec (opt-in) ou bruteforce (lf-0039).
+
+Backend escolhido por capability: ``sqlite_vec`` importável → vec0
+(KNN nativo); senão → bruteforce stdlib sobre BLOB float32 da mesma
+tabela ``vector_embeddings``. Ambos filtram por ``snapshot_id`` — o
+isolamento por snapshot é estrutural, nunca filtro pós-query.
+"""
+
+from __future__ import annotations
+
+import math
+import sqlite3
+import struct
+
+from code_atlas.search.vector.contracts import VectorSearchResult
+
+_UPSERT = (
+    "INSERT OR REPLACE INTO vector_embeddings "
+    "(snapshot_id, occurrence_id, symbol_key, model_id, model_version,"
+    " dim, content_hash, vector) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
+)
+_DELETE_SNAP = "DELETE FROM vector_embeddings WHERE snapshot_id = ?"
+
+
+def pack_vector(vec: list[float]) -> bytes:
+    """float32 little-endian — 4 bytes/dim, round-trip estável."""
+    return struct.pack(f"<{len(vec)}f", *vec)
+
+
+def unpack_vector(blob: bytes) -> list[float]:
+    n = len(blob) // 4
+    return list(struct.unpack(f"<{n}f", blob))
+
+
+def _cosine(a: list[float], b: list[float]) -> float:
+    dot = na = nb = 0.0
+    for x, y in zip(a, b, strict=True):
+        dot += x * y
+        na += x * x
+        nb += y * y
+    if na == 0.0 or nb == 0.0:
+        return 0.0
+    return max(0.0, min(1.0, dot / math.sqrt(na * nb)))
+
+
+class BruteForceIndex:
+    """Scan O(n·dim) com filtro de snapshot — honesto, sem deps."""
+
+    def __init__(
+        self, conn: sqlite3.Connection, model_id: str, model_version: str
+    ) -> None:
+        self._conn = conn
+        self._model_id = model_id
+        self._model_version = model_version
+
+    def upsert(
+        self, snapshot_id: int, occurrence_id: int,
+        symbol_key: str, vector: list[float],
+        content_hash: str = "",
+    ) -> None:
+        self._conn.execute(
+            _UPSERT,
+            (snapshot_id, occurrence_id, symbol_key, self._model_id,
+             self._model_version, len(vector), content_hash,
+             pack_vector(vector)),
+        )
+
+    def search(
+        self, snapshot_id: int, query: list[float], limit: int
+    ) -> list[VectorSearchResult]:
+        rows = self._conn.execute(
+            "SELECT occurrence_id, symbol_key, vector "
+            "FROM vector_embeddings "
+            "WHERE snapshot_id = ? AND model_id = ? AND model_version = ?",
+            (snapshot_id, self._model_id, self._model_version),
+        ).fetchall()
+        scored = [
+            VectorSearchResult(
+                symbol_key=r["symbol_key"],
+                occurrence_id=int(r["occurrence_id"]),
+                score=_cosine(query, unpack_vector(r["vector"])),
+                model_id=self._model_id,
+            )
+            for r in rows
+        ]
+        scored.sort(key=lambda r: (-r.score, r.occurrence_id))
+        return scored[: max(1, limit)]
+
+    def delete_snapshot(self, snapshot_id: int) -> None:
+        self._conn.execute(_DELETE_SNAP, (snapshot_id,))
+
+
+def sqlite_vec_available() -> bool:
+    """sqlite-vec instalado? (extra opt-in ``[vector]``)"""
+    try:
+        import sqlite_vec  # type: ignore[import-not-found]  # noqa: F401
+    except ImportError:
+        return False
+    return True
+
+
+class SqliteVecIndex(BruteForceIndex):
+    """Backend nativo via sqlite-vec quando o extra está instalado.
+
+    Mesma tabela de persistência (interoperável com bruteforce); só a
+    busca usa a virtual table vec0 — fallback ao scan se a extensão
+    falhar em runtime.
+    """
+
+    def __init__(
+        self, conn: sqlite3.Connection, model_id: str,
+        model_version: str, dim: int
+    ) -> None:
+        super().__init__(conn, model_id, model_version)
+        self._dim = dim
+        import sqlite_vec
+
+        sqlite_vec.load(conn)
+        # SQL nomeado — auditoria AST exige literal/nome, nunca expr.
+        create_vec_sql = (
+            "CREATE VIRTUAL TABLE IF NOT EXISTS vec_symbols USING vec0("
+            "snapshot_id INTEGER, occurrence_id INTEGER, "
+            f"embedding FLOAT[{dim}])"
+        )
+        conn.execute(create_vec_sql)
+
+    def upsert(
+        self, snapshot_id: int, occurrence_id: int,
+        symbol_key: str, vector: list[float],
+        content_hash: str = "",
+    ) -> None:
+        super().upsert(
+            snapshot_id, occurrence_id, symbol_key, vector, content_hash)
+        self._conn.execute(
+            "DELETE FROM vec_symbols WHERE occurrence_id = ?",
+            (occurrence_id,))
+        self._conn.execute(
+            "INSERT INTO vec_symbols "
+            "(snapshot_id, occurrence_id, embedding) VALUES (?, ?, ?)",
+            (snapshot_id, occurrence_id, pack_vector(vector)),
+        )
+
+    def search(
+        self, snapshot_id: int, query: list[float], limit: int
+    ) -> list[VectorSearchResult]:
+        try:
+            rows = self._conn.execute(
+                "SELECT occurrence_id, distance FROM vec_symbols "
+                "WHERE embedding MATCH ? AND k = ? "
+                "AND snapshot_id = ? ORDER BY distance",
+                (pack_vector(query), max(1, limit), snapshot_id),
+            ).fetchall()
+        except sqlite3.Error:
+            return super().search(snapshot_id, query, limit)
+        return [
+            VectorSearchResult(
+                symbol_key="",  # preenchido na fusão via occurrence
+                occurrence_id=int(r["occurrence_id"]),
+                score=max(0.0, 1.0 - float(r["distance"])),
+                model_id=self._model_id,
+            )
+            for r in rows
+        ]
+
+    def delete_snapshot(self, snapshot_id: int) -> None:
+        super().delete_snapshot(snapshot_id)
+        self._conn.execute(
+            "DELETE FROM vec_symbols WHERE snapshot_id = ?",
+            (snapshot_id,))

code_atlas/search/vector/provider.py

@@ -0,0 +1,50 @@
+"""Provider de embeddings local — baseline naïve stdlib (lf-0039).
+
+``NaiveEmbeddingProvider``: hashing de tokens com sinal (feature
+hashing) → vetor L2-normalizado. Determinístico, offline, zero deps.
+
+Baseline HONESTO: captura sobreposição lexical (útil para RRF), NÃO
+semântica real — sinônimos ("order"/"pedido") não aproximam. Trocável
+via seam ``EmbeddingProvider`` (provider externo só opt-in explícito;
+default nunca faz egress).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import re
+
+from code_atlas.search.vector.contracts import EmbeddingModelInfo
+
+_TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+
+
+class NaiveEmbeddingProvider:
+    """Feature hashing: sha256(token) → (dim, sinal); soma e normaliza."""
+
+    def __init__(self, dim: int = 256) -> None:
+        if dim <= 0:
+            raise ValueError("dim deve ser > 0")
+        self._dim = dim
+        self._info = EmbeddingModelInfo(
+            model_id="naive-hash", version="v1", dim=dim,
+            detail="feature hashing de tokens (baseline não-semântico)")
+
+    def info(self) -> EmbeddingModelInfo:
+        return self._info
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        return [self._embed_one(t) for t in texts]
+
+    def _embed_one(self, text: str) -> list[float]:
+        vec = [0.0] * self._dim
+        for tok in _TOKEN_RE.findall(text.lower()):
+            h = hashlib.sha256(tok.encode("utf-8")).digest()
+            bucket = int.from_bytes(h[:4], "big") % self._dim
+            sign = 1.0 if h[4] & 1 else -1.0
+            vec[bucket] += sign
+        norm = math.sqrt(sum(v * v for v in vec))
+        if norm == 0.0:
+            return vec
+        return [v / norm for v in vec]

code_atlas/storage/__init__.py

@@ -0,0 +1,14 @@
+"""Persistência SQLite (WAL, migrações, snapshots)."""
+
+from code_atlas.storage.db import connect, fts5_available
+from code_atlas.storage.migrations import MigrationError, migrate, schema_version
+from code_atlas.storage.store import Store
+
+__all__ = [
+    "MigrationError",
+    "Store",
+    "connect",
+    "fts5_available",
+    "migrate",
+    "schema_version",
+]

code_atlas/storage/db.py

@@ -0,0 +1,47 @@
+"""Fábrica de conexões SQLite (Blueprint ADR-006).
+
+Pragmas obrigatórios: WAL, foreign_keys=ON, busy_timeout, synchronous=NORMAL.
+Todos os caminhos de escrita usam transações explícitas; escritor único por
+workspace é coordenado pelo IndexCoordinator via ``BEGIN IMMEDIATE``.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+
+BUSY_TIMEOUT_MS = 5000
+
+
+def connect(path: Path | str, *, read_only: bool = False) -> sqlite3.Connection:
+    """Abre conexão configurada para o workspace Atlas.
+
+    ``read_only=True`` usa URI ``mode=ro`` para leitores concorrentes que
+    nunca devem escrever nem criar o arquivo.
+    """
+    if read_only:
+        uri = f"file:{Path(path).as_posix()}?mode=ro"
+        conn = sqlite3.connect(uri, uri=True, timeout=BUSY_TIMEOUT_MS / 1000)
+    else:
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        conn = sqlite3.connect(str(path), timeout=BUSY_TIMEOUT_MS / 1000)
+    conn.execute("PRAGMA foreign_keys = ON")
+    # timeout= do connect() já aplica busy_timeout; PRAGMA não aceita bind.
+    if not read_only:
+        conn.execute("PRAGMA journal_mode = WAL")
+        conn.execute("PRAGMA synchronous = NORMAL")
+        # 64MiB de page cache — barateia a manutenção de índices/FKs
+        # durante persist de snapshots grandes (lf-0023, medido ~4%).
+        conn.execute("PRAGMA cache_size = -64000")
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def fts5_available(conn: sqlite3.Connection) -> bool:
+    """True se a build do sqlite suporta FTS5."""
+    try:
+        conn.execute("CREATE VIRTUAL TABLE temp._fts_probe USING fts5(x)")
+        conn.execute("DROP TABLE temp._fts_probe")
+        return True
+    except sqlite3.Error:
+        return False