atif-sql 0.2.0__py3-none-any.whl

atif_embed/domain/errors.py

@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Domain error taxonomy for atif-embed.
+
+Every failure that can reach the CLI must be a :class:`DomainError`: the
+``embed`` command catches only that base class and turns it into a
+classified ``{kind, exit_code, hint}`` envelope. Anything escaping the
+taxonomy surfaces as a raw traceback instead.
+"""
+
+from __future__ import annotations
+
+
+class DomainError(Exception):
+    """Base class for atif-embed domain errors.
+
+    ``terminal`` distinguishes conditions no retry can clear (the store or
+    its config needs an operator) from transient runs the next tick may
+    succeed at. The CLI maps terminal errors to their own exit code so an
+    unattended lane can stop retrying instead of burning identical ticks —
+    the 2026-08-24 schema-stale condition retried 400+ times over 3 days
+    with zero escalation because every failure exited alike.
+    """
+
+    #: True when retrying without operator action cannot succeed.
+    terminal: bool = False
+
+
+class EmbeddingProviderMismatch(DomainError):  # noqa: N818 — names a store state, not an "*Error"
+    """Raised on a stamped-vs-active ``(model, dim)`` mismatch.
+
+    The Lance store records the identity of the embedder that wrote it, and it
+    differs from the identity of the embedder now asking to read or extend it.
+
+    Different embedding models produce vectors in incompatible spaces, so
+    appending or querying across a provider switch yields numerically valid
+    but semantically garbage cosine scores. This error is terminal: the store
+    must be dropped and re-embedded under the new provider.
+    """
+
+    terminal = True
+
+
+class EmbeddingStoreSchemaStale(DomainError):  # noqa: N818 — names a store state, not an "*Error"
+    """A Lance store's schema requires operator action before embeds can run.
+
+    ADDITIVE drift no longer raises this: a store missing ``text_hash``
+    migrates online via schema evolution + the sentinel staleness path (see
+    ``infrastructure.lance_store``). The class stays in the taxonomy for
+    schema states that genuinely cannot self-heal, and as the terminal
+    exemplar the refresh lane's retry suppression keys on.
+    """
+
+    terminal = True
+
+
+class EmbeddingProviderUnavailable(DomainError):  # noqa: N818 — names a provider state, not an "*Error"
+    """The embedding backend could not be reached or refused the request.
+
+    Covers what escapes tenacity's retry budget plus the non-retryable
+    botocore failures (expired or denied credentials, a rejected request
+    shape). Terminal for this run; the next run's staleness anti-join
+    re-picks whatever never got written.
+    """
+
+
+class EmbeddingProviderNotInstalled(DomainError):  # noqa: N818 — names an install state, not an "*Error"
+    """The selected provider's libraries are not installed.
+
+    EmbeddingGemma 2 runs through the optional ``local`` extra (torch,
+    transformers, sentence-transformers). Selecting it without the extra
+    cannot succeed on a retry: an operator installs the extra or selects
+    another provider, so the error is terminal.
+    """
+
+    terminal = True
+
+
+class EmbeddingResponseInvalid(DomainError):  # noqa: N818 — names a response state, not an "*Error"
+    """The backend answered, but not in the shape the adapter can read.
+
+    Distinct from :class:`EmbeddingProviderUnavailable` because it points at
+    a wire-format change rather than at credentials or capacity.
+    """
+
+
+__all__ = [
+    "DomainError",
+    "EmbeddingProviderMismatch",
+    "EmbeddingProviderNotInstalled",
+    "EmbeddingProviderUnavailable",
+    "EmbeddingResponseInvalid",
+    "EmbeddingStoreSchemaStale",
+]

atif_embed/domain/ports.py

@@ -0,0 +1,223 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Ports for atif-embed: what the embed use case NEEDS from the world.
+
+Four seams:
+
+* :class:`EmbeddingProvider` — async document embedding (batched,
+  float-widened) + sync query embedding, stamped with ``model_id`` /
+  ``dimension``.
+* :class:`VectorStorePort` — the Lance-backed store surface.
+* :class:`TextRowsPort` — the corpus seam. atif-embed may never import
+  atif-duck (import-linter independence contract), so the embeddable text
+  rows come through this port; the duckdb implementation in atif-embed's OWN
+  infrastructure reads the CONTRACT corpus layout directly (the ConverterPort
+  precedent from atif-corpus).
+* :class:`LakeStepsPort` — the corpus's steps as the DuckLake holds them.
+  atif-duck owns the lake, so atif-cli implements this port over it and
+  hands it to :class:`~atif_embed.infrastructure.lake_text_rows.LakeTextRows`,
+  which reads only what changed since the last complete pass.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol, Self
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Iterator
+    from pathlib import Path
+    from types import TracebackType
+
+    import polars as pl
+
+    from atif_embed.domain.text_stamp import PendingText
+
+
+class EmbeddingProvider(Protocol):
+    """Anything that can embed documents and queries into one vector space."""
+
+    @property
+    def model_id(self) -> str:
+        """Globally unique model identity stamped on every stored row."""
+        ...
+
+    @property
+    def dimension(self) -> int:
+        """Fixed output vector width."""
+        ...
+
+    async def embed_documents(self, texts: list[str]) -> list[list[float] | None]:
+        """Embed corpus documents; one slot per input text, in the same order.
+
+        A slot is ``None`` when that text could not be embedded. Returning
+        per-text outcomes rather than raising is what makes loss bounded: a
+        batch that fails terminally must not discard the sibling batches
+        whose embeddings were already billed.
+        """
+        ...
+
+    def embed_query(self, text: str) -> list[float]:
+        """Embed one query string as a float vector of length :attr:`dimension`."""
+        ...
+
+
+class VectorStorePort(Protocol):
+    """The embeddings store surface the backfill writes through."""
+
+    def table_identity(self) -> tuple[str, int] | None:
+        """Return the store's stamped ``(model, dim)``, or ``None`` when empty."""
+        ...
+
+    def get_embedded_hashes(self) -> dict[str, str]:
+        """Return ``{uuid: text_hash}`` for every row currently embedded."""
+        ...
+
+    def delete_uuids(self, uuids: Iterable[str]) -> int:
+        """Remove the rows for ``uuids``; return how many uuids were named."""
+        ...
+
+    def add_chunk(self, df: pl.DataFrame) -> None:
+        """Append one chunk of embedding rows."""
+        ...
+
+    def optimize(self) -> None:
+        """Compact accumulated fragments (best-effort)."""
+        ...
+
+    def ensure_index(self, *, metric: str = "cosine") -> None:
+        """Create the vector index if missing (best-effort)."""
+        ...
+
+
+class TextRowsPort(Protocol):
+    """Anything that can yield the corpus's embeddable text rows.
+
+    Contract semantics (CONTRACT-V2 §VSS): the embeddable unit is a step's
+    flattened text — main chain AND sidechain — of at least 32 characters,
+    keyed by the step's PRIMARY uuid (the first ``extra.source_uuids`` entry;
+    steps without source uuids are skipped, they cannot join back to the
+    ``messages``/edges surface).
+    """
+
+    def iter_unembedded(
+        self,
+        corpus_root: Path,
+        *,
+        embedded: dict[str, str] | None = None,
+        limit: int | None = None,
+    ) -> Iterator[PendingText]:
+        """Yield the texts needing embedding, lazily.
+
+        ``embedded`` is the store's ``{uuid: text_hash}`` map. A uuid absent
+        from it is NEW; a uuid present under a different hash is STALE (its
+        text changed since it was embedded) and is yielded with
+        ``replaces_existing=True``.
+
+        Yielding lazily bounds what the CALLER holds to one write-chunk. It
+        says nothing about what an implementation holds internally: whether
+        peak resident text is a fraction of the corpus is a property each
+        adapter must establish for itself.
+        """
+        ...
+
+    @property
+    def discovery(self) -> str:
+        """Which path the last :meth:`iter_unembedded` read (``corpus``, ``lake-full``, ...)."""
+        ...
+
+    def commit(self, *, stored_rows: int) -> None:
+        """Record that every row the last :meth:`iter_unembedded` yielded is now stored.
+
+        The use case calls it after a run that consumed the whole discovery
+        and embedded every row of it, with the store's row count afterwards.
+        An adapter with a watermark advances it here (and only when that
+        discovery ran to its end, not to a ``--limit``); one without does
+        nothing.
+        """
+        ...
+
+
+@dataclass(frozen=True, slots=True)
+class LakePosition:
+    """Where one corpus's lake history stands: which history, and how far along it.
+
+    ``lineage`` changes whenever the corpus's rows are loaded from scratch (a
+    ``lake rebuild``, or the first load of a corpus), because snapshot ids
+    from one history mean nothing in the next.
+    """
+
+    lineage: str
+    snapshot_id: int
+
+
+class LakeStepSnapshot(Protocol):
+    """One corpus's steps at one lake snapshot, open until closed."""
+
+    @property
+    def position(self) -> LakePosition:
+        """The snapshot being read."""
+        ...
+
+    def can_read_changes_since(self, since: LakePosition) -> bool:
+        """True when the lake still holds every snapshot after ``since`` in this lineage."""
+        ...
+
+    def step_texts(
+        self, *, min_chars: int, changed_since: LakePosition | None
+    ) -> Iterator[tuple[str, str]]:
+        """``(primary uuid, flattened text)`` for every qualifying step, in corpus order.
+
+        Corpus order is session id, then step id. A qualifying step has a
+        primary uuid, a non-NULL text and at least ``min_chars`` characters
+        of it. With ``changed_since``, only rows whose uuid appears in a step
+        row inserted or deleted after that position are read; those rows
+        still come back in corpus order, every occurrence of each uuid, so
+        the first-wins rule picks the same text a full read would.
+        """
+        ...
+
+    def step_keys(self) -> frozenset[str]:
+        """Every step's primary uuid in the corpus, whatever its text."""
+        ...
+
+    def close(self) -> None:
+        """Release the connection."""
+        ...
+
+    def __enter__(self) -> Self:
+        """Open for a ``with`` block."""
+        ...
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        """Close at the end of the ``with`` block."""
+        ...
+
+
+class LakeStepsPort(Protocol):
+    """Open the lake's view of one corpus's steps."""
+
+    def open_steps(self, corpus_root: Path) -> LakeStepSnapshot | None:
+        """The corpus's steps at the lake's current snapshot, or ``None`` to fall back.
+
+        ``None`` means no lake the caller should read: none exists, its
+        schema is stale, it doesn't hold this corpus, or it holds another
+        directory under the same name. The caller then reads the per-session
+        artifacts, as it did before the lake existed.
+        """
+        ...
+
+
+__all__ = [
+    "EmbeddingProvider",
+    "LakePosition",
+    "LakeStepSnapshot",
+    "LakeStepsPort",
+    "TextRowsPort",
+    "VectorStorePort",
+]

atif_embed/domain/sql_literal.py

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Single-quoted SQL literal escaping for the one predicate that needs it.
+
+LanceDB's delete predicate takes a string, not parameters, so ``lance_store``
+escapes the uuids it deletes with this. The DuckDB reader in
+``corpus_text_rows`` no longer needs it: its trajectory paths reach
+``read_json`` as a bound list parameter. Escaping is a correctness requirement
+of the predicate, and it lives in ``domain`` so a second caller would share
+ONE implementation instead of carrying a copy.
+
+:data:`SqlFragment` marks text that may be spliced into a statement; a value
+from outside the process is a plain ``str`` until it passes through here.
+
+atif-embed may never import atif-duck (the import-linter independence
+contract), so this module is deliberately atif-embed's own rather than a reuse
+of atif-duck's twin.
+
+Pure module: stdlib only, no duckdb, no lancedb.
+"""
+
+from __future__ import annotations
+
+from typing import NewType
+
+#: Text that may be spliced into a SQL statement or predicate.
+SqlFragment = NewType("SqlFragment", str)
+
+
+def sql_literal(value: str) -> SqlFragment:
+    """Escape ``value`` as a single-quoted SQL string literal.
+
+    Doubles every embedded single quote, per the SQL standard's own escape.
+
+    Parameters
+    ----------
+    value
+        Value to embed in a statement or predicate.
+
+    Returns
+    -------
+    SqlFragment
+        ``value`` wrapped in single quotes, embedded quotes doubled.
+    """
+    escaped = value.replace("'", "''")
+    return SqlFragment(f"'{escaped}'")
+
+
+__all__ = ["SqlFragment", "sql_literal"]

atif_embed/domain/text_stamp.py

@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Per-text invariants the store must record: the content stamp and the cap.
+
+Two rules, both about not losing information silently.
+
+**The stamp.** A step's uuid is stable across re-conversions but its flattened
+text is not: a harbor or converter fix changes the text under the same uuid.
+Keying the store on uuid alone therefore makes the first vector permanent —
+the anti-join sees the uuid, calls it embedded, and semantic search keeps
+ranking against pre-fix text with nothing in the store to reveal it. Every
+row also stamps :func:`text_hash` of the exact text it was built from, so a
+mismatch means STALE and the row is replaced.
+
+**The cap.** Texts over :data:`MAX_EMBEDDABLE_CHARS` are sent head-only, so
+their vector cannot match content past the cap. Every row records whether it
+was capped, which turns an otherwise unexplainable search miss into an
+attributable one.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from hashlib import blake2b
+
+#: 16 bytes (128 bit) of blake2b, hex-encoded. Collision risk is negligible
+#: at corpus scale and the digest is a third the width of sha256 in a column
+#: read in full on every discovery pass.
+_DIGEST_BYTES = 16
+
+#: Characters of a text that reach the embedder. Bedrock's body ceiling is
+#: 20 MB and a full batch is 96 texts, so 50K each leaves headroom. Both the
+#: row's ``truncated`` stamp and the adapter's wire-level clip read this one
+#: constant, so the flag can never disagree with what was actually sent.
+MAX_EMBEDDABLE_CHARS = 50_000
+
+
+def text_hash(text: str) -> str:
+    """Return the stable content stamp for one embeddable text."""
+    return blake2b(text.encode("utf-8"), digest_size=_DIGEST_BYTES).hexdigest()
+
+
+def clip_text(text: str) -> tuple[str, bool]:
+    """Return ``(text_to_send, was_truncated)`` for one embeddable text."""
+    if len(text) <= MAX_EMBEDDABLE_CHARS:
+        return text, False
+    return text[:MAX_EMBEDDABLE_CHARS], True
+
+
+@dataclass(frozen=True, slots=True)
+class PendingText:
+    """One text the backfill must embed, with its staleness verdict.
+
+    ``replaces_existing`` is True when the store already holds a row for
+    this uuid under a DIFFERENT hash: that row must be deleted before the
+    new vector is appended, or the uuid would fan out to two vectors and the
+    kNN join would rank against both.
+    """
+
+    uuid: str
+    text: str
+    text_hash: str
+    replaces_existing: bool
+
+    @property
+    def truncated(self) -> bool:
+        """True when this text exceeds the cap and embeds head-only."""
+        return len(self.text) > MAX_EMBEDDABLE_CHARS
+
+
+__all__ = ["MAX_EMBEDDABLE_CHARS", "PendingText", "clip_text", "text_hash"]

atif_embed/infrastructure/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Infrastructure adapters: Bedrock embedder, Lance store, corpus text reader."""