atif-sql 0.4.0__tar.gz → 0.6.0__tar.gz

{atif_sql-0.4.0 → atif_sql-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: atif-sql
-Version: 0.4.0
+Version: 0.6.0
 Summary: ATIF-native analytics over Claude Code agent trajectories: convert, materialize, and query agent sessions with DuckDB.
 Project-URL: Homepage, https://github.com/laithalsaadoon/atif-sql
 Project-URL: Repository, https://github.com/laithalsaadoon/atif-sql
@@ -21,9 +21,8 @@ Requires-Dist: anthropic>=1.11.0
 Requires-Dist: anyio>=4.11.0
 Requires-Dist: boto3>=1.42.91
 Requires-Dist: cyclopts>=5.1.1
-Requires-Dist: duckdb<2,>=1.5.2
+Requires-Dist: duckdb<3,>=2.0.0
 Requires-Dist: imagesize>=2.0
-Requires-Dist: lancedb<0.40,>=0.30
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: polars>=1.40.0
 Requires-Dist: pyarrow>=23.0.1
@@ -83,7 +82,7 @@ uv tool install 'atif-sql[local]' --torch-backend cpu
 ```
 
 Python 3.13 or newer. The install is substantial and deliberately so, because the analytics and
-vector paths carry `polars`, `pyarrow`, `lancedb`, and `duckdb`. Conversion itself needs none of
+vector paths carry `polars`, `pyarrow`, and `duckdb`. Conversion itself needs none of
 that weight: the ATIF models are vendored from Harbor and step pricing reads a vendored copy of
 litellm's price data, so neither `harbor` nor `litellm` is installed. Prebuilt wheels cover CPython
 3.13 on manylinux x86_64 and aarch64, macOS arm64, and Windows x86_64. Alpine and other musl targets
@@ -151,7 +150,7 @@ the only thing documented as installable is the `atif-sql` CLI above.
 | `atif-duck` | DuckDB views + macros over the materialized corpus (core surface plus the v2 analytics surface) |
 | `atif-models` | Model alias registry + structured-output LLM clients for GPT-6 and Claude 5.5 on Bedrock; no other package hardcodes an LLM model id |
 | `atif-analytics` | the LLM pipelines `atif-sql analyze` runs: classify, conflicts, friction, and perceived |
-| `atif-embed` | Cohere Embed v4 on Bedrock or EmbeddingGemma 2 on this machine + LanceDB vector store + embedding backfill |
+| `atif-embed` | Cohere Embed v4 on Bedrock or EmbeddingGemma 2 on this machine + a Lance vector store written through DuckDB's lance extension + embedding backfill |
 | `atif-cli` | the composition root, and the source of the `atif-sql` command (see [What you can do](#what-you-can-do)) |
 
 ## Quick start

{atif_sql-0.4.0 → atif_sql-0.6.0}/README.md

@@ -41,7 +41,7 @@ uv tool install 'atif-sql[local]' --torch-backend cpu
 ```
 
 Python 3.13 or newer. The install is substantial and deliberately so, because the analytics and
-vector paths carry `polars`, `pyarrow`, `lancedb`, and `duckdb`. Conversion itself needs none of
+vector paths carry `polars`, `pyarrow`, and `duckdb`. Conversion itself needs none of
 that weight: the ATIF models are vendored from Harbor and step pricing reads a vendored copy of
 litellm's price data, so neither `harbor` nor `litellm` is installed. Prebuilt wheels cover CPython
 3.13 on manylinux x86_64 and aarch64, macOS arm64, and Windows x86_64. Alpine and other musl targets
@@ -109,7 +109,7 @@ the only thing documented as installable is the `atif-sql` CLI above.
 | `atif-duck` | DuckDB views + macros over the materialized corpus (core surface plus the v2 analytics surface) |
 | `atif-models` | Model alias registry + structured-output LLM clients for GPT-6 and Claude 5.5 on Bedrock; no other package hardcodes an LLM model id |
 | `atif-analytics` | the LLM pipelines `atif-sql analyze` runs: classify, conflicts, friction, and perceived |
-| `atif-embed` | Cohere Embed v4 on Bedrock or EmbeddingGemma 2 on this machine + LanceDB vector store + embedding backfill |
+| `atif-embed` | Cohere Embed v4 on Bedrock or EmbeddingGemma 2 on this machine + a Lance vector store written through DuckDB's lance extension + embedding backfill |
 | `atif-cli` | the composition root, and the source of the `atif-sql` command (see [What you can do](#what-you-can-do)) |
 
 ## Quick start

{atif_sql-0.4.0 → atif_sql-0.6.0}/packages/atif-analytics/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atif-analytics"
-version = "0.4.0"
+version = "0.6.0"
 description = "LLM analytics pipelines over the materialized ATIF corpus: session classification, stance conflicts, user friction, and perceived errors."
 readme = "README.md"
 license = "Apache-2.0"
@@ -20,7 +20,7 @@ dependencies = [
     # a LOCAL source an external installer never sees, so a bare name would ship as
     # `Requires-Dist: atif-models` and resolve to whatever the newest release of
     # that name is. `[tool.commitizen] version_files` rewrites this line on a bump.
-    "atif-models==0.4.0",
+    "atif-models==0.6.0",
     "loguru>=0.7.3",
     "polars>=1.40.0",
     "pyarrow>=23.0.1",
@@ -37,7 +37,7 @@ Issues = "https://github.com/laithalsaadoon/atif-sql/issues"
 Changelog = "https://github.com/laithalsaadoon/atif-sql/blob/main/CHANGELOG.md"
 
 [build-system]
-requires = ["uv_build>=0.11.14,<0.13"]
+requires = ["uv_build>=0.11.14,<0.14"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{atif_sql-0.4.0 → atif_sql-0.6.0}/packages/atif-cli/README.md

@@ -16,13 +16,16 @@ atif-sql materialize [--force] [--quiesce-seconds N]
                      [--sessions id,id]   # contract-compatible extension:
                                           #   plan only the named sessions
 atif-sql status                        # corpus freshness, read-only
+atif-sql status --vector-probe         # time search's exact kNN over the store
+                                       #   (no Bedrock; the nightly compact lane
+                                       #   logs it per corpus)
 atif-sql query 'SQL' [--format auto|table|json|csv]
 atif-sql schema                        # static catalog, no duckdb bind, <50ms
 atif-sql examples                      # tested example queries per view/macro
                                        #   (alias: atif-sql query --examples)
 atif-sql analyze                       # v2 analytics pipelines; dry-run by
                                        #   default, spends only with --no-dry-run
-atif-sql embed --all --no-dry-run      # embed corpus steps into LanceDB
+atif-sql embed --all --no-dry-run      # embed corpus steps into the Lance store
                                        #   (a real run needs --all or --limit N)
 atif-sql search '<text>'               # embed the text, then semantic kNN
 atif-sql cron install | status         # print the refresh crontab block, or

{atif_sql-0.4.0 → atif_sql-0.6.0}/packages/atif-cli/pyproject.toml

@@ -5,7 +5,7 @@
 # import-linter can read what it is allowed to import, and so its tests have a
 # home — none of which needs an index.
 name = "atif-cli"
-version = "0.4.0"
+version = "0.6.0"
 description = "ATIF-native analytics over Claude Code agent trajectories: convert, materialize, and query agent sessions with DuckDB."
 readme = "README.md"
 license = "Apache-2.0"
@@ -29,17 +29,17 @@ dependencies = [
     # namespace to whatever the newest release of that name is, by whoever owns it.
     # One version moves the whole workspace, so `==` is the honest constraint, and
     # `[tool.commitizen] version_files` rewrites every one of these on each bump.
-    "atif-analytics==0.4.0",
-    "atif-converter==0.4.0",
-    "atif-corpus==0.4.0",
-    "atif-duck==0.4.0",
-    "atif-embed==0.4.0",
+    "atif-analytics==0.6.0",
+    "atif-converter==0.6.0",
+    "atif-corpus==0.6.0",
+    "atif-duck==0.6.0",
+    "atif-embed==0.6.0",
     "cyclopts>=5.1.1",
     # Imported directly by duck_errors.py to classify a DuckDB failure into the
     # parse/catalog/runtime exit-code taxonomy. Reaching it transitively through
     # atif-duck would make this package's exit codes depend on a sibling's
     # dependency choice.
-    "duckdb>=1.5.2,<2",
+    "duckdb>=2.0.0,<3",
     "loguru>=0.7.3",
 ]
 
@@ -50,7 +50,7 @@ Issues = "https://github.com/laithalsaadoon/atif-sql/issues"
 Changelog = "https://github.com/laithalsaadoon/atif-sql/blob/main/CHANGELOG.md"
 
 [build-system]
-requires = ["uv_build>=0.11.14,<0.13"]
+requires = ["uv_build>=0.11.14,<0.14"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{atif_sql-0.4.0 → atif_sql-0.6.0}/packages/atif-cli/src/atif_cli/app.py

@@ -16,7 +16,7 @@ The twelve registered commands — nine ``@app.command`` functions plus the
 * ``status``       corpus freshness — read-only, fast
 * ``query``        SQL over the atif-duck views/macros with classified errors
 * ``analyze``      run the analytics pipelines — CALLS BEDROCK, spends money
-* ``embed``        backfill step embeddings into LanceDB — CALLS BEDROCK
+* ``embed``        backfill step embeddings into the Lance store — CALLS BEDROCK
 * ``search``       embed one query string, then kNN over the store — CALLS BEDROCK
 * ``examples``     derived, test-executed example queries (also ``query --examples``)
 * ``schema``       static catalog dump, no DuckDB bind, <50ms
@@ -454,6 +454,15 @@ def _refuse_root(command: str, fmt: OutputFormat) -> None:
 #: PREPARE and EXECUTE (a prepared COPY parses as PREPARE). An allowlist, so a
 #: statement kind a future DuckDB adds is refused until someone reads what it
 #: does.
+#:
+#: PRAGMA and MULTI are out because DuckDB 2.0's parser no longer says what
+#: they hold. 1.5 expanded a PRAGMA while parsing, so ``PRAGMA table_info``
+#: came back as SELECT and ``COPY FROM DATABASE`` as COPY_DATABASE; 2.0 hands
+#: back the unexpanded PRAGMA for every pragma and for ``COPY FROM DATABASE``
+#: and ``IMPORT DATABASE`` alike, and one MULTI for a PIVOT that has to build
+#: its column list first, whatever statement wraps it (``COPY (PIVOT ...) TO``
+#: is one MULTI). Their SELECT forms still run: ``pragma_table_info('t')``,
+#: ``DESCRIBE``, ``SHOW TABLES``, and PIVOT with an explicit ``IN`` list.
 _QUERY_STATEMENT_KINDS: frozenset[str] = frozenset(
     {
         "SELECT",
@@ -469,7 +478,6 @@ _QUERY_STATEMENT_KINDS: frozenset[str] = frozenset(
         "DELETE",
         "MERGE_INTO",
         "CALL",
-        "PRAGMA",
         "TRANSACTION",
         "VACUUM",
         "ANALYZE",
@@ -485,10 +493,17 @@ def _refused_statement_kinds(con: Any, sql: str) -> list[str]:
     connection so a PRAGMA that reads a file while parsing meets the same
     allowlist the query would. A parse error propagates as DuckDB's own
     ``ParserException`` (exit 64), exactly as ``execute`` would have raised it.
+
+    A statement whose type the Python binding has no member for is refused as
+    ``UNKNOWN``: DuckDB 2.0's ``StatementType`` raises ``ValueError`` there
+    (``UPDATE EXTENSIONS`` is type 29), where 1.5 answered ``???``.
     """
     refused: list[str] = []
     for statement in con.extract_statements(sql):
-        kind = str(statement.type.name)
+        try:
+            kind = str(statement.type.name)
+        except ValueError:
+            kind = "UNKNOWN"
         if kind not in _QUERY_STATEMENT_KINDS and kind not in refused:
             refused.append(kind)
     return refused
@@ -557,7 +572,7 @@ def _harden_query_connection(
       ``register`` bound this connection to). A file grant lets
       ``read_parquet`` open that exact path. Nothing else under the corpus
       is named, so ``read_text`` of a ``trajectory.json`` is refused. DuckDB
-      1.5.5 has no read-only grant (measured: a ``read_parquet`` relation
+      has no read-only grant, on 1.5.5 or 2.0.0 (measured: a ``read_parquet`` relation
       bound before ``enable_external_access=false`` is refused at query time
       without a grant, and ``duckdb_settings()`` lists no write switch), so
       each grant is read-write and ``COPY ... TO <granted parquet>
@@ -1136,6 +1151,55 @@ def _lake_surface(corpus_root: Path) -> dict[str, Any]:
     }
 
 
+def _emit_vector_probe(corpus_root: Path, fmt: OutputFormat) -> None:
+    """``status --vector-probe``: time ``search``'s exact kNN over this corpus's store."""
+    import duckdb
+
+    from atif_cli.duck_errors import classify_duckdb_error
+    from atif_cli.vector_probe import PROBE_K, probe_store
+    from atif_embed.domain.errors import DomainError, LanceExtensionMissing
+
+    store = _embed_settings(fmt).resolve_lance_uri(corpus_root)
+    try:
+        probe = probe_store(store, corpus=corpus_root.name)
+    except LanceExtensionMissing as exc:
+        err = ClassifiedError(
+            kind="extension_missing",
+            exit_code=EXIT_CODES["extension_missing"],
+            message=f"the lance DuckDB extension is not installed, so the embeddings "
+            f"store at {store} cannot be read",
+            hint="run: atif-sql embed --install-extension (a one-time download)",
+        )
+        emit_error(err, fmt)
+        raise SystemExit(err.exit_code) from exc
+    except DomainError as exc:
+        # A store the extension can't open is terminal, as it is for embed.
+        kind = "terminal_state" if exc.terminal else "runtime_error"
+        err = ClassifiedError(kind=kind, exit_code=EXIT_CODES[kind], message=str(exc))
+        emit_error(err, fmt)
+        raise SystemExit(err.exit_code) from exc
+    except duckdb.Error as exc:
+        err = classify_duckdb_error(exc)
+        emit_error(err, fmt)
+        raise SystemExit(err.exit_code) from exc
+    if probe is None:
+        err = ClassifiedError(
+            kind="no_embeddings",
+            exit_code=EXIT_CODES["no_embeddings"],
+            message=f"no embeddings to scan at {store}",
+            hint="Run: atif-sql embed --all --no-dry-run (or --limit N)",
+        )
+        emit_error(err, fmt)
+        raise SystemExit(err.exit_code)
+    if resolve_format(fmt) is OutputFormat.TABLE:
+        print(
+            f"vector probe: {probe.vector_rows:,} vectors, exact top-{PROBE_K} scan in "
+            f"{probe.scan_s:.3f}s  ({probe.store})"
+        )
+    else:
+        emit_json(probe.as_dict(), fmt)
+
+
 def _dir_bytes(root: Path) -> int:
     """Total bytes of every file under ``root`` (0 if absent)."""
     if not root.is_dir():
@@ -1150,6 +1214,7 @@ def status(
     source_root: Path | None = None,
     corpus_root: Path | None = None,
     quiesce_seconds: int | None = None,
+    vector_probe: Annotated[bool, cyclopts.Parameter(name="--vector-probe")] = False,
     fmt: Annotated[OutputFormat, cyclopts.Parameter(name="--format")] = OutputFormat.AUTO,
 ) -> None:
     """Report corpus freshness: watermark age, counts, bytes, staleness.
@@ -1189,7 +1254,26 @@ def status(
     not installed; ``query`` binds ``message_embeddings`` empty rather than
     downloading it, so run ``atif-sql embed --install-extension``). Read
     from DuckDB's local extension listing; never installs anything.
+
+    ``--vector-probe`` prints a timing instead of the report. It runs the
+    exact top-10 cosine scan ``search`` runs over the embeddings store, with
+    the stored embedding of the smallest uuid as the query vector, so it
+    calls no embedding model and reaches no network. Only the scan is timed,
+    on the store's own bounded connection (1 GiB, 2 threads). Output:
+    ``{corpus, store, vector_rows, scan_s}``. It reads every stored vector,
+    so it costs what a ``search`` costs, and a plain ``status`` never runs
+    it. The nightly compact lane logs it per corpus to tell when ``search``
+    needs a vector index. Exit codes: 0, 2 no_embeddings (no store, or one
+    with no rows), 78 extension_missing (the lance extension isn't
+    installed; the probe never downloads it) or terminal_state (the store
+    holds a Lance dataset the extension can't open), and a DuckDB error
+    during the scan exits the way ``query``'s do (70 for a runtime error).
     """
+    if vector_probe:
+        settings = _corpus_settings(source_root, corpus_root, agent)
+        _emit_vector_probe(settings.corpus_root, fmt)
+        return
+
     import time
 
     from atif_cli.corpus import storage_layout
@@ -1364,7 +1448,11 @@ def query(
       parquet> (USE_TMP_FILE false)`` used to overwrite one, so every
       statement kind that names a file is refused BEFORE execution, for any
       uid, by DuckDB's own parser: COPY, EXPORT, ATTACH, DETACH, INSTALL,
-      LOAD, and PREPARE/EXECUTE (which could defer one). Exit 70, kind
+      LOAD, and PREPARE/EXECUTE (which could defer one). So are PRAGMA and
+      MULTI, which DuckDB 2.0 no longer expands while parsing (a PRAGMA can be
+      ``COPY FROM DATABASE`` or ``IMPORT DATABASE``, and a MULTI can wrap a
+      COPY): run ``pragma_table_info(...)``, ``DESCRIBE`` or ``SHOW`` instead,
+      and give a PIVOT an explicit ``IN`` list. Exit 70, kind
       ``sandbox_refused``.
     * A 0444 file mode does not bind root, so ``query`` refuses to run as
       uid 0 (exit 77) unless ``ATIF_SQL_ALLOW_ROOT=1`` is set, which logs a
@@ -1405,7 +1493,7 @@ def query(
     * 65  catalog_error unknown view/macro/column (try ``atif-sql schema``)
     * 65  embedding_mismatch the Lance store was written by another provider
     * 70  sandbox_refused a statement kind the sandbox never runs (COPY,
-      EXPORT, ATTACH, INSTALL, LOAD, PREPARE, ...)
+      EXPORT, ATTACH, INSTALL, LOAD, PREPARE, PRAGMA, ...)
     * 70  runtime_error everything else (an unmaterialized corpus, or SQL
       the sandbox refused)
     * 77  root_refused  running as uid 0 without ``ATIF_SQL_ALLOW_ROOT=1``
@@ -1507,7 +1595,10 @@ def query(
                         message=f"the query sandbox does not run {', '.join(refused)} statements",
                         hint="query runs SELECT and in-memory DDL/DML only; COPY, EXPORT, "
                         "ATTACH, DETACH, INSTALL, LOAD, PREPARE and EXECUTE are refused "
-                        "because they reach files or defer a statement past this check",
+                        "because they reach files or defer a statement past this check, and "
+                        "PRAGMA and MULTI because the parser no longer says what they hold "
+                        "(use pragma_table_info(...), DESCRIBE or SHOW for a PRAGMA, and an "
+                        "explicit IN list for a PIVOT)",
                     )
                     emit_error(err, fmt)
                     raise SystemExit(err.exit_code)
@@ -1783,8 +1874,9 @@ def _install_lance_extension(fmt: OutputFormat, *, quiet: bool = False) -> None:
                 from loguru import logger
 
                 logger.warning(
-                    "Could not install the lance DuckDB extension ({}); vector search stays "
-                    "unavailable until `atif-sql embed --install-extension` succeeds",
+                    "Could not install the lance DuckDB extension ({}); the embeddings store "
+                    "can't be written or searched until `atif-sql embed --install-extension` "
+                    "succeeds",
                     str(exc).splitlines()[0],
                 )
                 return
@@ -1809,7 +1901,7 @@ def embed(
     corpus_root: Path | None = None,
     fmt: Annotated[OutputFormat, cyclopts.Parameter(name="--format")] = OutputFormat.AUTO,
 ) -> None:
-    """Embed unembedded corpus steps and append them to LanceDB.
+    """Embed unembedded corpus steps and append them to the Lance store.
 
     Provider
     --------
@@ -1847,14 +1939,17 @@ def embed(
 
     Extension
     ---------
-    ``query`` and ``search`` read the store through DuckDB's lance extension
-    and never install it themselves (installing is a 242 MB download from
-    the extension repository, and ``query`` runs unattended). It is
-    installed here, where the network is already a deliberate act: every
-    REAL run installs it first, and ``--install-extension`` installs it and
-    exits without touching Bedrock or the store (``{"extension": "lance",
-    "installed": true, "install_path": ...}``). ``atif-sql status`` reports
-    whether it is present.
+    The store is written and read only through DuckDB's lance extension.
+    ``query`` and ``search`` never install it themselves (installing is a
+    242 MB download from the extension repository, and ``query`` runs
+    unattended). It is installed here, where the network is already a
+    deliberate act: every REAL run installs it first, and
+    ``--install-extension`` installs it and exits without touching Bedrock
+    or the store (``{"extension": "lance", "installed": true,
+    "install_path": ...}``). A dry run never installs it, so a dry run over
+    an existing store without the extension exits 78 and names
+    ``--install-extension``. ``atif-sql status`` reports whether it is
+    present.
 
     Cost
     ----
@@ -1969,6 +2064,7 @@ def _prune_orphans(corpus_root: Path | None, *, dry_run: bool, fmt: OutputFormat
     from atif_corpus.application.materialize import read_sink_pending
     from atif_corpus.domain.layout import CorpusLayout
     from atif_embed.application.prune import prune_orphans
+    from atif_embed.domain.errors import DomainError
     from atif_embed.infrastructure.lance_store import LanceVectorStore
 
     settings = _corpus_settings(None, corpus_root)
@@ -1987,12 +2083,21 @@ def _prune_orphans(corpus_root: Path | None, *, dry_run: bool, fmt: OutputFormat
             fmt,
         )
         raise SystemExit(EXIT_CODES["lake_unavailable"])
-    report = prune_orphans(
-        settings.corpus_root,
-        lake=lake_steps_port(),
-        store=LanceVectorStore(lance_uri, dim=embed_settings.embedding_dim),
-        dry_run=dry_run,
-    )
+    try:
+        report = prune_orphans(
+            settings.corpus_root,
+            lake=lake_steps_port(),
+            store=LanceVectorStore(
+                lance_uri, dim=embed_settings.embedding_dim, install_extension=not dry_run
+            ),
+            dry_run=dry_run,
+        )
+    except DomainError as exc:
+        # The store's own errors (a missing extension on a dry prune, a store
+        # the extension can't open) get the same envelope as embed's.
+        kind = "terminal_state" if exc.terminal else "runtime_error"
+        emit_error(ClassifiedError(kind=kind, exit_code=EXIT_CODES[kind], message=str(exc)), fmt)
+        raise SystemExit(EXIT_CODES[kind]) from exc
     if report is None:
         emit_error(
             ClassifiedError(
@@ -2072,7 +2177,8 @@ def search(
     extension_missing (a store exists but the lance DuckDB extension is not
     installed; run ``atif-sql embed --install-extension``), lake_unavailable
     (``--all-corpora`` without a usable lake) or terminal_state (the gemma
-    provider is selected and the ``local`` extra is not installed).
+    provider is selected and the ``local`` extra is not installed, or the store
+    holds a Lance dataset the extension can't open).
     """
     _refuse_root("search", fmt)
 
@@ -2083,7 +2189,11 @@ def search(
         classify_duckdb_error,
         classify_registration_error,
     )
-    from atif_duck.infrastructure.registry import lance_extension_installed, register
+    from atif_duck.infrastructure.registry import (
+        lance_extension_installed,
+        lance_store_unreadable,
+        register,
+    )
     from atif_embed.application.embed import embed_query
     from atif_embed.domain.errors import DomainError
 
@@ -2144,6 +2254,23 @@ def search(
 
         row = con.execute("SELECT count(*) FROM message_embeddings").fetchone()
         if not row or int(row[0]) == 0:
+            # No rows can also mean a store the extension can't open, which
+            # binds as an empty one; embedding into it can't fix that.
+            unreadable = [uri for uri in stores or [lance_uri] if lance_store_unreadable(con, uri)]
+            if unreadable:
+                emit_error(
+                    ClassifiedError(
+                        kind="terminal_state",
+                        exit_code=EXIT_CODES["terminal_state"],
+                        message=f"the embeddings store at {unreadable[0]} holds a Lance dataset "
+                        "the lance extension can't open; its newest manifest may be damaged "
+                        "or partly written",
+                        hint="restore the store from a backup, or move it aside and re-embed "
+                        "with `atif-sql embed --all --no-dry-run`",
+                    ),
+                    fmt,
+                )
+                raise SystemExit(EXIT_CODES["terminal_state"])
             emit_error(
                 ClassifiedError(
                     kind="no_embeddings",
@@ -2177,12 +2304,14 @@ def search(
 
 
 #: ``search``'s kNN, one text per shape. Rank by cosine similarity
-#: descending: ORDER BY array_cosine_distance (== 1 - sim) ASC is what
-#: triggers the cosine HNSW index lookup. Using array_distance here (L2) would
-#: silently bypass the index AND give wrong ranks: the raw int8-cast-to-float
-#: document vectors have magnitudes in the thousands while the query vector is
-#: unit-normalized — only cosine is magnitude-invariant. ``{dim}`` is the only
-#: placeholder, filled with ``int(len(vector))``.
+#: descending: ORDER BY array_cosine_distance (== 1 - sim) ASC with a LIMIT,
+#: which DuckDB runs as an exact top-k over a full scan of the Lance table
+#: (EXPLAIN shows a ``Lance Table Scan`` with ``Lance Limit: none`` under the
+#: join and a ``Top N`` above it; no vector index is read).
+#: Cosine is the only correct metric here, not a speed choice: the raw
+#: int8-cast-to-float document vectors have magnitudes in the thousands while
+#: the query vector is unit-normalized, so array_distance (L2) would give wrong
+#: ranks. ``{dim}`` is the only placeholder, filled with ``int(len(vector))``.
 _SEARCH_SQL = """
     WITH qv AS (SELECT CAST(? AS FLOAT[{dim}]) AS v)
     SELECT me.uuid                                                     AS uuid,

atif_sql-0.6.0/packages/atif-cli/src/atif_cli/vector_probe.py

@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Time the exact kNN scan ``search`` runs, with no Bedrock call and no network.
+
+``atif-sql status --vector-probe`` runs this, and the nightly compact lane in
+``scripts/atif-sql-refresh.sh`` logs the result for each corpus. ``search``
+reads no vector index: it ranks every stored vector, so what it pays grows
+with the store, and this number is how the lane notices when it's grown too
+far.
+
+The probe runs :data:`PROBE_SQL`, which ranks the store's rows the way
+``search`` ranks them (the same ``qv`` CTE, cosine similarity, and ``ORDER BY
+array_cosine_distance ... ASC`` with a ``LIMIT``), minus ``search``'s join back
+to ``steps``. Its query vector is the stored embedding of the smallest uuid,
+read in the same single statement that picks it, so a store gives the same
+statement every night and nothing is embedded. Only the scan is timed.
+Attaching the store, counting its rows and reading the probe vector all
+happen before the clock starts. Reading the probe vector reads every vector
+once, so the timed scan always runs on a warm page cache.
+
+The connection is the store's own, from
+:func:`atif_embed.infrastructure.lance_store.connect_db`: 1 GiB of DuckDB
+memory, 2 threads, a private spill directory, no extension autoload, and no
+download of the lance extension. The compact lane runs inside a 4G memory
+scope, and an unbounded connection would size itself from the whole host.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+#: How many neighbors the probe asks for: ``search``'s default ``-k``.
+PROBE_K = 10
+
+#: The probe's kNN. Every line from the ``qv`` CTE to ``LIMIT ?`` except the
+#: ``FROM`` matches a line of ``atif_cli.app._SEARCH_SQL``, and
+#: ``tests/test_vector_probe.py`` checks that, so a change to how ``search``
+#: ranks has to change this too. ``{dim}`` is filled with ``int()`` and
+#: ``{table}`` with the store's table name, never with caller text.
+PROBE_SQL = """
+    WITH qv AS (SELECT CAST(? AS FLOAT[{dim}]) AS v)
+    SELECT me.uuid                                                     AS uuid,
+           array_cosine_similarity(me.embedding, (SELECT v FROM qv))   AS sim
+    FROM {table} me
+    ORDER BY array_cosine_distance(me.embedding, (SELECT v FROM qv)) ASC
+    LIMIT ?
+"""
+
+
+@dataclass(frozen=True)
+class VectorProbe:
+    """One timed scan of one store.
+
+    ``hits`` holds the ``(uuid, sim)`` rows the scan returned, best first.
+    It's for tests and isn't part of :meth:`as_dict`: the lane logs the
+    timing, not the neighbors.
+    """
+
+    corpus: str
+    store: Path
+    vector_rows: int
+    scan_s: float
+    hits: tuple[tuple[str, float], ...]
+
+    def as_dict(self) -> dict[str, Any]:
+        """The probe's JSON payload: ``{corpus, store, vector_rows, scan_s}``."""
+        return {
+            "corpus": self.corpus,
+            "store": str(self.store),
+            "vector_rows": self.vector_rows,
+            "scan_s": self.scan_s,
+        }
+
+
+def probe_sql(*, dim: int, table: str) -> str:
+    """:data:`PROBE_SQL` for a ``dim``-wide store whose table is ``table``."""
+    return PROBE_SQL.format(dim=int(dim), table=table)
+
+
+def probe_store(store: Path, *, corpus: str, k: int = PROBE_K) -> VectorProbe | None:
+    """Time one exact top-``k`` scan of the store at ``store``.
+
+    Returns ``None`` when there's nothing to scan: no store directory, no
+    embeddings table in it, or no rows. The directory is checked before
+    connecting, because opening a store creates its directory.
+
+    Raises
+    ------
+    LanceExtensionMissing
+        The lance extension isn't installed. The probe never downloads it.
+    duckdb.Error
+        The store couldn't be read.
+    """
+    from atif_embed.infrastructure.lance_store import (
+        STORE_ALIAS,
+        TABLE_NAME,
+        connect_db,
+        has_table,
+    )
+
+    if not store.is_dir():
+        return None
+    table = f"{STORE_ALIAS}.main.{TABLE_NAME}"
+    with connect_db(store) as conn:
+        if not has_table(conn):
+            return None
+        con = conn.con
+        vector_rows = int(con.execute(f"SELECT count(*) FROM {table}").fetchall()[0][0])  # noqa: S608  # nosec B608 - table is the store's constant name
+        if vector_rows == 0:
+            return None
+        # One statement, so a delete landing between "which uuid" and "its
+        # vector" can't leave nothing to read. It's one scan that reads every
+        # vector, which also leaves them in the page cache: the timed scan
+        # below runs warm every night, so nights compare like with like.
+        first = con.execute(
+            f"SELECT uuid, embedding FROM {table} ORDER BY uuid LIMIT 1"  # noqa: S608  # nosec B608 - table is the store's constant name
+        ).fetchall()
+        if not first:
+            return None
+        vector = first[0][1]
+        sql = probe_sql(dim=len(vector), table=table)
+        started = time.perf_counter()
+        rows = con.execute(sql, [list(vector), int(k)]).fetchall()
+        scan_s = time.perf_counter() - started
+    return VectorProbe(
+        corpus=corpus,
+        store=store,
+        vector_rows=vector_rows,
+        scan_s=round(scan_s, 3),
+        hits=tuple((str(uuid), float(sim)) for uuid, sim in rows),
+    )
+
+
+__all__ = ["PROBE_K", "PROBE_SQL", "VectorProbe", "probe_sql", "probe_store"]

{atif_sql-0.4.0 → atif_sql-0.6.0}/packages/atif-converter/README.md

@@ -29,7 +29,7 @@ value: `atif_converter.domain.pricing` returns the same floats, bit for bit,
 without litellm being installed. It reads `domain/model_prices.json`, a
 filtered copy of litellm's public `model_prices_and_context_window.json` (MIT;
 the notice and source ref are in its `meta` block), and repeats litellm
-1.104.2's arithmetic in the same float operation order. The table holds the
+1.106.3's arithmetic in the same float operation order. The table holds the
 Claude and OpenAI text models our transcripts name under the `anthropic`,
 `openai`, `bedrock` and `bedrock_converse` providers, and keeps litellm's
 `fallback_generalizations` rules, which route an unmapped `claude-<family>-<n>`

{atif_sql-0.4.0 → atif_sql-0.6.0}/packages/atif-converter/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atif-converter"
-version = "0.4.0"
+version = "0.6.0"
 description = "Claude Code and Codex transcripts -> ATIF conversion, with the fidelity policy per agent."
 readme = "README.md"
 license = "Apache-2.0"
@@ -37,7 +37,7 @@ Issues = "https://github.com/laithalsaadoon/atif-sql/issues"
 Changelog = "https://github.com/laithalsaadoon/atif-sql/blob/main/CHANGELOG.md"
 
 [build-system]
-requires = ["uv_build>=0.11.14,<0.13"]
+requires = ["uv_build>=0.11.14,<0.14"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]