atif-sql 0.2.0__py3-none-any.whl

atif_models/infrastructure/openai_bedrock.py

@@ -0,0 +1,314 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""OpenAI GPT-5.6 structured-output adapter on bedrock-runtime (the default).
+
+Implements :class:`atif_models.domain.ports.LlmStructuredProvider` with the
+OpenAI chat-completions body shape on ``invoke_model``:
+
+.. code-block:: json
+
+    {"messages": [{"role": "system", ...}, {"role": "user", ...}],
+     "max_completion_tokens": 32000,
+     "reasoning_effort": "high",
+     "response_format": {"type": "json_schema",
+                         "json_schema": {"name": ..., "strict": true, "schema": ...}}}
+
+Retry policy: tenacity owns the semantic loop (10 attempts, exponential
+2..60s) over :data:`RETRY_CODES` + network errors; botocore's own retries
+are OFF (``max_attempts=0``) so tenacity sees every error immediately. The
+blocking ``invoke_model`` call is dispatched via ``anyio.to_thread`` under
+an ``anyio.CapacityLimiter`` so cancellation scopes propagate.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+from typing import Any
+
+import anyio
+import anyio.to_thread
+import boto3
+from botocore.config import Config as BotoConfig
+from botocore.exceptions import (
+    ClientError,
+    ConnectionError as BotoConnectionError,
+    EndpointConnectionError,
+    ReadTimeoutError,
+    SSLError,
+)
+from loguru import logger
+from pydantic import ValidationError
+from tenacity import (
+    retry,
+    retry_if_exception,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+from atif_models.domain.ports import (
+    CallUsage,
+    ProviderUnavailable,
+    RefusalError,
+    SchemaT,
+    UsageAccumulator,
+)
+from atif_models.domain.registry import ModelSpec
+from atif_models.domain.schema import to_openai_strict
+
+#: Bedrock error codes worth retrying. TWIN of
+#: ``atif_embed.infrastructure.cohere_bedrock.RETRY_CODES`` — the two
+#: packages may not import each other (independence contract), so each pins
+#: this set against the same literal in its own test suite.
+RETRY_CODES: frozenset[str] = frozenset(
+    {
+        "ThrottlingException",
+        "ServiceUnavailableException",
+        "ModelTimeoutException",
+        "ModelErrorException",
+        "ProvisionedThroughputExceededException",
+        "TooManyRequestsException",
+        "InternalServerException",
+        "InternalFailure",
+    }
+)
+
+#: finish_reason values that mean "the model declined" (terminal).
+_REFUSAL_FINISH_REASONS = {"content_filter", "refusal"}
+
+#: One-step reasoning_effort degradation ladder for the finish_reason=length
+#: retry. Truncation is DETERMINISTIC for a fixed (prompt, parameters) pair —
+#: resending the same request would truncate identically and just double the
+#: bill — so the single retry changes a parameter: less reasoning leaves more
+#: of the completion budget for the answer. Efforts at "low" or below don't
+#: retry (nothing left to degrade to).
+_EFFORT_DEGRADE: dict[str, str] = {
+    "max": "xhigh",
+    "xhigh": "high",
+    "high": "medium",
+    "medium": "low",
+}
+
+
+class _LengthTruncation(ProviderUnavailable):
+    """Internal marker for ``finish_reason=length``.
+
+    Callers only ever see :class:`ProviderUnavailable`;
+    :meth:`OpenAiBedrockProvider.classify_structured` catches this one to run
+    the one-shot effort-degrade retry.
+    """
+
+
+def _is_retryable(exc: BaseException) -> bool:
+    """True for throttle/service ``ClientError`` codes + network errors."""
+    if isinstance(exc, SSLError | BotoConnectionError | EndpointConnectionError | ReadTimeoutError):
+        return True
+    if not isinstance(exc, ClientError):
+        return False
+    code = exc.response.get("Error", {}).get("Code")
+    return code in RETRY_CODES
+
+
+def _log_before_sleep(retry_state: Any) -> None:
+    """Log one WARNING per backoff (loguru — stdlib logging is banned)."""
+    exc = retry_state.outcome.exception() if retry_state.outcome else None
+    logger.warning(
+        "bedrock invoke retry {}/10 in {:.1f}s after {}: {}",
+        retry_state.attempt_number,
+        getattr(retry_state.next_action, "sleep", 0.0),
+        type(exc).__name__ if exc else "?",
+        exc,
+    )
+
+
+def extract_usage(payload: dict[str, Any]) -> CallUsage:
+    """Pull :class:`CallUsage` out of one chat-completions response.
+
+    GPT-5.6 on Bedrock reports ``usage.prompt_tokens`` /
+    ``usage.completion_tokens`` plus ``completion_tokens_details.reasoning_tokens``
+    and ``prompt_tokens_details.cached_tokens``. Missing fields default to 0.
+    """
+    usage = payload.get("usage") or {}
+    completion_details = usage.get("completion_tokens_details") or {}
+    prompt_details = usage.get("prompt_tokens_details") or {}
+    return CallUsage(
+        input_tokens=int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0),
+        output_tokens=int(usage.get("completion_tokens") or usage.get("output_tokens") or 0),
+        reasoning_tokens=int(completion_details.get("reasoning_tokens") or 0),
+        cached_tokens=int(prompt_details.get("cached_tokens") or 0),
+    )
+
+
+class OpenAiBedrockProvider:
+    """The default :class:`~atif_models.domain.ports.LlmStructuredProvider`.
+
+    Everything provider-specific is fixed at construction: the resolved
+    :class:`~atif_models.domain.registry.ModelSpec`, the AWS region, the
+    concurrency ceiling, and (optionally) a shared per-pipeline
+    :class:`~atif_models.domain.ports.UsageAccumulator`.
+    """
+
+    def __init__(
+        self,
+        spec: ModelSpec,
+        *,
+        region: str,
+        concurrency: int = 16,
+        usage: UsageAccumulator | None = None,
+    ) -> None:
+        self._spec = spec
+        self._region = region
+        self._limiter = anyio.CapacityLimiter(concurrency)
+        self.usage = usage if usage is not None else UsageAccumulator()
+        self._client_lock = threading.Lock()
+        self._client: Any = None
+        self._pool_size = max(32, concurrency * 2)
+
+    @property
+    def provider(self) -> str:
+        """Stable provider tag for logs and store stamps."""
+        return "openai-bedrock"
+
+    @property
+    def spec(self) -> ModelSpec:
+        """The resolved model spec this adapter is bound to."""
+        return self._spec
+
+    def _get_client(self) -> Any:
+        """Lazily build one shared thread-safe ``bedrock-runtime`` client.
+
+        botocore retries are OFF (``max_attempts=0``) — the tenacity
+        decorator on :meth:`_invoke_sync` owns the retry policy. Pool
+        sized to 2× concurrency with a floor of 32; ``read_timeout=600``
+        because high reasoning effort can hold the connection well past
+        botocore's 60s default.
+        """
+        with self._client_lock:
+            if self._client is None:
+                boto_cfg = BotoConfig(
+                    region_name=self._region,
+                    retries={"max_attempts": 0, "mode": "standard"},
+                    max_pool_connections=self._pool_size,
+                    connect_timeout=10,
+                    read_timeout=600,
+                )
+                self._client = boto3.client("bedrock-runtime", config=boto_cfg)
+            return self._client
+
+    def build_body(self, *, system: str, prompt: str, schema: type[SchemaT]) -> dict[str, Any]:
+        """Return the exact ``invoke_model`` body for one call (pure; unit-testable)."""
+        return {
+            "messages": [
+                {"role": "system", "content": system},
+                {"role": "user", "content": prompt},
+            ],
+            "max_completion_tokens": self._spec.max_completion_tokens,
+            "reasoning_effort": self._spec.reasoning_effort,
+            "response_format": {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": schema.__name__,
+                    "strict": True,
+                    "schema": to_openai_strict(schema),
+                },
+            },
+        }
+
+    async def classify_structured(
+        self, *, system: str, prompt: str, schema: type[SchemaT]
+    ) -> SchemaT:
+        """One structured-output call under the concurrency limiter.
+
+        ``finish_reason=length`` retries ONCE with ``reasoning_effort``
+        degraded one step (high→medium→low): truncation is deterministic
+        for a fixed (prompt, parameters) pair, so the second chance must
+        change a parameter, not resend. Still-truncated (or already at the
+        floor) raises :class:`ProviderUnavailable` for the retry queue.
+        """
+        body = self.build_body(system=system, prompt=prompt, schema=schema)
+        async with self._limiter:
+            payload = await anyio.to_thread.run_sync(lambda: self._invoke_sync(body))
+            try:
+                return self._parse_payload(payload, schema)
+            except _LengthTruncation as first:
+                degraded = _EFFORT_DEGRADE.get(str(body.get("reasoning_effort")))
+                if degraded is None:
+                    raise ProviderUnavailable(str(first)) from first
+                logger.warning(
+                    "bedrock invoke: finish_reason=length at effort={} — "
+                    "retrying once at effort={} (deterministic truncation "
+                    "needs a parameter change, not a resend)",
+                    body.get("reasoning_effort"),
+                    degraded,
+                )
+                retry_body = {**body, "reasoning_effort": degraded}
+                payload = await anyio.to_thread.run_sync(lambda: self._invoke_sync(retry_body))
+                try:
+                    return self._parse_payload(payload, schema)
+                except _LengthTruncation as second:
+                    raise ProviderUnavailable(str(second)) from second
+
+    @retry(
+        stop=stop_after_attempt(10),
+        wait=wait_exponential(multiplier=2, min=2, max=60),
+        retry=retry_if_exception(_is_retryable),
+        before_sleep=_log_before_sleep,
+        reraise=True,
+    )
+    def _invoke_sync(self, body: dict[str, Any]) -> dict[str, Any]:
+        """One blocking ``invoke_model`` call; returns the parsed JSON payload."""
+        client = self._get_client()
+        resp = client.invoke_model(
+            modelId=self._spec.model_id,
+            body=json.dumps(body),
+            contentType="application/json",
+            accept="application/json",
+        )
+        return json.loads(resp["body"].read())
+
+    def _parse_payload(self, payload: dict[str, Any], schema: type[SchemaT]) -> SchemaT:
+        """choices[0].message.content → json → ``schema.model_validate``.
+
+        Usage is accumulated FIRST (even a length-truncated call billed
+        tokens), then finish_reason gates: ``length`` →
+        :class:`ProviderUnavailable` with a hint, ``content_filter`` /
+        ``refusal`` → :class:`RefusalError`.
+        """
+        self.usage.add(extract_usage(payload))
+        choices = payload.get("choices") or []
+        if not choices:
+            msg = f"no choices in response (keys: {sorted(payload.keys())})"
+            raise ProviderUnavailable(msg)
+        choice = choices[0]
+        finish_reason = choice.get("finish_reason")
+        if finish_reason in _REFUSAL_FINISH_REASONS:
+            msg = f"model declined the input (finish_reason={finish_reason})"
+            raise RefusalError(msg)
+        message = choice.get("message") or {}
+        if message.get("refusal"):
+            msg = f"model refused: {message['refusal']}"
+            raise RefusalError(msg)
+        if finish_reason == "length":
+            msg = (
+                "response truncated (finish_reason=length) — raise "
+                f"max_completion_tokens (currently {self._spec.max_completion_tokens}), "
+                "lower reasoning_effort, or shrink the prompt"
+            )
+            raise _LengthTruncation(msg)
+        content = message.get("content")
+        if not isinstance(content, str) or not content:
+            msg = f"no message content (finish_reason={finish_reason!r})"
+            raise ProviderUnavailable(msg)
+        try:
+            data = json.loads(content)
+        except json.JSONDecodeError as exc:
+            msg = f"content is not valid JSON: {exc}"
+            raise ProviderUnavailable(msg) from exc
+        try:
+            return schema.model_validate(data)
+        except ValidationError as exc:
+            msg = f"strict output failed pydantic validation: {exc}"
+            raise ProviderUnavailable(msg) from exc
+
+
+__all__ = ["RETRY_CODES", "OpenAiBedrockProvider", "extract_usage"]

atif_models/infrastructure/settings.py

@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Env-driven LLM settings for atif-models (CONTRACT-V2 §Model registry).
+
+Pydantic v2 ``BaseSettings`` under the workspace-wide ``ATIF_SQL_`` prefix.
+Carries the family/region/concurrency knobs plus the per-pipeline size
+overrides — ``ATIF_SQL_LLM_SIZE_CLASSIFY`` and friends. Reading process env
+is I/O, so this lives in ``infrastructure``.
+"""
+
+from __future__ import annotations
+
+from typing import cast
+
+from pydantic import field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from atif_models.domain.registry import DEFAULT_FAMILY, Family, ModelSpec, Size, resolve
+
+#: Families with a wired provider adapter. The registry knows every family's
+#: model ids, but only these can actually be called: an unwired family
+#: resolves a model id and then sends it the OpenAI chat-completions body
+#: (``response_format`` / ``reasoning_effort``), which is a Bedrock 400 on
+#: every call. Rejecting at settings load turns that into a startup refusal.
+RUNNABLE_FAMILIES: frozenset[Family] = frozenset({"openai"})
+
+
+class LlmSettings(BaseSettings):
+    """Env-driven model selection for the LLM analytics pipelines."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="ATIF_SQL_",
+        env_file=".env",
+        extra="ignore",
+    )
+
+    llm_family: Family = DEFAULT_FAMILY
+    llm_region: str = "us-east-1"
+    llm_concurrency: int = 16
+
+    @field_validator("llm_family")
+    @classmethod
+    def _family_must_be_runnable(cls, value: Family) -> Family:
+        """Refuse a family that has no provider adapter behind it."""
+        if value in RUNNABLE_FAMILIES:
+            return value
+        runnable = ", ".join(sorted(RUNNABLE_FAMILIES))
+        msg = (
+            f"llm_family={value!r} has no provider adapter, so every pipeline call "
+            f"would send an OpenAI chat-completions body to a {value} model id and "
+            f"get a Bedrock 400. Runnable families: {runnable}. "
+            f"Unset ATIF_SQL_LLM_FAMILY or set it to one of those."
+        )
+        raise ValueError(msg)
+
+    # Per-pipeline size assignments (CONTRACT-V2 §Pipeline size assignments):
+    # conflicts is the hardest judgment task (sol), friction is a
+    # per-message enum (luna). classify moved to luna on 2026-09-27 when its
+    # schema shrank to work category + goal; the trajectory pipeline (and its
+    # size) left the same day.
+    llm_size_classify: Size = "small"
+    llm_size_conflicts: Size = "large"
+    llm_size_friction: Size = "small"
+    # perceived judges user-visible evidence over full transcripts — medium
+    # (terra). Env: ATIF_SQL_LLM_SIZE_PERCEIVED.
+    llm_size_perceived: Size = "medium"
+
+    def size_for(self, pipeline: str) -> Size:
+        """The configured size alias for ``pipeline``.
+
+        Pipelines: classify / conflicts / friction / perceived.
+        """
+        try:
+            size = getattr(self, f"llm_size_{pipeline}")
+        except AttributeError:
+            msg = f"unknown pipeline {pipeline!r} — no llm_size_{pipeline} setting"
+            raise KeyError(msg) from None
+        return cast("Size", size)
+
+    def spec_for(self, pipeline: str) -> ModelSpec:
+        """Resolve ``pipeline`` to its :class:`~atif_models.domain.registry.ModelSpec`."""
+        return resolve(self.size_for(pipeline), self.llm_family)
+
+
+__all__ = ["LlmSettings"]

atif_sql-0.2.0.dist-info/METADATA

@@ -0,0 +1,272 @@
+Metadata-Version: 2.5
+Name: atif-sql
+Version: 0.2.0
+Summary: ATIF-native analytics over Claude Code agent trajectories: convert, materialize, and query agent sessions with DuckDB.
+Project-URL: Homepage, https://github.com/laithalsaadoon/atif-sql
+Project-URL: Repository, https://github.com/laithalsaadoon/atif-sql
+Project-URL: Issues, https://github.com/laithalsaadoon/atif-sql/issues
+Project-URL: Changelog, https://github.com/laithalsaadoon/atif-sql/blob/main/CHANGELOG.md
+Author: Laith Al-Saadoon
+License-Expression: Apache-2.0
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development
+Classifier: Typing :: Typed
+Requires-Python: >=3.13
+Requires-Dist: anyio>=4.11.0
+Requires-Dist: boto3>=1.42.91
+Requires-Dist: cyclopts>=5.1.1
+Requires-Dist: duckdb<2,>=1.5.2
+Requires-Dist: imagesize>=2.0
+Requires-Dist: lancedb<0.40,>=0.30
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: polars>=1.40.0
+Requires-Dist: pyarrow>=23.0.1
+Requires-Dist: pydantic-settings>=2.13.1
+Requires-Dist: pydantic>=2.13.2
+Requires-Dist: pytz>=2024.1
+Requires-Dist: tenacity>=9.1.4
+Requires-Dist: zstandard>=0.25.0
+Provides-Extra: local
+Requires-Dist: pillow>=12.3.0; extra == 'local'
+Requires-Dist: sentence-transformers>=6.1.0; extra == 'local'
+Requires-Dist: torch>=2.14; extra == 'local'
+Requires-Dist: torchvision>=0.29; extra == 'local'
+Requires-Dist: transformers>=5.19.0; extra == 'local'
+Description-Content-Type: text/markdown
+
+# atif-sql
+
+[![CI](https://github.com/laithalsaadoon/atif-sql/actions/workflows/check.yml/badge.svg?branch=main)](https://github.com/laithalsaadoon/atif-sql/actions/workflows/check.yml)
+[![security](https://github.com/laithalsaadoon/atif-sql/actions/workflows/security.yml/badge.svg?branch=main)](https://github.com/laithalsaadoon/atif-sql/actions/workflows/security.yml)
+[![CodeQL](https://github.com/laithalsaadoon/atif-sql/actions/workflows/codeql.yml/badge.svg?branch=main)](https://github.com/laithalsaadoon/atif-sql/actions/workflows/codeql.yml)
+[![Scorecard](https://github.com/laithalsaadoon/atif-sql/actions/workflows/scorecard.yml/badge.svg?branch=main)](https://github.com/laithalsaadoon/atif-sql/actions/workflows/scorecard.yml)
+[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/laithalsaadoon/atif-sql/badge)](https://scorecard.dev/viewer/?uri=github.com/laithalsaadoon/atif-sql)
+![Python 3.13+](https://img.shields.io/badge/python-3.13%2B-blue.svg)
+[![License Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
+
+ATIF-native analytics over agent trajectories.
+
+Claude Code sessions (`~/.claude/projects/**/*.jsonl`) and Codex CLI rollouts
+(`~/.codex/sessions/**/rollout-*.jsonl`) are converted to ATIF (Harbor's Agent
+Trajectory Interchange Format, see the Harbor ATIF RFC: RFC-0001 in
+<https://github.com/laude-institute/harbor>), materialized as a corpus, and
+queried through DuckDB views. Converting once at the boundary (with an
+explicit, tested fidelity policy for what upstream drops) beats re-deriving
+trajectory semantics inside every SQL view. Both agents land in the same views,
+and `sessions.agent` says which one a row came from.
+
+## Install
+
+One command, one package, every capability. Conversion, materialization, the DuckDB surface,
+the LLM analytics pipelines, and semantic search are all in the box, and nothing has to be
+installed afterwards to make a command work.
+
+```bash
+uv tool install atif-sql     # the CLI on PATH
+uvx atif-sql schema          # or run it without installing
+```
+
+To keep semantic search on your own machine, add the one optional extra, `local`. It runs
+Google's EmbeddingGemma 2 for `embed` and `search` in place of Cohere on Bedrock (see
+[Search without sending text off the machine](#search-without-sending-text-off-the-machine)).
+On Linux, `--torch-backend cpu` installs PyTorch's CPU build rather than PyPI's CUDA build and
+its gigabytes of NVIDIA libraries:
+
+```bash
+uv tool install 'atif-sql[local]' --torch-backend cpu
+```
+
+Python 3.13 or newer. The install is substantial and deliberately so, because the analytics and
+vector paths carry `polars`, `pyarrow`, `lancedb`, and `duckdb`. Conversion itself needs none of
+that weight: the ATIF models are vendored from Harbor and step pricing reads a vendored copy of
+litellm's price data, so neither `harbor` nor `litellm` is installed. Prebuilt wheels cover CPython
+3.13 on manylinux x86_64 and aarch64, macOS arm64, and Windows x86_64. Alpine and other musl targets
+are not supported. [RELEASING.md](RELEASING.md) says how to measure the install.
+
+`atif-sql analyze`, `atif-sql embed`, and `atif-sql search` call Amazon Bedrock and cost money.
+`analyze` is a dry run until you pass `--no-dry-run`, `embed` refuses to run without `--limit` or
+`--all` (and `--dry-run` previews it), and `search` embeds one query per call. With
+`ATIF_SQL_EMBED_PROVIDER=gemma`, `embed` and `search` run on this machine instead and cost
+nothing. Nothing else in the tool needs a credential.
+
+## What you can do
+
+Start with the question you want to answer or the work you want to keep. Each command links
+to its flags and exit codes in [the CLI reference](docs/reference/cli.md).
+
+Analysis, embedding, and text search use Amazon Bedrock. `analyze` previews by default;
+`embed` needs `--dry-run` to preview an explicit scope; `search` charges for a query embedding
+on each successful call. Rows marked **costs money** describe paid execution; embedding and
+search on this machine are free.
+
+| Task | Command | Outcome |
+| --- | --- | --- |
+| Keep a history of your Claude Code work | [`atif-sql materialize`](docs/reference/cli.md#materialize) | Saves quiet sessions for later queries and keeps them when source transcripts disappear. Re-runs update changed sessions and those with an older converter generation. |
+| Bring your Codex work into the same analysis workflow | [`atif-sql materialize --agent codex`](docs/reference/cli.md#materialize) | Saves Codex rollouts in a separate corpus with the same query surface as Claude Code. |
+| Inspect or share one conversation as ATIF | [`atif-sql convert <session.jsonl>`](docs/reference/cli.md#convert) | Prints its trajectory and loss report. Add `--trajectory-out` to save the trajectory and adjacent `edges.jsonl` file. |
+| Know whether your recent work is included | [`atif-sql status`](docs/reference/cli.md#status) | Shows corpus freshness, session counts, and what another `materialize` pass would update, without changing data. |
+| Query a large history without rereading every transcript | [`atif-sql lake rebuild`](docs/reference/cli.md#lake-rebuild) | Builds the shared DuckLake query store. Later `materialize` passes keep it current. |
+| Check that query results match your saved conversations | [`atif-sql lake verify`](docs/reference/cli.md#lake-verify) | Compares stored rows with session artifacts and exits 65 if they differ. |
+| Diagnose missing or stale query data | [`atif-sql lake status`](docs/reference/cli.md#lake-status) | Shows whether the lake exists, is current, and contains the expected corpora, plus its last write. |
+| Recover disk space from query storage | [`atif-sql lake compact`](docs/reference/cli.md#lake-compact) | Merges small files, expires old snapshots, and removes unreferenced files while keeping current rows. |
+| Find the fields you need to answer a question | [`atif-sql schema`](docs/reference/cli.md#schema) | Lists columns and macro signatures, tagged by required data: `core`, `analytics`, or `vss`. No corpus needed. |
+| Start with a query that already works | [`atif-sql examples`](docs/reference/cli.md#examples) | Gives tested queries to copy or adapt. Filters select the required data and query type. |
+| Answer your own question about past sessions | [`atif-sql query '<sql>'`](docs/reference/cli.md#query) | Returns a table to read or JSON to pipe into another tool, through a read-only sandboxed connection. |
+| Compare work across Claude Code and Codex | [`atif-sql query --all-corpora 'SELECT agent, count(*) FROM sessions GROUP BY agent'`](docs/reference/cli.md#query) | Counts sessions by agent across all corpora in the lake. Other queries can use `sessions.corpus` to identify each source. Needs the lake. |
+| See which tools your agents rely on most | [`atif-sql query 'SELECT * FROM tool_rank(30) LIMIT 10'`](docs/reference/cli.md#query) | Ranks tool use by call count over the last 30 days. |
+| Estimate the cost of a session | [`atif-sql query 'SELECT * FROM cost_estimate((SELECT session_id FROM sessions LIMIT 1)) LIMIT 10'`](docs/reference/cli.md#query) | Gives a lower-bound USD estimate and priced/unpriced step counts. Unpriced steps and cache-read charges are excluded. Replace the subquery to choose a session. |
+| Find recurring moments when you correct your agents | [`atif-sql query 'SELECT * FROM friction_counts(30) LIMIT 10'`](docs/reference/cli.md#query) | Counts detected friction labels over the last 30 days, after paid analysis has run. |
+| Plan a review of agent interactions before paying | [`atif-sql analyze`](docs/reference/cli.md#analyze) | Previews the work and estimated cost. **Costs money** with `--no-dry-run`, which fills the classifications, conflicts, user friction, and perceived errors views. |
+| Preview how much history to prepare for semantic search | [`atif-sql embed --limit 100 --dry-run`](docs/reference/cli.md#embed) | Previews embedding up to 100 steps. Removing `--dry-run` **costs money** and prepares those steps for search. |
+| Find prior work when you remember the topic, not the words | [`atif-sql search '<text>'`](docs/reference/cli.md#search) | **Costs money** for one query embedding. Finds similar steps with session identifiers and snippets. Needs existing embeddings from `embed`. |
+| Search your sessions without sending text off the machine | [`atif-sql embed --limit 200`](docs/reference/cli.md#embed) | With `ATIF_SQL_EMBED_PROVIDER=gemma` and the `local` extra, embeds 200 steps with EmbeddingGemma 2 on this machine, free, into a store of its own; `search` then reads that store. The first run downloads 1.5 GB. |
+| Find related work across both agent histories | [`atif-sql search --all-corpora '<text>'`](docs/reference/cli.md#search) | **Costs money.** Searches all corpora's embedding stores and identifies each result's corpus. Needs the lake and existing embeddings. |
+| Free space in a corpus from an older version | [`atif-sql corpus slim`](docs/reference/cli.md#corpus-slim) | Previews byte savings. `--no-dry-run` compresses JSON and removes old parquet only after lake verification succeeds. |
+| Keep your history refreshed without remembering each run | [`atif-sql cron install --script <refresh-script>`](docs/reference/cli.md#cron-install) | Prints a schedule to review and install manually. The schedule includes paid embedding and analytics. Obtain `scripts/atif-sql-refresh.sh` from a repo checkout; packaged installs need its explicit path. |
+| Check whether scheduled refreshes ran | [`atif-sql cron status --script <refresh-script>`](docs/reference/cli.md#cron-status) | Shows each lane's last completion, skip, and current lock holder from the refresh script's logs. |
+
+## How the source is organized
+
+The directories under `packages/` are internal structure, not separate installs. They
+exist so `import-linter` can enforce the layer and independence contracts at the source level;
+the only thing documented as installable is the `atif-sql` CLI above.
+
+| Directory | What |
+| --- | --- |
+| `atif-converter` | Claude Code and Codex CLI transcript → ATIF converters (ours, built on Harbor's ATIF models, vendored) + per-agent fidelity policy (loss accounting per session) |
+| `atif-corpus` | Corpus materialization: discovery, watermarks, quiescence, atomic artifact writes |
+| `atif-duck` | DuckDB views + macros over the materialized corpus (core surface plus the v2 analytics surface) |
+| `atif-models` | Model alias registry + structured-output LLM client; no other package hardcodes a model id |
+| `atif-analytics` | the LLM pipelines `atif-sql analyze` runs: classify, conflicts, friction, and perceived |
+| `atif-embed` | Cohere Embed v4 on Bedrock or EmbeddingGemma 2 on this machine + LanceDB vector store + embedding backfill |
+| `atif-cli` | the composition root, and the source of the `atif-sql` command (see [What you can do](#what-you-can-do)) |
+
+## Quick start
+
+Build the corpus from the local transcripts, then query it:
+
+```bash
+atif-sql materialize                   # discover sessions, convert, write the corpus
+atif-sql lake rebuild                  # load every corpus into the DuckLake once
+atif-sql status                        # corpus and lake freshness, read-only
+atif-sql query 'SELECT * FROM sessions LIMIT 5'
+```
+
+The lake lives at `~/.atif-sql/lake/` (`ATIF_SQL_LAKE_ROOT` moves it) and holds every corpus. Without
+one, `query` warns once and reads the per-session files.
+
+Each session is stored once. `materialize` writes the ATIF document as `trajectory.json.zst`,
+and `edges.jsonl` and `session_events.jsonl` compressed the same way, beside a plain `meta.json`,
+`loss_report.json`, and the raw source archive. The lake holds the queryable rows, and it loads a
+session from its compressed trajectory. `zstd -dc trajectory.json.zst` prints the plain ATIF
+document. Without a lake, `query --no-lake` still works, but it parses every session's trajectory
+on each run, so it's slow on a large corpus.
+
+A corpus written by an earlier version keeps working as it is; `status` counts its sessions under
+`layout`, and `atif-sql corpus slim` converts it.
+
+`materialize` converts sessions across a process pool, `min(8, cpu_count)` workers by default.
+`--workers N` (or `ATIF_SQL_MATERIALIZE_WORKERS`) sets the size, and `--workers 1` runs the
+single-process path. The pool doesn't change a byte of output: every worker writes the same
+artifacts through the same per-session staging directory and atomic rename.
+
+### Codex CLI transcripts
+
+`convert`, `materialize`, and `status` all take `--agent claude-code|codex`,
+defaulting to `claude-code`. Pick `codex` and both roots move with it: the source
+becomes `$CODEX_HOME` (default `~/.codex`) `/sessions`, and the corpus becomes
+`~/.atif-sql/corpus/codex`. An explicit `--source-root`, `--corpus-root`, or the
+matching `ATIF_SQL_*` env var still wins.
+
+```bash
+atif-sql materialize --agent codex
+atif-sql status --agent codex
+atif-sql query "SELECT agent, count(*) FROM sessions GROUP BY 1"
+```
+
+One corpus holds one agent, so a Codex corpus and a Claude Code corpus stay
+separate directories, and one `query` reads one of them. Pass `--corpus-root` to
+pick which, or `--agent codex` to get the Codex default. `--all-corpora` reads
+every corpus the lake holds at once, and `sessions.corpus` says which one a
+row came from.
+
+### Search without sending text off the machine
+
+Set `ATIF_SQL_EMBED_PROVIDER=gemma` and `embed` and `search` use Google's
+[EmbeddingGemma 2](https://huggingface.co/google/embeddinggemma-2) on this machine: step text
+and queries stay local, and nothing bills an AWS account. It needs the `local` extra (see
+[Install](#install)); in a checkout, `uv sync --all-packages --extra local`.
+
+```bash
+export ATIF_SQL_EMBED_PROVIDER=gemma
+atif-sql embed --limit 200 --dry-run   # the plan names the provider, model, width and store
+atif-sql embed --limit 200             # downloads the model on first use, then embeds
+atif-sql search 'the flaky watermark test'
+```
+
+- **The first run downloads about 1.5 GB.** The model goes into the Hugging Face cache once,
+  and later runs read it from there; `HF_HUB_OFFLINE=1` keeps every run off the network. Only
+  the 270M-parameter text tower loads, in about 4 GB of memory on a CPU.
+- **A CPU embeds about one step a second on a real corpus.** 200 steps of a Claude Code corpus
+  took 156 s on a shared 16-core machine: short steps run at about 30 a second, and the few
+  long ones (up to the model's 8,192-token window) take most of the time. A CUDA GPU with
+  bfloat16 is faster; the lockfile installs PyTorch's CPU build, so install a CUDA torch build
+  to use one.
+- **Each provider keeps its own store.** Gemma writes `<corpus_root>/embeddings_lance_gemma`
+  beside Cohere's `embeddings_lance`, so switching back and forth never mixes the two vector
+  spaces, and `search` reads the store of the provider selected when it runs.
+- **The width is 768 by default.** `ATIF_SQL_OUTPUT_DIMENSION` takes 512, 256, or 128 for a
+  smaller store at some cost in ranking quality; a store keeps the width it was written at.
+
+Working on `atif-sql` itself is a different setup: a clone, `mise`, and `mise run check` as the
+definition of done. `CONTRIBUTING.md` has it.
+
+## Agent workflow: schema → examples → query
+
+```bash
+atif-sql schema                    # every view + macro signature (<50 ms)
+atif-sql examples                  # tested example queries, grouped core/analytics/vss
+atif-sql query 'SELECT * FROM tool_rank(30) LIMIT 10'
+```
+
+`atif-sql examples` (or `atif-sql query --examples`) emits runnable queries
+derived from the catalog, not hardcoded strings, and every one is executed
+by the test suite against a fixture corpus, so the listing cannot rot.
+Piped output is JSON; filter with `--requires core|analytics|vss` and
+`--category view|table-macro|scalar-macro`.
+
+## Contributing
+
+Setup, the gates `mise run check` runs, the import-linter contracts you will trip, and the
+Conventional Commit rule the `commit-msg` hook enforces: **[CONTRIBUTING.md](CONTRIBUTING.md)**.
+
+Releases are cut by commitizen and published to PyPI over OIDC Trusted Publishing;
+**[RELEASING.md](RELEASING.md)** covers the flow, the versioning model, and the install-weight
+measurements.
+
+## Security
+
+Report a vulnerability privately through
+[GitHub's advisory form](https://github.com/laithalsaadoon/atif-sql/security/advisories/new),
+not in a public issue. Supported versions, the disclosure expectations, and what counts as a
+vulnerability in a tool that reads local transcripts are in
+**[SECURITY.md](SECURITY.md)**.
+
+`atif-sql query` runs agent-composed SQL, so it runs it in a box: sized to the host before the
+corpus is registered (override with `ATIF_SQL_QUERY_MEMORY_LIMIT` and `ATIF_SQL_QUERY_THREADS`),
+a private spill directory outside the corpus that's removed on exit, no extension installs at
+query time (`atif-sql embed --install-extension` is where the lance extension comes from), file
+facing statements (`COPY`, `EXPORT`, `ATTACH`, `INSTALL`, `LOAD`, `PREPARE`, `EXECUTE`) refused
+before they run, and a refusal to run as root unless `ATIF_SQL_ALLOW_ROOT=1` says so. The details
+and the one accepted disclosure (`duckdb_settings()` lists the granted paths) are in
+[docs/reference/cli.md](docs/reference/cli.md#query).
+
+## License
+
+[Apache License 2.0](LICENSE). Each module directory carries the same license
+file, so a published distribution ships it too.