atif-sql 0.2.0__py3-none-any.whl

atif_duck/domain/lake.py

@@ -0,0 +1,427 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""The DuckLake the corpora are queried through: tables, partitions, schema identity.
+
+One lake holds every corpus (both Claude Code corpora and the Codex one). Each
+lake table is the registry's raw reader of the same artifact kind
+(:mod:`atif_duck.domain.raw_readers`) with two identity columns in front,
+``corpus`` (the corpus directory's name) and ``agent`` (the agent the corpus
+holds). The per-session artifacts stay the source of truth: the lake is
+loaded FROM the registry's raw relations over them, so a lake row is by
+construction the row the per-session path would have produced, and the views
+bind over either with no change.
+
+Nothing here is declared twice. Every payload column comes from the raw
+reader shapes, which come from the catalog and the columnar contract; the
+only lake-local decisions are the identity columns, the rename rule for a
+source column that collides with one, the derived ``started_at`` the sessions
+table is partitioned by, and the partition specs.
+
+Partitioning: every table is partitioned by ``agent`` and ``corpus`` first,
+then by month where a row has a time. ``corpus`` is there because a query is
+scoped to one corpus by default, and a corpus-pure file is what lets that scope
+skip the other corpora's files. It also keeps compaction from merging two
+corpora's rows into one file. ``agent`` is implied by ``corpus`` (one corpus
+holds one agent), so it adds no files, only a readable directory level. The
+two one-row-per-session tables (``loss_reports``, ``session_meta``) have no
+time column and stay at ``agent``/``corpus``: splitting a few thousand small
+rows by month would multiply files without pruning anything a query asks for.
+
+Schema identity: the lake records :data:`LAKE_SCHEMA_VERSION`, a digest of
+the rendered table definitions (:func:`lake_schema_digest`), and the columnar
+schema version it was loaded under. Any of the three differing from the
+running code makes the lake STALE: the writer rebuilds it and a reader falls
+back to the per-session path. ``tests/test_lake.py`` pins the digest
+beside the version, so a change to any shape the lake derives from fails until
+someone decides whether it also needs a version bump.
+
+Pure constants and string building: no duckdb import, no filesystem access.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+
+from atif_duck.domain.catalog import VIEW_SCHEMA
+from atif_duck.domain.columnar import COLUMNAR_SCHEMA_VERSION
+from atif_duck.domain.raw_readers import (
+    CORPUS_COLUMN,
+    EDGE_COLUMNS,
+    LOSS_REPORT_COLUMNS,
+    LOSS_REPORT_PATH_COLUMN,
+    META_COLUMNS,
+    RAW_EDGES,
+    RAW_LOSS_REPORTS,
+    RAW_META,
+    RAW_SESSION_EVENTS,
+    RAW_STEPS,
+    RAW_TOOL_CALLS,
+    RAW_TOOL_RESULTS,
+    RAW_TRAJECTORIES,
+    SESSION_ID_PATH,
+    TRAJECTORY_RELATION_COLUMNS,
+)
+from atif_duck.domain.sql_literal import SqlFragment
+
+#: Bump when the lake's meaning changes in a way the table definitions do not
+#: show (what a row is loaded from, how a key is derived). A change the
+#: definitions DO show moves :func:`lake_schema_digest` on its own; the pin test
+#: makes the author decide whether it also warrants a bump.
+LAKE_SCHEMA_VERSION: int = 1
+
+#: The lake tables whose rows keep their source order. A step's tool calls
+#: and results carry no ordinal, so their order within a step is the order the
+#: writer inserted them in, which DuckLake keeps as ``rowid``. A parallel
+#: partitioned insert interleaves rows from its threads (measured: results
+#: of one step out of order in a few sessions per hundred), so the writer
+#: inserts these tables on one thread. Part of :func:`lake_schema_digest`: a
+#: lake written without the guarantee reads back misordered, so it is stale.
+SOURCE_ORDERED_TABLES: tuple[str, ...] = ("steps", "tool_calls", "tool_results")
+
+#: Parquet row group size for every lake file. DuckDB's default (122,880 rows)
+#: puts a whole month of a corpus's tool results in one row group, so reading
+#: one session's results decompressed every other session's too (measured on
+#: the copied corpora: 0.7 s per session). The step-level tables' rows land
+#: in session order, so a small group holds a few sessions and a session read
+#: touches one or two. Part of :func:`lake_schema_digest`, so a lake written in
+#: another layout is rebuilt rather than left slow.
+LAKE_ROW_GROUP_SIZE: int = 4096
+
+#: The name the lake is attached under on every connection that reads or
+#: writes it. Not part of the queryable surface: the views are.
+LAKE_ALIAS: str = "atif_lake"
+
+#: The metadata database DuckLake attaches beside :data:`LAKE_ALIAS`.
+LAKE_METADATA_ALIAS: str = f"__ducklake_metadata_{LAKE_ALIAS}"
+
+#: The identity every lake table starts with.
+AGENT_COLUMN: str = "agent"
+SESSION_KEY: str = "session_id"
+IDENTITY_COLUMNS: tuple[tuple[str, str], ...] = (
+    (CORPUS_COLUMN, "VARCHAR"),
+    (AGENT_COLUMN, "VARCHAR"),
+    (SESSION_KEY, "VARCHAR"),
+)
+
+#: A source column whose name is one of the identity names is stored under
+#: this prefix (the trajectory's own ``agent`` struct becomes ``src_agent``).
+COLLISION_PREFIX: str = "src_"
+
+#: ``lake_info`` keys (the lake's recorded schema identity).
+INFO_SCHEMA_VERSION: str = "lake_schema_version"
+INFO_SCHEMA_DIGEST: str = "lake_schema_digest"
+INFO_COLUMNAR_SCHEMA: str = "columnar_schema_version"
+
+#: The two bookkeeping tables. Their shape never changes, so a stale lake can
+#: still say which corpora it holds and a rebuild can find them again.
+LAKE_INFO_TABLE: str = "lake_info"
+CORPORA_TABLE: str = "corpora"
+LAKE_INFO_COLUMNS: tuple[tuple[str, str], ...] = (("key", "VARCHAR"), ("value", "VARCHAR"))
+CORPORA_COLUMNS: tuple[tuple[str, str], ...] = (
+    (CORPUS_COLUMN, "VARCHAR"),
+    (AGENT_COLUMN, "VARCHAR"),
+    ("corpus_root", "VARCHAR"),
+    ("registered_at", "VARCHAR"),
+)
+
+#: The source for the sessions table: the trajectory rows plus the first
+#: step's time, which the table is partitioned by.
+_SESSIONS_SOURCE: str = (
+    f"(SELECT t.*, s.started_at FROM {RAW_TRAJECTORIES} t LEFT JOIN "  # noqa: S608  # nosec B608 - module constants only
+    f"(SELECT session_id, min(ts) AS started_at FROM {RAW_STEPS} GROUP BY session_id) s "
+    f"ON s.session_id = t.{SESSION_ID_PATH})"
+)
+
+_MONTHLY: tuple[str, ...] = ("year({ts})", "month({ts})")
+
+
+@dataclass(frozen=True, slots=True)
+class LakeColumn:
+    """One payload column: its lake name, type, and the source column it copies."""
+
+    name: str
+    sql_type: str
+    source: str
+    #: Computed at load time rather than copied from the relation, so the
+    #: reader view does not select it back (``sessions.started_at``).
+    derived: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class LakeTable:
+    """One lake table and the raw relation it is loaded from and read back as."""
+
+    #: Table name inside the lake.
+    name: str
+    #: The raw reader (a registry relation name) whose shape the table stores.
+    relation: str
+    #: What the load selects from: the relation itself, or a constant subquery.
+    load_source: str
+    #: The source column holding the session key (``session_id`` or ``session_id_path``).
+    key_source: str
+    #: Payload columns after the identity columns, in order.
+    payload: tuple[LakeColumn, ...]
+    #: Partition expressions, after the identity-derived ``agent, corpus``.
+    partition_by: tuple[str, ...]
+    #: Source columns the relation exposes that the reader view must rebuild
+    #: from the identity (``corpus`` on the trajectories relation).
+    relation_extras: tuple[str, ...] = ()
+
+    @property
+    def columns(self) -> tuple[tuple[str, str], ...]:
+        """Every lake column, identity first."""
+        return (*IDENTITY_COLUMNS, *((c.name, c.sql_type) for c in self.payload))
+
+
+def _payload(
+    columns: tuple[tuple[str, str], ...], *, key_source: str, skip: tuple[str, ...] = ()
+) -> tuple[LakeColumn, ...]:
+    """Map a relation's columns onto lake payload columns (key and identity renamed)."""
+    reserved = {name for name, _ in IDENTITY_COLUMNS}
+    out: list[LakeColumn] = []
+    for name, sql_type in columns:
+        if name == key_source or name in skip:
+            continue
+        lake_name = f"{COLLISION_PREFIX}{name}" if name in reserved else name
+        out.append(LakeColumn(name=lake_name, sql_type=sql_type, source=name))
+    return tuple(out)
+
+
+def _monthly(ts: str) -> tuple[str, ...]:
+    return tuple(part.format(ts=ts) for part in _MONTHLY)
+
+
+def _view_table(name: str, relation: str, view: str) -> LakeTable:
+    return LakeTable(
+        name=name,
+        relation=relation,
+        load_source=relation,
+        key_source=SESSION_KEY,
+        payload=_payload(VIEW_SCHEMA[view], key_source=SESSION_KEY),
+        partition_by=_monthly("ts"),
+    )
+
+
+#: Every lake table, in load order. The sessions table reads the steps
+#: relation for ``started_at``, which the registry binds before the load runs.
+LAKE_TABLES: tuple[LakeTable, ...] = (
+    LakeTable(
+        name="sessions",
+        relation=RAW_TRAJECTORIES,
+        load_source=_SESSIONS_SOURCE,
+        key_source=SESSION_ID_PATH,
+        payload=(
+            *_payload(TRAJECTORY_RELATION_COLUMNS, key_source=SESSION_ID_PATH),
+            LakeColumn(name="started_at", sql_type="TIMESTAMP", source="started_at", derived=True),
+        ),
+        partition_by=_monthly("started_at"),
+        relation_extras=(CORPUS_COLUMN,),
+    ),
+    _view_table("steps", RAW_STEPS, "steps"),
+    _view_table("tool_calls", RAW_TOOL_CALLS, "tool_calls"),
+    _view_table("tool_results", RAW_TOOL_RESULTS, "tool_results"),
+    _view_table("session_events", RAW_SESSION_EVENTS, "session_events"),
+    LakeTable(
+        name="edges",
+        relation=RAW_EDGES,
+        load_source=RAW_EDGES,
+        key_source=SESSION_ID_PATH,
+        payload=_payload(tuple(EDGE_COLUMNS.items()), key_source=SESSION_ID_PATH),
+        partition_by=_monthly("ts"),
+    ),
+    LakeTable(
+        name="loss_reports",
+        relation=RAW_LOSS_REPORTS,
+        load_source=RAW_LOSS_REPORTS,
+        key_source=SESSION_ID_PATH,
+        payload=_payload(
+            (*LOSS_REPORT_COLUMNS.items(), LOSS_REPORT_PATH_COLUMN), key_source=SESSION_ID_PATH
+        ),
+        partition_by=(),
+    ),
+    LakeTable(
+        name="session_meta",
+        relation=RAW_META,
+        load_source=RAW_META,
+        key_source=SESSION_ID_PATH,
+        payload=_payload(tuple(META_COLUMNS.items()), key_source=SESSION_ID_PATH),
+        partition_by=(),
+    ),
+)
+
+LAKE_TABLE_NAMES: tuple[str, ...] = tuple(table.name for table in LAKE_TABLES)
+
+
+# ---------------------------------------------------------------------------
+# Statements. Every one is a module constant built from LAKE_TABLES, so the
+# AST audit in tests/test_sql_text_boundaries.py can prove each placeholder
+# is catalog text; the only values that ever vary (corpus, agent, session
+# ids) are bound parameters.
+# ---------------------------------------------------------------------------
+
+#: ``CREATE TABLE`` for the two bookkeeping tables.
+BOOTSTRAP_CREATE_SQL: tuple[SqlFragment, ...] = (
+    SqlFragment(
+        f"CREATE TABLE {LAKE_ALIAS}.{LAKE_INFO_TABLE} "
+        f"({', '.join(f'{name} {sql_type}' for name, sql_type in LAKE_INFO_COLUMNS)})"
+    ),
+    SqlFragment(
+        f"CREATE TABLE {LAKE_ALIAS}.{CORPORA_TABLE} "
+        f"({', '.join(f'{name} {sql_type}' for name, sql_type in CORPORA_COLUMNS)})"
+    ),
+)
+
+#: ``CREATE TABLE`` per lake table.
+CREATE_TABLE_SQL: dict[str, SqlFragment] = {
+    t.name: SqlFragment(
+        f"CREATE TABLE {LAKE_ALIAS}.{t.name} "
+        f"({', '.join(f'{name} {sql_type}' for name, sql_type in t.columns)})"
+    )
+    for t in LAKE_TABLES
+}
+
+#: ``ALTER TABLE ... SET PARTITIONED BY`` per lake table: agent and corpus
+#: first, then the table's own time partition.
+PARTITION_SQL: dict[str, SqlFragment] = {
+    t.name: SqlFragment(
+        f"ALTER TABLE {LAKE_ALIAS}.{t.name} SET PARTITIONED BY "
+        f"({', '.join((AGENT_COLUMN, CORPUS_COLUMN, *t.partition_by))})"
+    )
+    for t in LAKE_TABLES
+}
+
+#: Delete one corpus's rows for a bound list of session ids (params: corpus, ids).
+DELETE_SESSIONS_SQL: dict[str, SqlFragment] = {
+    t.name: SqlFragment(
+        f"DELETE FROM {LAKE_ALIAS}.{t.name} WHERE {CORPUS_COLUMN} = ? "  # noqa: S608  # nosec B608 - constants only; corpus and ids are bound
+        f"AND {SESSION_KEY} IN (SELECT unnest(?::VARCHAR[]))"
+    )
+    for t in LAKE_TABLES
+}
+
+#: Delete every row of one corpus (param: corpus).
+DELETE_CORPUS_SQL: dict[str, SqlFragment] = {
+    t.name: SqlFragment(f"DELETE FROM {LAKE_ALIAS}.{t.name} WHERE {CORPUS_COLUMN} = ?")  # noqa: S608  # nosec B608 - constants only; corpus is bound
+    for t in LAKE_TABLES
+}
+
+#: Copy the bound raw relation's rows into the lake (params: corpus, agent).
+#: Every column is cast to its declared type, so a relation that infers a
+#: looser type (an all-NULL column) still lands in the table's shape.
+INSERT_SQL: dict[str, SqlFragment] = {
+    t.name: SqlFragment(
+        f"INSERT INTO {LAKE_ALIAS}.{t.name} ({', '.join(name for name, _ in t.columns)}) "  # noqa: S608  # nosec B608 - constants only; corpus and agent are bound
+        f"SELECT CAST(? AS VARCHAR), CAST(? AS VARCHAR), CAST({t.key_source} AS VARCHAR), "
+        f"{', '.join(f'CAST({c.source} AS {c.sql_type})' for c in t.payload)} "
+        f"FROM {t.load_source}"
+    )
+    for t in LAKE_TABLES
+}
+
+#: Each lake table read back in its raw relation's shape, every corpus. A
+#: reader appends ``WHERE corpus = <literal>`` to scope it.
+READER_SELECT_SQL: dict[str, SqlFragment] = {
+    t.name: SqlFragment(
+        "SELECT "  # noqa: S608  # nosec B608 - constants only
+        + ", ".join(
+            (
+                *(f"{c.name} AS {c.source}" for c in t.payload if not c.derived),
+                f"{SESSION_KEY} AS {t.key_source}",
+                *t.relation_extras,
+            )
+        )
+        + f" FROM {LAKE_ALIAS}.{t.name}"
+    )
+    for t in LAKE_TABLES
+}
+
+#: Per-session ``(session_id, rows, content hash)`` over the registry's raw
+#: relation, columns cast as the load casts them. ``sum(hash(...))`` rather
+#: than ``bit_xor``: order-free like an XOR, but two identical rows do not
+#: cancel out.
+ARTIFACT_HASH_SQL: dict[str, SqlFragment] = {
+    t.name: SqlFragment(
+        f"SELECT CAST({t.key_source} AS VARCHAR) AS session_id, count(*) AS n, "  # noqa: S608  # nosec B608 - constants only
+        f"sum(CAST(hash({', '.join(f'CAST({c.source} AS {c.sql_type})' for c in t.payload)}) "
+        f"AS HUGEINT)) AS h FROM {t.load_source} GROUP BY 1"
+    )
+    for t in LAKE_TABLES
+}
+
+#: The same over the lake table for one corpus (param: corpus).
+LAKE_HASH_SQL: dict[str, SqlFragment] = {
+    t.name: SqlFragment(
+        f"SELECT {SESSION_KEY} AS session_id, count(*) AS n, "  # noqa: S608  # nosec B608 - constants only; corpus is bound
+        f"sum(CAST(hash({', '.join(c.name for c in t.payload)}) AS HUGEINT)) AS h "
+        f"FROM {LAKE_ALIAS}.{t.name} WHERE {CORPUS_COLUMN} = ? GROUP BY 1"
+    )
+    for t in LAKE_TABLES
+}
+
+
+def lake_schema_digest() -> str:
+    """sha256 over every table's name, columns and partition spec, bookkeeping and file layout included."""
+    spec = {
+        "row_group_size": LAKE_ROW_GROUP_SIZE,
+        "source_ordered_tables": SOURCE_ORDERED_TABLES,
+        "tables": [
+            {"name": t.name, "columns": t.columns, "partition_by": t.partition_by}
+            for t in LAKE_TABLES
+        ],
+        LAKE_INFO_TABLE: LAKE_INFO_COLUMNS,
+        CORPORA_TABLE: CORPORA_COLUMNS,
+    }
+    return hashlib.sha256(json.dumps(spec, sort_keys=True).encode("utf-8")).hexdigest()
+
+
+def expected_lake_info() -> dict[str, str]:
+    """The ``lake_info`` rows a lake written by this code carries."""
+    return {
+        INFO_SCHEMA_VERSION: str(LAKE_SCHEMA_VERSION),
+        INFO_SCHEMA_DIGEST: lake_schema_digest(),
+        INFO_COLUMNAR_SCHEMA: str(COLUMNAR_SCHEMA_VERSION),
+    }
+
+
+def lake_info_mismatches(recorded: dict[str, str]) -> tuple[str, ...]:
+    """The ``lake_info`` keys whose recorded value differs from this code's (sorted)."""
+    expected = expected_lake_info()
+    return tuple(sorted(key for key, value in expected.items() if recorded.get(key) != value))
+
+
+__all__ = [
+    "AGENT_COLUMN",
+    "ARTIFACT_HASH_SQL",
+    "BOOTSTRAP_CREATE_SQL",
+    "COLLISION_PREFIX",
+    "CORPORA_COLUMNS",
+    "CORPORA_TABLE",
+    "CREATE_TABLE_SQL",
+    "DELETE_CORPUS_SQL",
+    "DELETE_SESSIONS_SQL",
+    "IDENTITY_COLUMNS",
+    "INFO_COLUMNAR_SCHEMA",
+    "INFO_SCHEMA_DIGEST",
+    "INFO_SCHEMA_VERSION",
+    "INSERT_SQL",
+    "LAKE_ALIAS",
+    "LAKE_HASH_SQL",
+    "LAKE_INFO_COLUMNS",
+    "LAKE_INFO_TABLE",
+    "LAKE_METADATA_ALIAS",
+    "LAKE_ROW_GROUP_SIZE",
+    "LAKE_SCHEMA_VERSION",
+    "LAKE_TABLES",
+    "LAKE_TABLE_NAMES",
+    "PARTITION_SQL",
+    "READER_SELECT_SQL",
+    "SESSION_KEY",
+    "SOURCE_ORDERED_TABLES",
+    "LakeColumn",
+    "LakeTable",
+    "expected_lake_info",
+    "lake_info_mismatches",
+    "lake_schema_digest",
+]

atif_duck/domain/lake_sessions.py

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""The statements that read one corpus's sessions back out of the lake, a batch at a time.
+
+The LLM pipelines (atif-analytics) walk a corpus session by session: bounds
+for every session, then the steps of the ones a gate or the renderer needs.
+:class:`atif_duck.infrastructure.lake_sessions.LakeSessionReader` runs these
+statements for them (atif-cli adapts its rows to atif-analytics' session
+port, since the two packages may not import each other).
+
+Every statement is a module constant over the lake tables of
+:mod:`atif_duck.domain.lake`, so the AST audit in
+``tests/test_sql_text_boundaries.py`` reads them like any other constant. The
+corpus and the session ids are bound parameters; a batch of ids is one
+``VARCHAR[]`` parameter.
+
+Row order. A step's tool calls and results have no ordinal column; their
+order is the order the writer inserted them in, which the lake keeps as
+``rowid`` (DuckLake numbers a data file's rows from its ``row_id_start``, and
+merges keep the numbers). The writer inserts the step-level tables
+(:data:`atif_duck.domain.lake.SOURCE_ORDERED_TABLES`) on one thread so that order is the source
+order; the statements below sort by ``step_id`` and then ``rowid``.
+
+Pure constants: no duckdb import, no filesystem access.
+"""
+
+from __future__ import annotations
+
+from atif_duck.domain.lake import LAKE_ALIAS, SESSION_KEY
+from atif_duck.domain.raw_readers import CORPUS_COLUMN
+from atif_duck.domain.sql_literal import SqlFragment
+
+_SCOPE: str = f"{CORPUS_COLUMN} = ? AND {SESSION_KEY} IN (SELECT unnest(?::VARCHAR[]))"
+_ORDER: str = f"ORDER BY {SESSION_KEY}, step_id, rowid"
+
+#: ``(session_id, materialized_at, source_mtime_ns)`` for every session of a
+#: corpus (param: corpus). What a session's ``meta.json`` said when the lake
+#: loaded it, so a reader can tell a current session from one the lake holds an
+#: older write of.
+SESSION_FRESHNESS_SQL: SqlFragment = SqlFragment(
+    f"SELECT {SESSION_KEY}, materialized_at, source_mtime_ns "  # noqa: S608  # nosec B608 - constants only; corpus is bound
+    f"FROM {LAKE_ALIAS}.session_meta WHERE {CORPUS_COLUMN} = ?"
+)
+
+#: ``(session_id, newest record ts)`` for every session of a corpus (param:
+#: corpus): the checkpoint's first bound, from ``edges`` like the file path.
+LAST_EDGE_TS_SQL: SqlFragment = SqlFragment(
+    f"SELECT {SESSION_KEY}, max(ts) FROM {LAKE_ALIAS}.edges "  # noqa: S608  # nosec B608 - constants only; corpus is bound
+    f"WHERE {CORPUS_COLUMN} = ? GROUP BY 1"
+)
+
+#: Every step of a batch of sessions without its tool payloads (params:
+#: corpus, ids): ``(session_id, step_id, ts, source, message, is_sidechain,
+#: is_compact_summary, source_uuids)``.
+TURNS_SQL: SqlFragment = SqlFragment(
+    "SELECT "  # noqa: S608  # nosec B608 - constants only; corpus and ids are bound
+    f"{SESSION_KEY}, step_id, ts, source, message, is_sidechain, is_compact_summary, "
+    f"CAST(source_uuids AS VARCHAR) FROM {LAKE_ALIAS}.steps WHERE {_SCOPE} {_ORDER}"
+)
+
+#: ``(session_id, step_id)`` of every step with an error result in a batch
+#: (params: corpus, ids). The gates need the flag, not the result bodies.
+ERROR_STEPS_SQL: SqlFragment = SqlFragment(
+    f"SELECT DISTINCT {SESSION_KEY}, step_id FROM {LAKE_ALIAS}.tool_results "  # noqa: S608  # nosec B608 - constants only; corpus and ids are bound
+    f"WHERE {_SCOPE} AND is_error"
+)
+
+#: Every tool call of a batch, in source order (params: corpus, ids):
+#: ``(session_id, step_id, tool_name, tool_input)`` with the input as JSON text.
+TOOL_CALLS_SQL: SqlFragment = SqlFragment(
+    f"SELECT {SESSION_KEY}, step_id, tool_name, CAST(tool_input AS VARCHAR) "  # noqa: S608  # nosec B608 - constants only; corpus and ids are bound
+    f"FROM {LAKE_ALIAS}.tool_calls WHERE {_SCOPE} {_ORDER}"
+)
+
+#: Every tool result of a batch, in source order (params: corpus, ids):
+#: ``(session_id, step_id, tool_use_id, content, is_error)`` with the content
+#: as JSON text.
+TOOL_RESULTS_SQL: SqlFragment = SqlFragment(
+    f"SELECT {SESSION_KEY}, step_id, tool_use_id, CAST(content AS VARCHAR), is_error "  # noqa: S608  # nosec B608 - constants only; corpus and ids are bound
+    f"FROM {LAKE_ALIAS}.tool_results WHERE {_SCOPE} {_ORDER}"
+)
+
+#: The non-empty raw-record uuids of one session (params: corpus, session id).
+EDGE_UUIDS_SQL: SqlFragment = SqlFragment(
+    f"SELECT DISTINCT uuid FROM {LAKE_ALIAS}.edges "  # noqa: S608  # nosec B608 - constants only; corpus and the id are bound
+    f"WHERE {CORPUS_COLUMN} = ? AND {SESSION_KEY} = ? AND uuid IS NOT NULL AND uuid <> ''"
+)
+
+__all__ = [
+    "EDGE_UUIDS_SQL",
+    "ERROR_STEPS_SQL",
+    "LAST_EDGE_TS_SQL",
+    "SESSION_FRESHNESS_SQL",
+    "TOOL_CALLS_SQL",
+    "TOOL_RESULTS_SQL",
+    "TURNS_SQL",
+]

atif_duck/domain/raw_readers.py

@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""The raw readers' names and column shapes, shared by the registry and the lake.
+
+The registry (:mod:`atif_duck.infrastructure.registry`) binds one relation per
+corpus artifact kind under these names, and every business view reads them
+and nothing else. The lake (:mod:`atif_duck.domain.lake`) derives its tables
+from the same shapes, so a column declared here is declared once for both the
+per-session path and the lake path.
+
+The names are deliberately absent from :data:`atif_duck.domain.catalog.VIEW_NAMES`:
+they describe corpus files rather than being queryable business surface.
+
+Pure constants: no duckdb import, no filesystem access.
+"""
+
+from __future__ import annotations
+
+from atif_duck.domain.columnar import META_COLUMNAR_KEY, SESSION_COLUMNS
+
+#: The trajectory's top-level members minus ``steps``, plus the two
+#: path-derived keys. ``v_raw_trajectories`` carries these, then ``corpus``.
+RAW_TRAJECTORIES: str = "v_raw_trajectories"
+#: One row per ATIF step, in the ``steps`` view's column shape.
+RAW_STEPS: str = "v_raw_steps"
+#: One row per tool call, in the ``tool_calls`` view's column shape.
+RAW_TOOL_CALLS: str = "v_raw_tool_calls"
+#: One row per observation result, in the ``tool_results`` view's column shape.
+RAW_TOOL_RESULTS: str = "v_raw_tool_results"
+#: One row per kept non-message record, in the ``session_events`` view's shape.
+RAW_SESSION_EVENTS: str = "v_raw_session_events"
+#: One row per RAW transcript record, from ``edges.jsonl``.
+RAW_EDGES: str = "v_raw_edges"
+#: One row per session, from ``loss_report.json``.
+RAW_LOSS_REPORTS: str = "v_raw_loss_reports"
+#: One row per complete session, from ``meta.json``.
+RAW_META: str = "v_raw_meta"
+
+#: The column every raw reader except the step-level four keys its session
+#: on: the ``sessions/<id>/`` directory name.
+SESSION_ID_PATH: str = "session_id_path"
+
+#: The ``corpus`` column ``v_raw_trajectories`` ends with: the corpus
+#: directory's name, which is how the lake and the ``sessions`` view name a
+#: corpus.
+CORPUS_COLUMN: str = "corpus"
+
+#: ``v_raw_trajectories`` columns before ``corpus``, in order: the
+#: trajectory's top-level members minus ``steps``, then the two path-derived
+#: keys.
+TRAJECTORY_RELATION_COLUMNS: tuple[tuple[str, str], ...] = (
+    *(column for column in SESSION_COLUMNS if column[0] != SESSION_ID_PATH),
+    ("trajectory_path", "VARCHAR"),
+    (SESSION_ID_PATH, "VARCHAR"),
+)
+
+# Explicit projection for ``v_raw_edges``: one line per RAW transcript record
+# per docs/CONTRACT.md. ``parent_uuid`` is declared VARCHAR outright: a root
+# record leaves it null, so inferred typing resolves it as a NULL-vs-string
+# JSON union and every downstream view then needs its own CAST. Declaring the
+# type in the one explicit-columns reader pays that cost once.
+EDGE_COLUMNS: dict[str, str] = {
+    "uuid": "VARCHAR",
+    "parent_uuid": "VARCHAR",
+    "message_id": "VARCHAR",
+    "type": "VARCHAR",
+    "ts": "TIMESTAMP",
+    "is_sidechain": "BOOLEAN",
+    "is_compact_summary": "BOOLEAN",
+    "source_file": "VARCHAR",
+    "tool_use_ids": "JSON",
+}
+
+# Explicit projection for ``v_raw_loss_reports``: atif_converter
+# ``LossReport.to_json()`` shape (record_counts / gaps_observed stay JSON —
+# enum-keyed dict and list respectively).
+LOSS_REPORT_COLUMNS: dict[str, str] = {
+    "record_counts": "JSON",
+    "records_total": "BIGINT",
+    "records_converted": "BIGINT",
+    # Added with session_events (converter schema 2). A report written before
+    # then has no such key and reads as NULL, which the view keeps as NULL:
+    # "not measured" rather than "none captured".
+    "records_captured": "BIGINT",
+    "records_dropped": "BIGINT",
+    "gaps_observed": "JSON",
+    "subagent_files_found": "BIGINT",
+    "subagent_files_convertible": "BIGINT",
+    "workflow_subagent_files_found": "BIGINT",
+}
+
+#: The path column ``v_raw_loss_reports`` adds (the ``loss_reports`` view
+#: returns it).
+LOSS_REPORT_PATH_COLUMN: tuple[str, str] = ("report_path", "VARCHAR")
+
+# Explicit projection for ``v_raw_meta`` per docs/CONTRACT.md. An explicit
+# ``columns=`` projection nulls a missing key rather than failing, which is
+# what lets a corpus written before a key existed still register.
+META_COLUMNS: dict[str, str] = {
+    "session_id": "VARCHAR",
+    "source_mtime_ns": "BIGINT",
+    "source_files": "JSON",
+    "harbor_version": "VARCHAR",
+    "converter_version": "VARCHAR",
+    "materialized_at": "VARCHAR",
+    # Which agent wrote the transcript. Added 2026-09-11 with Codex support, so
+    # a session materialized before then has no such key and reads as NULL.
+    # Queries read ``sessions.agent`` (from the trajectory) instead; this
+    # column is provenance for an operator reading meta.json directly.
+    "agent": "VARCHAR",
+    # Which columnar schema the session's parquet artifacts were written
+    # against. Absent (NULL) on a session materialized before the artifacts
+    # existed; that session is read from trajectory.json, which is always
+    # correct.
+    META_COLUMNAR_KEY: "BIGINT",
+    # The converter's output version, and whether the source transcript still
+    # exists. NULL on a session written before the key existed.
+    "converter_schema": "BIGINT",
+    "source_present": "BOOLEAN",
+    "source_removed_at": "VARCHAR",
+}
+
+
+__all__ = [
+    "CORPUS_COLUMN",
+    "EDGE_COLUMNS",
+    "LOSS_REPORT_COLUMNS",
+    "LOSS_REPORT_PATH_COLUMN",
+    "META_COLUMNS",
+    "RAW_EDGES",
+    "RAW_LOSS_REPORTS",
+    "RAW_META",
+    "RAW_SESSION_EVENTS",
+    "RAW_STEPS",
+    "RAW_TOOL_CALLS",
+    "RAW_TOOL_RESULTS",
+    "RAW_TRAJECTORIES",
+    "SESSION_ID_PATH",
+    "TRAJECTORY_RELATION_COLUMNS",
+]