atif-sql 0.2.0__py3-none-any.whl

atif_cli/output.py

@@ -0,0 +1,297 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Agent-friendly output formatting for the atif-sql CLI.
+
+One rule holds across every command: output goes through :func:`emit_rows`
+(tabular) or :func:`emit_json` (structured), ``--format auto`` resolves to a
+human table on a TTY and machine-readable JSON on a pipe, and classified
+errors land on stderr as one readable line (TTY) or a JSON envelope (pipe)
+with a stable exit code. A command that writes to stdout directly breaks the
+pipe contract an agent depends on.
+
+No dataframe library. The tabular surface is ``(columns, rows)`` straight off
+the DuckDB cursor, so the table renderer is a small width-aligned formatter
+and JSON/CSV go through the stdlib. That is what keeps this module importable
+on the lean path — no duckdb, no dataframe stack — which the
+fresh-interpreter lean-import test pins.
+
+Unbounded results go through :func:`emit_cursor`, which walks the cursor in
+``fetchmany`` batches and writes each batch straight to stdout, so the
+client holds one batch of Python tuples instead of every row. DuckDB's
+``memory_limit`` does not bound that client-side copy at all — it caps the
+engine's own heap.
+
+What streaming buys is measured, and it is narrower than "less memory
+always". It wins when a large result comes off a cheap scan: 3,000,000 rows
+out of ``range()`` peak near 18 MB of RSS above baseline streamed, against
+roughly 540 MB under ``fetchall``. It does NOT win on a JSON-parsing view.
+``SELECT * FROM steps`` over a real corpus (115,265 rows, 149 MB of JSON)
+holds the view's scan resident near 3 GB for the whole write, and paired
+runs put streaming and ``fetchall`` in one overlapping band with no
+consistent ordering between them. Streaming is the default because it is
+the mode whose client-side cost stays flat as a result grows, not because
+it lowers the peak on every query.
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import sys
+from enum import StrEnum
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Iterator, Sequence
+
+    from atif_cli.errors import ClassifiedError
+
+#: Cell truncation width, TABLE format ONLY. JSON and CSV emit the full value:
+#: truncation is a terminal-legibility device, and silently shortening a value
+#: on a machine-readable path would corrupt the data an agent reads back.
+_TABLE_CELL_MAX = 120
+
+#: Rows per ``fetchmany`` in :func:`emit_cursor`. Big enough that the
+#: per-batch overhead is noise and the table renderer measures its column
+#: widths over a representative sample; small enough that peak resident rows
+#: stay bounded regardless of result size.
+_STREAM_BATCH_ROWS = 10_000
+
+
+class OutputFormat(StrEnum):
+    """Supported output formats.
+
+    ``AUTO`` resolves to ``TABLE`` when stdout is a TTY and ``JSON``
+    otherwise. A string Enum is what lets cyclopts parse ``--format json``
+    with no custom converter. The contract names ``auto|json|csv``; ``table``
+    is what ``auto`` resolves to on a TTY, and it is accepted explicitly so a
+    caller can force the human grid onto a pipe.
+    """
+
+    AUTO = "auto"
+    TABLE = "table"
+    JSON = "json"
+    CSV = "csv"
+
+
+def resolve_format(fmt: OutputFormat | str) -> OutputFormat:
+    """Resolve ``AUTO`` against the current stdout; no-op for explicit formats."""
+    # `OutputFormat` is a `StrEnum`, so a member IS a str and the constructor is
+    # the identity on one — no branch is needed to accept either input shape.
+    resolved = OutputFormat(fmt)
+    if resolved is not OutputFormat.AUTO:
+        return resolved
+    return OutputFormat.TABLE if sys.stdout.isatty() else OutputFormat.JSON
+
+
+def _cell(value: Any) -> str:
+    """Render one table cell: NULL marker, truncated, newline-flattened."""
+    if value is None:
+        return "∅"
+    text = str(value).replace("\n", "\\n")
+    if len(text) > _TABLE_CELL_MAX:
+        return text[: _TABLE_CELL_MAX - 1] + "…"
+    return text
+
+
+def json_object_keys(columns: Sequence[str]) -> list[str]:
+    """Column names made unique, so a JSON row object keeps every column.
+
+    A JSON row object is keyed by column name, and SQL column names are not
+    unique: any un-aliased self-join
+    (``SELECT s.session_id, m.session_id FROM sessions s JOIN messages m
+    USING (session_id)``) returns two columns with one name. Keying a dict
+    by them keeps only the LAST value and returns fewer columns than were
+    asked for, with no error. Repeats get a ``_<n>`` suffix instead, counting
+    from 1 in column order and skipping suffixes already taken by an
+    explicit alias, so the mapping is a pure function of ``columns``.
+    """
+    taken = set(columns)
+    seen: set[str] = set()
+    keys: list[str] = []
+    for column in columns:
+        if column not in seen:
+            seen.add(column)
+            keys.append(column)
+            continue
+        suffix = 1
+        while f"{column}_{suffix}" in taken or f"{column}_{suffix}" in seen:
+            suffix += 1
+        key = f"{column}_{suffix}"
+        seen.add(key)
+        keys.append(key)
+    return keys
+
+
+def _batched(rows: Sequence[Sequence[Any]], size: int) -> Iterator[Sequence[Sequence[Any]]]:
+    """Slice an in-memory row sequence into batches of at most ``size``."""
+    for start in range(0, len(rows), size):
+        yield rows[start : start + size]
+
+
+def emit_rows(
+    columns: Sequence[str],
+    rows: Sequence[Sequence[Any]],
+    fmt: OutputFormat | str = OutputFormat.AUTO,
+) -> None:
+    """Write an in-memory query result to stdout in the requested format.
+
+    Parameters
+    ----------
+    columns
+        Column names, from ``cursor.description``.
+    rows
+        Row tuples, from ``cursor.fetchall()``.
+    fmt
+        One of :class:`OutputFormat`; ``AUTO`` resolves per
+        :func:`resolve_format`.
+
+    JSON emits an ARRAY OF ROW OBJECTS, not a columns/rows pair: an agent
+    parsing the result should not have to zip two lists to read a field. CSV
+    emits a header + rows; TABLE a width-aligned plain-text grid with a
+    trailing row count. Results whose size is not known to be small belong in
+    :func:`emit_cursor`.
+    """
+    emit_row_batches(columns, _batched(rows, _STREAM_BATCH_ROWS), fmt)
+
+
+def emit_cursor(cursor: Any, fmt: OutputFormat | str = OutputFormat.AUTO) -> None:
+    """Write a DuckDB cursor's whole result to stdout without materializing it.
+
+    Reads ``cursor.description`` for the column names, then drains
+    ``cursor.fetchmany`` in :data:`_STREAM_BATCH_ROWS` batches through
+    :func:`emit_row_batches`. Duck-typed on purpose: this module stays off
+    the ``import duckdb`` path.
+
+    Output is written as batches arrive, so a driver error raised partway
+    through the scan surfaces AFTER some rows are already on stdout.
+    """
+    columns = [d[0] for d in cursor.description or ()]
+
+    def batches() -> Iterator[Sequence[Sequence[Any]]]:
+        while True:
+            batch = cursor.fetchmany(_STREAM_BATCH_ROWS)
+            if not batch:
+                return
+            yield batch
+
+    emit_row_batches(columns, batches(), fmt)
+
+
+def emit_row_batches(
+    columns: Sequence[str],
+    batches: Iterable[Sequence[Sequence[Any]]],
+    fmt: OutputFormat | str = OutputFormat.AUTO,
+) -> None:
+    """Write successive row batches to stdout in the requested format.
+
+    Each batch is serialized and written before the next is pulled, so peak
+    memory is one batch rather than the whole result. The JSON array
+    brackets and commas, the CSV header, and the TABLE header/rule/row-count
+    are emitted around the stream.
+    """
+    resolved = resolve_format(fmt)
+    if resolved is OutputFormat.JSON:
+        _write_json_stream(columns, batches)
+        return
+    if resolved is OutputFormat.CSV:
+        writer = csv.writer(sys.stdout)
+        writer.writerow(columns)
+        for batch in batches:
+            writer.writerows(batch)
+        return
+    _write_table_stream(columns, batches)
+
+
+def _write_json_stream(
+    columns: Sequence[str],
+    batches: Iterable[Sequence[Sequence[Any]]],
+) -> None:
+    """Emit a JSON array of row objects, one batch of elements at a time."""
+    keys = json_object_keys(columns)
+    sys.stdout.write("[")
+    first = True
+    for batch in batches:
+        for row in batch:
+            if not first:
+                sys.stdout.write(", ")
+            first = False
+            sys.stdout.write(json.dumps(dict(zip(keys, row, strict=True)), default=str))
+    sys.stdout.write("]\n")
+
+
+def _write_table_stream(
+    columns: Sequence[str],
+    batches: Iterable[Sequence[Sequence[Any]]],
+) -> None:
+    """Emit a width-aligned plain-text grid, sizing columns off the first batch.
+
+    Column widths are fixed by the header plus the first batch, which is the
+    widest window available without holding the whole result. A later batch
+    with a wider cell overflows its column rather than being truncated —
+    misalignment is preferable to hiding a value.
+    """
+    stream = iter(batches)
+    head = [[_cell(v) for v in row] for row in next(stream, ())]
+    widths = [
+        max(len(str(col)), *(len(r[i]) for r in head)) if head else len(str(col))
+        for i, col in enumerate(columns)
+    ]
+
+    def line(cells: Sequence[str]) -> str:
+        return " | ".join(cell.ljust(w) for cell, w in zip(cells, widths, strict=True))
+
+    sys.stdout.write(line([str(col) for col in columns]) + "\n")
+    sys.stdout.write("-+-".join("-" * w for w in widths) + "\n")
+    total = 0
+    for row in head:
+        sys.stdout.write(line(row) + "\n")
+        total += 1
+    for batch in stream:
+        for raw_row in batch:
+            sys.stdout.write(line([_cell(v) for v in raw_row]) + "\n")
+            total += 1
+    sys.stdout.write(f"({total} row{'s' if total != 1 else ''})\n")
+
+
+def emit_json(payload: Any, fmt: OutputFormat | str = OutputFormat.AUTO) -> None:
+    """Write a non-tabular payload as pretty JSON (schema, status, reports).
+
+    Every format reduces to one JSON document here, because CSV over a nested
+    dict has no meaning. ``fmt`` is accepted for call-site symmetry with
+    :func:`emit_rows` and is never branched on.
+    Callers that want a bespoke human layout handle the TABLE branch
+    themselves before calling this.
+    """
+    del fmt
+    sys.stdout.write(json.dumps(payload, indent=2, default=str))
+    sys.stdout.write("\n")
+
+
+def emit_error(err: ClassifiedError, fmt: OutputFormat | str = OutputFormat.AUTO) -> None:
+    """Write a classified error to stderr in the requested format.
+
+    Agents running ``--format json`` (or piping) get the structured
+    envelope; humans on a TTY get a readable line plus the hint. The caller
+    exits with :attr:`ClassifiedError.exit_code`.
+    """
+    resolved = resolve_format(fmt)
+    if resolved is OutputFormat.TABLE:
+        sys.stderr.write(f"[{err.kind}] {err.message}\n")
+        if err.hint:
+            sys.stderr.write(f"hint: {err.hint}\n")
+    else:
+        sys.stderr.write(json.dumps(err.to_payload(), default=str))
+        sys.stderr.write("\n")
+
+
+__all__ = [
+    "OutputFormat",
+    "emit_cursor",
+    "emit_error",
+    "emit_json",
+    "emit_row_batches",
+    "emit_rows",
+    "json_object_keys",
+    "resolve_format",
+]

atif_converter/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""atif-converter: Claude Code -> ATIF conversion with an explicit fidelity policy."""

atif_converter/application/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Application layer: use cases composing census + harbor adapter."""

atif_converter/application/convert_and_audit.py

@@ -0,0 +1,294 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Use case: convert one session AND account for what the conversion lost.
+
+Composes the raw-side census (:mod:`atif_converter.infrastructure.census`)
+with the harbor adapter
+(:mod:`atif_converter.infrastructure.harbor_adapter`) into a
+``(ConversionResult, LossReport)`` pair, the honest answer to "what did we
+just materialize, and what did upstream drop on the floor?".
+
+ONE READ. The session's files are read once, by
+:func:`~atif_converter.infrastructure.harbor_adapter.read_session`, which
+fingerprints and parses each file in the same pass. The converter, the census,
+the edges emitter and the enrichment pass all consume that one list of
+records, so the trajectory, the loss report and ``edges.jsonl`` describe the
+same bytes by construction. Before this, the converter and the audit each
+parsed the files and the snapshot hashed them three times over.
+
+SNAPSHOT DISCIPLINE: the fingerprints taken by that read are re-checked once
+the artifacts are built. A Claude Code session that resumes writing during
+the read, or anywhere before the check, fails the session: a stat pair alone
+would miss a same-length rewrite inside one mtime tick, so the re-check hashes
+the bytes again. That second hash pass is the one read the audit still pays
+beyond the parse.
+
+ATTACHMENTS AND TYPED SIGNALS. Before conversion, every inline base64
+attachment in the loaded records is replaced by a placeholder and its bytes
+collected (:mod:`atif_converter.domain.blobs`), so neither the trajectory nor
+the loss report nor the edges ever carry base64. After enrichment,
+:mod:`atif_converter.domain.result_signals` writes the typed tool-outcome,
+attachment and subagent fields, the subagent links read from the same pass's
+``agent-*.meta.json`` sidecars.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from atif_converter.domain.blobs import BlobCollector, extract_claude_code_blobs
+from atif_converter.domain.edges import edges_jsonl_lines
+from atif_converter.domain.enrichment import enrich_trajectory
+from atif_converter.domain.errors import SourceMutatedDuringConversion
+from atif_converter.domain.fidelity import (
+    CONVERTIBLE_RECORD_TYPES,
+    AnyRecordType,
+    FidelityGap,
+    LossReport,
+)
+from atif_converter.domain.result_signals import annotate_claude_code_trajectory
+from atif_converter.domain.session_events import (
+    claude_reported_cost,
+    claude_session_events,
+    session_events_jsonl_lines,
+)
+from atif_converter.infrastructure.census import SessionCensus, census_from_snapshot
+from atif_converter.infrastructure.harbor_adapter import (
+    ConversionResult,
+    convert_loaded_session,
+    read_session,
+    validate_trajectory,
+)
+from atif_converter.infrastructure.raw_records import (
+    LoadedSession,
+    SessionSnapshot,
+    mutated_files,
+)
+from atif_converter.infrastructure.source_archive import SourceArchiveWriter
+
+#: ``agent-<id>.meta.json``: the sidecar's name carries the subagent id.
+_SIDECAR_PREFIX = "agent-"
+_SIDECAR_SUFFIX = ".meta.json"
+
+#: Gaps every conversion exhibits regardless of session content. The parent
+#: chain is flattened into a timestamp-sorted step list, and nothing after
+#: conversion rebuilds it inside the trajectory (``edges.jsonl`` keeps it).
+_STRUCTURAL_GAPS: frozenset[FidelityGap] = frozenset({FidelityGap.PARENT_CHAIN_FLATTENED})
+
+_ENRICHMENT_REFUSAL_KEYS: tuple[str, ...] = (
+    "enrichment_unattributed_steps",
+    "enrichment_truncated_at_step",
+)
+
+
+def _attributed_uuids(trajectory: dict[str, Any]) -> set[str]:
+    uuids: set[str] = set()
+    for step in trajectory.get("steps") or []:
+        extra = step.get("extra") if isinstance(step, dict) else None
+        source_uuids = extra.get("source_uuids") if isinstance(extra, dict) else None
+        if isinstance(source_uuids, list):
+            uuids.update(uuid for uuid in source_uuids if isinstance(uuid, str))
+    return uuids
+
+
+def _has_cache_creation_usage(records: list[dict[str, Any]]) -> bool:
+    for record in records:
+        if record.get("type") != "assistant":
+            continue
+        message = record.get("message")
+        usage = message.get("usage") if isinstance(message, dict) else None
+        if isinstance(usage, dict) and isinstance(usage.get("cache_creation_input_tokens"), int):
+            return True
+    return False
+
+
+def _unrepaired_gaps(enriched: dict[str, Any], records: list[dict[str, Any]]) -> set[FidelityGap]:
+    """The three gaps enrichment repairs, reported only where it DIDN'T.
+
+    * ``uuid_not_preserved``: enrichment refused to attribute some step
+      (an unattributed-steps count or a truncation marker on the trajectory).
+    * ``compact_summary_unhandled``: a compaction-summary record exists and
+      its uuid landed in no step's ``source_uuids``, so no step carries the
+      ``is_compact_summary`` flag for it.
+    * ``cache_split_partial``: assistant usage carries
+      ``cache_creation_input_tokens`` but the trajectory has no
+      ``cache_creation_total``.
+    """
+    gaps: set[FidelityGap] = set()
+    extra = enriched.get("extra")
+    extra = extra if isinstance(extra, dict) else {}
+    if any(extra.get(key) is not None for key in _ENRICHMENT_REFUSAL_KEYS):
+        gaps.add(FidelityGap.UUID_NOT_PRESERVED)
+    compact_uuids = {
+        record["uuid"]
+        for record in records
+        if record.get("isCompactSummary") and isinstance(record.get("uuid"), str)
+    }
+    if compact_uuids and not compact_uuids <= _attributed_uuids(enriched):
+        gaps.add(FidelityGap.COMPACT_SUMMARY_UNHANDLED)
+    if _has_cache_creation_usage(records) and "cache_creation_total" not in extra:
+        gaps.add(FidelityGap.CACHE_SPLIT_PARTIAL)
+    return gaps
+
+
+def _loss_report(
+    census: SessionCensus,
+    *,
+    captured: int,
+    unrepaired: set[FidelityGap],
+) -> LossReport:
+    converted = sum(
+        count
+        for record_type, count in census.record_counts.items()
+        if record_type in CONVERTIBLE_RECORD_TYPES
+    )
+    total = sum(census.record_counts.values())
+    dropped = total - converted - captured
+
+    gaps = set(_STRUCTURAL_GAPS) | unrepaired
+    if dropped:
+        gaps.add(FidelityGap.NON_MESSAGE_RECORDS_DROPPED)
+    if census.subagent_files:
+        gaps.add(FidelityGap.SUBAGENTS_INLINED)
+    # Named rather than inlined: LossReport is shared by both agents, so its
+    # key type is the UNION of the two record taxonomies and a same-typed dict
+    # of one agent's members does not match the constructor overload without
+    # being widened here.
+    record_counts: dict[AnyRecordType, int] = {**census.record_counts}
+    return LossReport(
+        record_counts=record_counts,
+        records_converted=converted,
+        records_captured=captured,
+        records_dropped=dropped,
+        gaps_observed=frozenset(gaps),
+        subagent_files_found=len(census.subagent_files) + len(census.workflow_subagent_files),
+        # Our converter discovers every side file itself, workflow-nested ones
+        # included, so all of them are convertible.
+        subagent_files_convertible=len(census.subagent_files) + len(census.workflow_subagent_files),
+        workflow_subagent_files_found=len(census.workflow_subagent_files),
+    )
+
+
+def _stamp_reported_cost(enriched: dict[str, Any], reported: dict[str, Any]) -> None:
+    """Put Claude Code's own cost figure beside the computed estimate."""
+    if not reported:
+        return
+    final_metrics = enriched.setdefault("final_metrics", {})
+    extra = final_metrics.get("extra")
+    if not isinstance(extra, dict):
+        extra = {}
+        final_metrics["extra"] = extra
+    extra.update(reported)
+
+
+def _sidecars_by_agent_id(loaded: LoadedSession) -> dict[str, Any]:
+    """Parsed ``agent-<id>.meta.json`` sidecars keyed by ``<id>``; first path wins."""
+    by_agent: dict[str, Any] = {}
+    for path, value in loaded.sidecars.items():
+        name = path.name
+        if name.startswith(_SIDECAR_PREFIX) and name.endswith(_SIDECAR_SUFFIX):
+            agent_id = name[len(_SIDECAR_PREFIX) : -len(_SIDECAR_SUFFIX)]
+            if agent_id:
+                by_agent.setdefault(agent_id, value)
+    return by_agent
+
+
+def _refuse_if_mutated(
+    snapshot: SessionSnapshot, archive: SourceArchiveWriter | None = None
+) -> None:
+    mutated = mutated_files(snapshot, archive=archive)
+    if not mutated:
+        if archive is not None:
+            # The verifying read already archived the transcripts AND the
+            # parsed sidecars; the sweep takes only files nothing parsed.
+            archive.archive_side_files(
+                snapshot.session_jsonl,
+                already=(*snapshot.files, *snapshot.sidecar_fingerprints),
+            )
+        return
+    logger.warning(
+        "convert_and_audit: {} source file(s) changed while converting {}; refusing",
+        len(mutated),
+        snapshot.session_jsonl,
+    )
+    raise SourceMutatedDuringConversion(snapshot.session_jsonl, mutated)
+
+
+def convert_and_audit(
+    session_jsonl: Path,
+    *,
+    include_subagents: bool = True,
+    archive: SourceArchiveWriter | None = None,
+) -> tuple[ConversionResult, LossReport]:
+    """Convert ``session_jsonl`` to ATIF and produce its loss accounting.
+
+    The returned :class:`ConversionResult` carries the ENRICHED trajectory
+    (``source_uuids`` / ``is_compact_summary`` on steps,
+    ``cache_creation_total`` on the trajectory, see
+    :func:`~atif_converter.domain.enrichment.enrich_trajectory`) plus the
+    ready-to-write ``edges.jsonl`` lines derived from the RAW records, per
+    the corpus contract. Validation is re-run post-enrichment.
+
+    The census counts EVERY raw record, so ``LossReport.records_dropped``
+    is an upper bound on what the materialized trajectory is missing.
+
+    Note: ``records_converted`` counts raw user/assistant RECORDS, not ATIF
+    steps. harbor legitimately bundles several assistant events (one
+    ``message.id``) into a single agent step, so the two numbers differ by
+    design.
+
+    With ``archive``, the verifying re-read also streams every source file
+    into that writer (see
+    :mod:`atif_converter.infrastructure.source_archive`): the archive costs
+    no extra open of a transcript and holds exactly the bytes that were
+    parsed. It is complete only when this function returns; on any raise the
+    caller must discard it.
+
+    Raises
+    ------
+        InvalidSessionInput: ``session_jsonl`` is not an existing ``.jsonl`` file.
+        SourceMutatedDuringConversion: a source file changed anywhere between
+            the read and the end of the audit, so the artifacts would not
+            describe the session as it stands.
+    """
+    loaded = read_session(session_jsonl)
+    snapshot = loaded.snapshot
+    collector = BlobCollector()
+    blob_index = extract_claude_code_blobs(
+        (record for records in loaded.records_by_file.values() for record in records),
+        collector,
+    )
+    result = convert_loaded_session(loaded, include_subagents=include_subagents)
+
+    pairs = loaded.record_pairs()
+    census = census_from_snapshot(snapshot, pairs)
+    edges_lines = tuple(edges_jsonl_lines(pairs))
+    events = claude_session_events(pairs)
+    events_lines = tuple(session_events_jsonl_lines(events))
+    records = [record for record, _src in pairs]
+    # The harbor trajectory has no other consumer, so enrich in place rather
+    # than deep-copying a whole large transcript's worth of steps.
+    enriched = enrich_trajectory(result.trajectory, records, copy_input=False)
+    annotate_claude_code_trajectory(
+        enriched, records, blob_index, sidecars=_sidecars_by_agent_id(loaded)
+    )
+    _stamp_reported_cost(enriched, claude_reported_cost(pairs))
+    report = _loss_report(
+        census,
+        captured=len(events),
+        unrepaired=_unrepaired_gaps(enriched, records),
+    )
+    del pairs, loaded, records, events
+    _refuse_if_mutated(snapshot, archive)
+
+    enriched_result = ConversionResult(
+        trajectory=enriched,
+        validation_errors=validate_trajectory(enriched),
+        edges_lines=edges_lines,
+        events_lines=events_lines,
+        blobs=collector.blobs,
+    )
+    return enriched_result, report