atif-sql 0.2.0__py3-none-any.whl

atif_converter/infrastructure/raw_records.py

@@ -0,0 +1,426 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Read the raw records of a session, main JSONL plus every side-file, in one pass.
+
+The single raw-record reader the converter, the census, the edges emitter and
+the enrichment pass share. Discovery mirrors the contract: rglob over the
+session's sibling dir filtered to ``*.jsonl`` (``*.meta.json`` excluded by the
+suffix filter), which includes workflow-nested
+``subagents/workflows/wf_*/agent-*.jsonl``. File reads only: no harbor, no
+env, no network.
+
+ONE READ, ONE PARSE. :func:`load_session` opens each discovered file once and
+streams its bytes through a digest on the way to the JSON parser, so the
+fingerprint it records is the fingerprint of exactly the bytes that were
+parsed. The parsed records are handed to the converter AND to the audit
+(census, edges, enrichment), so the three artifacts cannot describe different
+bytes: they come from one list. Before this pass existed the converter and the
+audit each parsed the files and the snapshot hashed them separately, three
+hash passes and two parses of a 135 MB session.
+
+SIDECARS. Claude Code writes one ``agent-<id>.meta.json`` beside each subagent
+transcript, recording the subagent's type, its description and the
+``toolUseId`` of the Task/Agent call that spawned it. They are not JSONL and
+carry no records, so they stay out of :attr:`LoadedSession.records_by_file`
+(the census and edges count records, and a sidecar is not one). The same pass
+reads each one once, fingerprints it, and parses it into
+:attr:`LoadedSession.sidecars`, and the snapshot re-check covers them too, so a
+sidecar rewritten mid-conversion refuses the session exactly as a transcript
+would.
+
+A :class:`SessionSnapshot` names a fixed set of source files and carries the
+change evidence for each, an mtime/size/content-digest fingerprint, and
+NOTHING else. Claude Code appends to a live session at any moment, so the
+snapshot is re-checked with :func:`mutated_files` once the artifacts are
+built; an unchanged fingerprint set is the evidence that the session held
+still for the whole window, including while it was being read. The digest is
+what catches a same-length rewrite inside one mtime tick, so the re-check
+hashes the bytes again rather than trusting the stat pair.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, NamedTuple, Protocol, override
+
+from loguru import logger
+
+if TYPE_CHECKING:
+    from collections.abc import Buffer, Callable, Iterable
+
+    from atif_converter.infrastructure.source_archive import SourceArchiveWriter
+
+
+def discover_session_files(session_jsonl: Path) -> list[Path]:
+    """The main session JSONL plus every ``*.jsonl`` under its sibling dir.
+
+    Deterministic order: main file first, then sorted side-files.
+    """
+    files = [session_jsonl]
+    side_dir = session_jsonl.parent / session_jsonl.stem
+    if side_dir.is_dir():
+        files.extend(sorted(side_dir.rglob("*.jsonl")))
+    return files
+
+
+def discover_sidecar_files(session_jsonl: Path) -> list[Path]:
+    """Every subagent ``agent-*.meta.json`` under the session's sibling dir, sorted.
+
+    Workflow-nested sidecars (``subagents/workflows/wf_*/agent-*.meta.json``)
+    included, as the transcripts they describe are.
+    """
+    side_dir = session_jsonl.parent / session_jsonl.stem
+    if not side_dir.is_dir():
+        return []
+    return sorted(side_dir.rglob("agent-*.meta.json"))
+
+
+class FileFingerprint(NamedTuple):
+    """Change evidence for one source file: mtime, size, and content digest.
+
+    Each field covers a case the others miss.
+
+    Size catches the append that mtime misses: two appends inside one
+    filesystem timestamp tick share an ``st_mtime_ns``, and a session resuming
+    mid-conversion is exactly that fast, but an append always changes size.
+
+    mtime catches most in-place edits that leave the length alone.
+
+    The digest catches what neither does: a same-length rewrite landing inside
+    one mtime tick. That is not hypothetical here: on the xfs volume this runs
+    on, 45 consecutive same-size rewrites were absorbed into a single
+    ``st_mtime_ns``, so ``{"uuid": "aaa"}`` becoming ``{"uuid": "bbb"}`` is
+    invisible to the stat pair. Different bytes behind an identical stat is
+    precisely the state this snapshot exists to refuse, so the bytes are read.
+    """
+
+    mtime_ns: int
+    size: int
+    content_hash: bytes
+
+
+class _Digest(Protocol):
+    """The two methods of a ``hashlib`` object this module uses."""
+
+    def update(self, data: Buffer, /) -> None: ...
+
+    def digest(self) -> bytes: ...
+
+
+def _new_digest() -> _Digest:
+    """A fresh content digest.
+
+    sha256 rather than blake2b: the digest is never persisted or compared
+    across hosts, so only its speed matters, and sha256 has hardware
+    instructions on current x86 (SHA-NI) and arm64 (SHA2) while blake2b does
+    not. Measured on the 76 MB benchmark session: 1.6 GB/s against 0.9 GB/s.
+    """
+    return hashlib.sha256()
+
+
+#: Digest chunk size; the transcripts are streamed rather than read whole.
+_HASH_CHUNK_BYTES = 1024 * 1024
+
+
+class _DigestingReader(io.RawIOBase):
+    """A raw byte stream that folds every byte it hands out into a digest.
+
+    Sits between the file and the text decoder so the one read that feeds the
+    JSON parser also produces the content fingerprint. The digest is complete
+    once the reader has been drained, which iterating the wrapping text handle
+    to its end guarantees.
+    """
+
+    def __init__(self, raw: io.RawIOBase, digest: _Digest) -> None:
+        super().__init__()
+        self._raw = raw
+        self._digest = digest
+
+    @override
+    def readable(self) -> bool:
+        return True
+
+    @override
+    def readinto(self, buffer: Buffer, /) -> int | None:
+        count = self._raw.readinto(buffer)
+        if count is None:
+            return None
+        if count:
+            self._digest.update(memoryview(buffer)[:count])
+        return int(count)
+
+    @override
+    def close(self) -> None:
+        self._raw.close()
+        super().close()
+
+
+def _content_hash(path: Path, tee: Callable[[bytes], object] | None = None) -> bytes:
+    """Digest of ``path``'s bytes; each chunk also goes to ``tee`` when one is given."""
+    digest = _new_digest()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(_HASH_CHUNK_BYTES), b""):
+            digest.update(chunk)
+            if tee is not None:
+                tee(chunk)
+    return digest.digest()
+
+
+def _fingerprint(path: Path, tee: Callable[[bytes], object] | None = None) -> FileFingerprint:
+    stat = path.stat()
+    return FileFingerprint(
+        mtime_ns=stat.st_mtime_ns,
+        size=stat.st_size,
+        content_hash=_content_hash(path, tee),
+    )
+
+
+def parse_jsonl_records(lines: Iterable[str], path: Path) -> list[Any]:
+    """Parse JSONL lines the way harbor's reading loop does.
+
+    Blank lines are skipped; a line that is not JSON is logged at debug and
+    skipped rather than failing the session. A line that IS JSON but not an
+    object is kept as-is, as harbor keeps it (the converter fails on it later,
+    in the same place harbor fails); the audit readers filter to objects.
+
+    Iterating a text handle splits on newlines only, where ``str.splitlines``
+    would also split on U+2028 / U+2029 / form feed, which are legal unescaped
+    inside a JSON string. Callers pass the handle, never ``read().splitlines()``.
+    """
+    records: list[Any] = []
+    for line in lines:
+        stripped = line.strip()
+        if not stripped:
+            continue
+        try:
+            records.append(json.loads(stripped))
+        except json.JSONDecodeError as exc:
+            logger.debug("Skipping malformed JSONL line in {}: {}", path, exc)
+    return records
+
+
+def _read_and_fingerprint(path: Path) -> tuple[FileFingerprint, list[Any]]:
+    """One pass over ``path``: stat, then hash and parse the same bytes.
+
+    The stat is taken BEFORE the read so an append that lands between the two
+    still shows as movement when the post-conversion re-check compares stats;
+    the digest covers the bytes the parser actually saw.
+    """
+    stat = path.stat()
+    digest = _new_digest()
+    with (
+        path.open("rb", buffering=0) as raw,
+        io.TextIOWrapper(
+            io.BufferedReader(_DigestingReader(raw, digest), _HASH_CHUNK_BYTES),
+            encoding="utf-8",
+        ) as handle,
+    ):
+        records = parse_jsonl_records(handle, path)
+    fingerprint = FileFingerprint(
+        mtime_ns=stat.st_mtime_ns,
+        size=stat.st_size,
+        content_hash=digest.digest(),
+    )
+    return fingerprint, records
+
+
+def _read_sidecar(path: Path) -> tuple[FileFingerprint, Any]:
+    """One pass over a small JSON sidecar: stat, then hash and parse the same bytes.
+
+    A sidecar that is not valid UTF-8 JSON parses to ``None`` (logged at debug)
+    rather than failing the session: it only ever adds linkage, and the
+    subagent's transcript converts without it.
+    """
+    stat = path.stat()
+    data = path.read_bytes()
+    digest = _new_digest()
+    digest.update(data)
+    try:
+        value: Any = json.loads(data.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        logger.debug("Skipping unreadable sidecar {}: {}", path, exc)
+        value = None
+    fingerprint = FileFingerprint(
+        mtime_ns=stat.st_mtime_ns,
+        size=stat.st_size,
+        content_hash=digest.digest(),
+    )
+    return fingerprint, value
+
+
+@dataclass(frozen=True, slots=True)
+class SessionSnapshot:
+    """The fixed file list of one session plus each file's change evidence.
+
+    Holds no records. A :class:`LoadedSession` pairs one with the records
+    parsed from those same bytes; :func:`read_snapshot_records` re-parses
+    exactly the snapshot's file list for a caller that holds only the snapshot.
+    """
+
+    session_jsonl: Path
+
+    #: Fingerprint per discovered file, sampled by the read that parsed it (or
+    #: by :func:`take_session_snapshot`, which reads for the digest alone).
+    fingerprints: dict[Path, FileFingerprint]
+
+    #: Fingerprint per discovered ``agent-*.meta.json`` sidecar. Kept apart
+    #: from :attr:`fingerprints` because :attr:`files` is the RECORD-bearing
+    #: file list every JSONL reader walks, and a sidecar holds no records.
+    sidecar_fingerprints: dict[Path, FileFingerprint] = field(default_factory=dict)
+
+    @property
+    def files(self) -> tuple[Path, ...]:
+        """Every discovered file, main first then sorted side-files."""
+        return tuple(self.fingerprints)
+
+
+@dataclass(frozen=True, slots=True)
+class LoadedSession:
+    """One session's snapshot plus every record parsed from those same bytes.
+
+    ``records_by_file`` keeps each file's records in file order under the
+    file's path, main first then sorted side-files, so the converter can
+    address the main transcript and the staged side-files by name while the
+    audit walks every record of every file. Values are whatever JSON the lines
+    held, objects or not, as harbor keeps them; :meth:`record_pairs` is the
+    objects-only view the census, edges and enrichment consume.
+    """
+
+    snapshot: SessionSnapshot
+    records_by_file: dict[Path, list[Any]]
+
+    #: Each ``agent-*.meta.json`` sidecar's parsed JSON (``None`` when it did
+    #: not parse), keyed by path, sorted. Read in the same pass as the records.
+    sidecars: dict[Path, Any] = field(default_factory=dict)
+
+    @property
+    def main_records(self) -> list[Any]:
+        """The main transcript's records, in file order."""
+        return self.records_by_file[self.snapshot.session_jsonl]
+
+    def record_pairs(self) -> list[tuple[dict[str, Any], str]]:
+        """Every OBJECT record with its source file, in file order.
+
+        ``source_file`` is relative to the session JSONL's parent directory,
+        the contract's edge field. Non-object JSON values are dropped here, as
+        the audit never counted them.
+        """
+        base = self.snapshot.session_jsonl.parent
+        pairs: list[tuple[dict[str, Any], str]] = []
+        for path, records in self.records_by_file.items():
+            source_file = path.relative_to(base).as_posix()
+            pairs.extend((record, source_file) for record in records if isinstance(record, dict))
+        return pairs
+
+
+def load_session(session_jsonl: Path) -> LoadedSession:
+    """Read every discovered file once: fingerprint and parse in a single pass.
+
+    Raises
+    ------
+        OSError: a discovered file cannot be opened or read.
+        UnicodeDecodeError: a file is not UTF-8.
+    """
+    fingerprints: dict[Path, FileFingerprint] = {}
+    records_by_file: dict[Path, list[Any]] = {}
+    for path in discover_session_files(session_jsonl):
+        fingerprints[path], records_by_file[path] = _read_and_fingerprint(path)
+    sidecar_fingerprints: dict[Path, FileFingerprint] = {}
+    sidecars: dict[Path, Any] = {}
+    for path in discover_sidecar_files(session_jsonl):
+        sidecar_fingerprints[path], sidecars[path] = _read_sidecar(path)
+    return LoadedSession(
+        snapshot=SessionSnapshot(
+            session_jsonl=session_jsonl,
+            fingerprints=fingerprints,
+            sidecar_fingerprints=sidecar_fingerprints,
+        ),
+        records_by_file=records_by_file,
+        sidecars=sidecars,
+    )
+
+
+def take_session_snapshot(session_jsonl: Path) -> SessionSnapshot:
+    """Fingerprint every discovered file; holds digests, never parsed records.
+
+    The use cases go through :func:`load_session`, which fingerprints as a
+    by-product of the read. This is for a caller that wants the change
+    evidence alone.
+    """
+    return SessionSnapshot(
+        session_jsonl=session_jsonl,
+        fingerprints={path: _fingerprint(path) for path in discover_session_files(session_jsonl)},
+        sidecar_fingerprints={
+            path: _fingerprint(path) for path in discover_sidecar_files(session_jsonl)
+        },
+    )
+
+
+def read_snapshot_records(snapshot: SessionSnapshot) -> list[tuple[dict[str, Any], str]]:
+    """Parse every object record of exactly the snapshot's file list.
+
+    Returns ``(record, source_file)`` pairs in file order, ``source_file``
+    relative to the session JSONL's parent directory (the contract's edge
+    field). Malformed lines are skipped (same policy as harbor itself). Files
+    added since the snapshot are NOT read: the snapshot's list is the whole
+    input, and :func:`mutated_files` reports the newcomer afterwards.
+    """
+    base = snapshot.session_jsonl.parent
+    pairs: list[tuple[dict[str, Any], str]] = []
+    for path in snapshot.files:
+        source_file = path.relative_to(base).as_posix()
+        with path.open(encoding="utf-8") as handle:
+            records = parse_jsonl_records(handle, path)
+        pairs.extend((record, source_file) for record in records if isinstance(record, dict))
+    return pairs
+
+
+def mutated_files(
+    snapshot: SessionSnapshot,
+    *,
+    archive: SourceArchiveWriter | None = None,
+) -> tuple[Path, ...]:
+    """Files that changed, vanished, or appeared since the snapshot was taken.
+
+    Re-stats AND re-hashes every file: the digest is the half of the evidence
+    that catches a same-length rewrite inside one mtime tick. A newly APPEARED
+    side-file counts: it carries records the snapshot never saw, so artifacts
+    derived from the snapshot are already incomplete.
+
+    With ``archive``, every byte this re-hash reads is also streamed into the
+    session's source archive, so archiving costs no extra open. The archive is
+    only meaningful when the result is empty: an unchanged digest is what proves
+    the archived bytes are the bytes that were parsed. A caller that gets a
+    non-empty result must discard it.
+    """
+    moved: list[Path] = []
+    for path, fingerprint in (
+        *snapshot.fingerprints.items(),
+        *snapshot.sidecar_fingerprints.items(),
+    ):
+        try:
+            if archive is None:
+                current = _fingerprint(path)
+            else:
+                with archive.member(path) as sink:
+                    current = _fingerprint(path, sink)
+                archive.record(path, size=current.size, sha256=current.content_hash)
+            if current != fingerprint:
+                moved.append(path)
+        except OSError:
+            moved.append(path)
+    moved.extend(
+        path
+        for path in discover_session_files(snapshot.session_jsonl)
+        if path not in snapshot.fingerprints
+    )
+    moved.extend(
+        path
+        for path in discover_sidecar_files(snapshot.session_jsonl)
+        if path not in snapshot.sidecar_fingerprints
+    )
+    return tuple(moved)

atif_converter/infrastructure/source_archive.py

@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Write a zstd-compressed copy of a session's raw source files while they are verified.
+
+The corpus keeps every session's artifacts after the source transcript is gone,
+and a future converter can only re-run on a session whose raw bytes survived.
+So each successful conversion also leaves an ARCHIVE: one ``<relative path>.zst``
+per source file, relative to the main transcript's parent directory, in a
+directory the caller names.
+
+WHERE THE BYTES COME FROM. The transcripts are read once for parsing (see
+:mod:`atif_converter.infrastructure.raw_records`), and the second, verifying
+read re-hashes every byte to prove nothing moved. The archive rides that second
+read: :func:`~atif_converter.infrastructure.raw_records.mutated_files` hands each
+chunk it hashes to :meth:`SourceArchiveWriter.member` as well. No file is opened
+a third time, and the digest that vouches for the archived bytes is the one the
+snapshot check compares against the parse's digest, so an archived transcript is
+byte-for-byte the input the artifacts were built from. If anything moved, the
+conversion raises and the caller throws the half-written archive away.
+
+The ``agent-*.meta.json`` sidecars are parsed in the same pass as the
+transcripts and re-hashed by the same verifying read, so they ride it too. The
+other files under the session's side directory (the ``tool-results/`` spill
+files, workflow state) are not parsed by any converter today, so
+:meth:`SourceArchiveWriter.archive_side_files` reads each of them once, here.
+They are archived anyway because a future converter may want them and the
+originals expire with the transcript.
+
+Level 3 is deliberate: measured on a 76 MB transcript, level 3 compresses at
+about 300 MB/s to 2.8x and level 19 reaches 2.9x at 5 MB/s. The archive sits on
+the conversion hot path, so the extra 3% is not worth sixty times the CPU.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, BinaryIO
+
+import zstandard
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Collection, Generator
+    from pathlib import Path
+
+#: zstd level for every archived file; see the module docstring.
+ARCHIVE_ZSTD_LEVEL = 3
+
+#: Suffix appended to each archived file's relative path.
+ARCHIVE_SUFFIX = ".zst"
+
+#: Archive files hold raw transcript bytes, so they get the transcripts' own
+#: owner-only mode rather than whatever the umask allows.
+_ARCHIVE_FILE_MODE = 0o600
+
+#: Read size for side files, the same 1 MiB chunk the transcript reader uses.
+_CHUNK_BYTES = 1024 * 1024
+
+
+@dataclass(frozen=True, slots=True)
+class ArchivedSourceFile:
+    """One archived file: where it came from, and what its uncompressed bytes were."""
+
+    #: POSIX path relative to the main transcript's parent directory.
+    relative_path: str
+    #: Uncompressed size in bytes.
+    size: int
+    #: Hex sha256 of the uncompressed bytes.
+    sha256: str
+
+
+class SourceArchiveWriter:
+    """Collects one session's archive under ``archive_dir``.
+
+    ``base`` is the main transcript's parent directory; every member's name is
+    its path relative to ``base``, so the archive restores to the same shape
+    (``<session>.jsonl`` beside ``<session>/subagents/...``) the converter
+    discovers side files from.
+    """
+
+    def __init__(self, archive_dir: Path, *, base: Path) -> None:
+        self.archive_dir = archive_dir
+        self.base = base
+        self._files: dict[str, ArchivedSourceFile] = {}
+
+    @property
+    def files(self) -> tuple[ArchivedSourceFile, ...]:
+        """Every archived file, sorted by relative path."""
+        return tuple(self._files[name] for name in sorted(self._files))
+
+    def _relative(self, path: Path) -> str:
+        return path.relative_to(self.base).as_posix()
+
+    def _open_member(self, relative_path: str) -> BinaryIO:
+        target = self.archive_dir / f"{relative_path}{ARCHIVE_SUFFIX}"
+        target.parent.mkdir(parents=True, exist_ok=True)
+        fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _ARCHIVE_FILE_MODE)
+        return os.fdopen(fd, "wb")
+
+    @contextmanager
+    def member(self, path: Path) -> Generator[Callable[[bytes], object]]:
+        """Stream one source file into the archive; yields the chunk sink.
+
+        The caller hashes the same chunks itself and reports the digest through
+        :meth:`record`, which is how the verifying read's digest, not a second
+        one, vouches for the archived bytes.
+        """
+        with (
+            self._open_member(self._relative(path)) as raw,
+            zstandard.ZstdCompressor(level=ARCHIVE_ZSTD_LEVEL).stream_writer(
+                raw, closefd=False
+            ) as compressed,
+        ):
+            yield compressed.write
+
+    def record(self, path: Path, *, size: int, sha256: bytes) -> None:
+        """Note the uncompressed size and digest of a member written through :meth:`member`."""
+        relative = self._relative(path)
+        self._files[relative] = ArchivedSourceFile(
+            relative_path=relative, size=size, sha256=sha256.hex()
+        )
+
+    def archive_side_files(self, session_jsonl: Path, *, already: Collection[Path]) -> None:
+        """Archive every other regular file under the session's side directory.
+
+        ``already`` holds the files the verifying read archived (the
+        transcripts and the parsed sidecars); they are skipped. Symlinks are skipped too: following one would copy bytes from
+        outside the session into its archive.
+        """
+        side_dir = session_jsonl.parent / session_jsonl.stem
+        if not side_dir.is_dir() or side_dir.is_symlink():
+            return
+        skip = set(already)
+        for path in sorted(side_dir.rglob("*")):
+            if path in skip or path.is_symlink() or not path.is_file():
+                continue
+            digest = hashlib.sha256()
+            size = 0
+            with self.member(path) as sink, path.open("rb") as handle:
+                for chunk in iter(lambda: handle.read(_CHUNK_BYTES), b""):
+                    digest.update(chunk)
+                    sink(chunk)
+                    size += len(chunk)
+            self.record(path, size=size, sha256=digest.digest())
+
+
+__all__ = [
+    "ARCHIVE_SUFFIX",
+    "ARCHIVE_ZSTD_LEVEL",
+    "ArchivedSourceFile",
+    "SourceArchiveWriter",
+]

atif_corpus/__init__.py

@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""atif-corpus: corpus materialization for ATIF trajectories.
+
+Owns CONTRACT.md §Materialized corpus layout — session discovery over the
+raw transcript corpus, per-session watermarks (convert only what changed),
+quiescence detection (skip sessions still being written), and atomic
+artifact writes. Hexagonal: ``domain`` (pure decisions) < ``infrastructure``
+(fs/env effects) < ``application`` (the materialize use case). Conversion
+happens behind :class:`atif_corpus.domain.ports.ConverterPort`; atif-cli
+adapts the real converter (independence contract: this package never
+imports atif-converter or atif-duck).
+"""

atif_corpus/application/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Application layer: the materialize use case.
+
+Orchestrates domain decisions (plan) with infrastructure effects (scan,
+convert, write) behind the :class:`atif_corpus.domain.ports.ConverterPort`.
+"""