atif-sql 0.2.0__py3-none-any.whl

atif_corpus/domain/__init__.py

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Pure domain layer for corpus materialization.
+
+Value objects and pure functions only: no filesystem, no env, no clock.
+Every timestamp is passed IN (epoch nanoseconds from ``os.stat().st_mtime_ns``
+or ``time.time_ns()``) so the materialization decision is a deterministic
+function of its inputs — the property the plan-determinism tests pin.
+"""

atif_corpus/domain/agents.py

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Which agent produced a transcript — atif-corpus's copy of the one enum.
+
+DELIBERATE TWIN of :mod:`atif_converter.domain.agents`, and the same kind of
+twin as ``sql_literal`` and ``embedding_guard`` elsewhere in this workspace:
+the import-linter independence contract forbids atif-corpus from importing
+atif-converter, and the agent identity is needed on both sides of that
+boundary — the converter picks an adapter with it, the corpus picks a discovery
+layout and a corpus slug with it.
+
+``packages/atif-corpus/tests/test_agents_twin_pin.py`` reads the atif-converter
+copy as SOURCE TEXT (AST, never imported) and fails if the two disagree on a
+member name or a value, so the twin cannot drift silently.
+
+The string values are WIRE CONTRACT: the ``--agent`` flag's spellings, harbor's
+``Trajectory.agent.name``, and the ``agent`` key in ``meta.json``.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+
+class AgentSource(StrEnum):
+    """A coding agent whose transcripts this workspace can materialize."""
+
+    CLAUDE_CODE = "claude-code"
+    CODEX = "codex"
+
+
+#: The agent assumed when a caller names none.
+DEFAULT_AGENT: AgentSource = AgentSource.CLAUDE_CODE

atif_corpus/domain/generation.py

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Pure rule: does an artifact set's recorded provenance match what this pass writes.
+
+The watermark answers "did the SOURCE move?". It cannot answer "would this
+version of the code write the same artifacts?", so a converter upgrade or a
+columnar schema bump used to leave every existing session on the old output
+forever. The *generation* is the second question: a small mapping of
+``meta.json`` keys to the values this pass stamps (``converter_schema``, the
+converter's own output version, and ``columnar_schema`` when the columnar
+producer runs), and a session whose
+recorded values differ is stale even when no source byte changed.
+
+Only the EXPECTED keys are compared. A pass that does not write columnar
+artifacts expects no ``columnar_schema``, so it neither re-converts a session
+that has one nor one that lacks one.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+
+def generation_matches(recorded: Mapping[str, object], expected: Mapping[str, object]) -> bool:
+    """True when every expected key is recorded with exactly the expected value.
+
+    A missing key does not match: a session materialized before a key existed
+    (``columnar_schema`` on a ``--no-columnar`` session) is exactly the one the
+    new generation has to reach.
+    """
+    return all(key in recorded and recorded[key] == value for key, value in expected.items())

atif_corpus/domain/layout.py

@@ -0,0 +1,168 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""The contract's materialized-corpus directory shape, as one value object.
+
+CONTRACT.md §Materialized corpus layout is the ONLY specification of where
+artifacts live; atif-duck reads this same shape without importing us, so the
+paths must be computed in exactly one place on the writer side. This module
+is that place — every writer path in the application layer goes through
+:class:`CorpusLayout`, and the layout tests pin the contract strings.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+#: Per-session artifact filenames fixed by CONTRACT.md.
+TRAJECTORY_FILENAME = "trajectory.json"
+LOSS_REPORT_FILENAME = "loss_report.json"
+EDGES_FILENAME = "edges.jsonl"
+#: One line per kept non-message record (hooks, API errors, cost-state, ...).
+SESSION_EVENTS_FILENAME = "session_events.jsonl"
+META_FILENAME = "meta.json"
+WATERMARK_FILENAME = "watermark.json"
+#: Corpus-level record of sessions whose source converted to nothing.
+EMPTY_SESSIONS_FILENAME = "empty_sessions.json"
+#: Corpus-level record of published sessions a ``SessionSink`` has not yet
+#: taken (a failed or interrupted sync); retried on the next pass.
+SINK_PENDING_FILENAME = "sink_pending.json"
+#: Per-session directory holding the zstd copy of the raw source files.
+SOURCE_ARCHIVE_DIRNAME = "source"
+
+#: The suffix a compressed artifact carries after its logical name.
+COMPRESSED_SUFFIX = ".zst"
+#: The artifacts materialize stores zstd-compressed, as ``<name>.zst``: the
+#: bulk of a corpus's bytes. ``meta.json`` and ``loss_report.json`` stay plain
+#: (small, and ``meta.json`` is what every reader and the writer check first).
+#: A session written before this keeps its plain files until ``atif-sql corpus
+#: slim`` compresses them; readers accept either spelling. atif-duck carries
+#: the twin names (``atif_duck.domain.artifacts``), pinned by atif-cli's tests.
+COMPRESSED_ARTIFACT_FILENAMES: tuple[str, ...] = (
+    TRAJECTORY_FILENAME,
+    EDGES_FILENAME,
+    SESSION_EVENTS_FILENAME,
+)
+
+
+def stored_filename(name: str) -> str:
+    """The name a NEW session stores the artifact ``name`` under."""
+    return f"{name}{COMPRESSED_SUFFIX}" if name in COMPRESSED_ARTIFACT_FILENAMES else name
+
+
+#: A blob's name: the lowercase hex SHA-256 of its bytes.
+_BLOB_HASH_RE = re.compile(r"^[0-9a-f]{64}$")
+#: A blob's extension: short, lowercase alphanumeric (``png``, ``jpg``, ``bin``).
+_BLOB_EXTENSION_RE = re.compile(r"^[a-z0-9]{1,8}$")
+
+
+class InvalidBlobNameError(ValueError):
+    """A blob hash or extension that must not become part of a corpus path."""
+
+
+@dataclass(frozen=True, slots=True)
+class CorpusLayout:
+    """Computes every contract path under one ``corpus_root``.
+
+    Pure path arithmetic — nothing here touches the filesystem, so the
+    layout can be asserted against the contract without a tmpdir.
+    """
+
+    #: Root of the materialized corpus (default ``~/.atif-sql/corpus/<slug>``).
+    corpus_root: Path
+
+    @property
+    def sessions_dir(self) -> Path:
+        """Parent directory of every per-session artifact directory."""
+        return self.corpus_root / "sessions"
+
+    @property
+    def watermark_path(self) -> Path:
+        """``{path: mtime_ns}`` across the source corpus, updated per pass."""
+        return self.corpus_root / WATERMARK_FILENAME
+
+    @property
+    def staging_dir(self) -> Path:
+        """Scratch area for whole-session-dir atomic swaps.
+
+        Deliberately OUTSIDE ``sessions/`` so no reader glob (DuckDB's
+        ``read_json`` matches dot-dirs) and no source-removal walk can ever
+        observe a half-written session dir; same filesystem as
+        ``sessions/`` so ``os.replace`` of the staged dir stays atomic.
+        """
+        return self.corpus_root / ".staging"
+
+    @property
+    def empty_sessions_path(self) -> Path:
+        """``{session_id: generation}`` for sessions whose source held nothing to convert.
+
+        Kept apart from ``watermark.json`` on purpose: that file's shape is
+        ``{path: mtime_ns}`` and every reader coerces its values to ``int``, so
+        a structured entry there would read as corruption and cost a full
+        re-materialization on any version that predates this file.
+        """
+        return self.corpus_root / EMPTY_SESSIONS_FILENAME
+
+    @property
+    def sink_pending_path(self) -> Path:
+        """``{"session_ids": [...]}``: published sessions the session sink still owes.
+
+        Written BEFORE a pass publishes anything (every session it may
+        publish, plus what was already pending) and rewritten after the sink
+        ran, so a pass that dies between a session's swap and its sync still
+        leaves the session recorded.
+        """
+        return self.corpus_root / SINK_PENDING_FILENAME
+
+    @property
+    def blobs_dir(self) -> Path:
+        """The content-addressed attachment store shared by every session.
+
+        Outside ``sessions/`` on purpose: no reader glob and no per-session
+        swap touches it, retention never deletes from it, and a blob two
+        sessions share is stored once.
+        """
+        return self.corpus_root / "blobs"
+
+    def blob_path(self, sha256: str, extension: str) -> Path:
+        """``<corpus_root>/blobs/sha256/<first two hex>/<sha256>.<extension>``.
+
+        Raises
+        ------
+        InvalidBlobNameError
+            ``sha256`` is not 64 lowercase hex digits or ``extension`` is not
+            one to eight lowercase letters and digits.
+        """
+        if not _BLOB_HASH_RE.match(sha256) or not _BLOB_EXTENSION_RE.match(extension):
+            msg = f"invalid blob name {sha256!r}.{extension!r}"
+            raise InvalidBlobNameError(msg)
+        return self.blobs_dir / "sha256" / sha256[:2] / f"{sha256}.{extension}"
+
+    def session_dir(self, session_id: str) -> Path:
+        """``<corpus_root>/sessions/<session_id>/``."""
+        return self.sessions_dir / session_id
+
+    def trajectory_path(self, session_id: str) -> Path:
+        """Compact ATIF-v1.7 trajectory JSON for one session, as materialize stores it (zstd)."""
+        return self.session_dir(session_id) / stored_filename(TRAJECTORY_FILENAME)
+
+    def loss_report_path(self, session_id: str) -> Path:
+        """``atif_converter`` LossReport JSON for one session."""
+        return self.session_dir(session_id) / LOSS_REPORT_FILENAME
+
+    def edges_path(self, session_id: str) -> Path:
+        """One line per RAW record: uuid/parent_uuid edge list, as materialize stores it (zstd)."""
+        return self.session_dir(session_id) / stored_filename(EDGES_FILENAME)
+
+    def session_events_path(self, session_id: str) -> Path:
+        """One line per kept non-message record, as materialize stores it (zstd)."""
+        return self.session_dir(session_id) / stored_filename(SESSION_EVENTS_FILENAME)
+
+    def meta_path(self, session_id: str) -> Path:
+        """Provenance record: source files, mtimes, versions, materialized_at."""
+        return self.session_dir(session_id) / META_FILENAME
+
+    def source_archive_dir(self, session_id: str) -> Path:
+        """The raw source archive: one zstd ``<relative path>.zst`` per source file."""
+        return self.session_dir(session_id) / SOURCE_ARCHIVE_DIRNAME

atif_corpus/domain/ports.py

@@ -0,0 +1,175 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""The ports materialization needs filled: a converter and, optionally, an artifact producer and a session sink.
+
+atif-corpus may never import atif-converter or atif-duck (import-linter
+independence contract), so both Protocols are typed to CONTRACT.md's artifact
+shapes rather than to either package's internals. atif-cli adapts the real
+converter and the real columnar producer to these ports; tests use
+:class:`atif_corpus.infrastructure.fake_converter.FakeConverter` and a fake
+producer of their own.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Protocol
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+    from pathlib import Path
+
+
+class EmptySourceError(Exception):
+    """The session holds nothing to convert (no user or assistant record).
+
+    Not a failure: the transcript is well-formed and simply empty, and it stays
+    empty until something writes to it. An adapter translates its converter's
+    own "empty" verdict into this so the use case can record the session as
+    empty, keyed on its source mtimes, instead of retrying it every pass. Any
+    OTHER exception is a real failure.
+    """
+
+
+@dataclass(frozen=True, slots=True)
+class ArchivedSource:
+    """One file of a session's raw source archive, as the converter wrote it.
+
+    The compressed copy lives at ``<archive_dir>/<relative_path>.zst``;
+    ``size`` and ``sha256`` describe the UNCOMPRESSED bytes, so a restore can
+    prove it got the original back.
+    """
+
+    #: POSIX path relative to the main transcript's parent directory.
+    relative_path: str
+    #: Uncompressed size in bytes.
+    size: int
+    #: Hex sha256 of the uncompressed bytes.
+    sha256: str
+
+
+@dataclass(frozen=True, slots=True)
+class BlobOutput:
+    """One attachment's bytes, content-addressed, for the corpus blob store.
+
+    ``sha256`` is the lowercase hex digest of ``data`` and ``extension`` the
+    file extension its media type maps to (``png``, ``jpg``, ``pdf``, ``bin``);
+    the writer validates both before either becomes part of a path.
+    """
+
+    sha256: str
+    extension: str
+    data: bytes
+
+
+@dataclass(frozen=True, slots=True)
+class ConversionOutput:
+    """Everything one conversion yields that the corpus writes to disk.
+
+    Shapes are the contract's, pre-serialized by the adapter:
+
+    * ``trajectory_dict`` — the ATIF-v1.7 trajectory, ready for compact
+      ``json.dumps(separators=(",", ":"))``.
+    * ``loss_report_dict`` — ``LossReport.to_json()``-shaped dict.
+    * ``edges_lines`` — one already-serialized JSON line per RAW record
+      (uuid, parent_uuid, message_id, type, ts, is_sidechain,
+      is_compact_summary, source_file, tool_use_ids), WITHOUT trailing
+      newlines; the writer owns line termination.
+    * ``events_lines`` — one already-serialized JSON line per kept
+      NON-MESSAGE record (``session_events.jsonl``: seq, ts, event_type,
+      subtype, uuid, parent_uuid, tool_use_id, is_sidechain, source_file,
+      payload, payload_bytes, payload_truncated), same termination rule.
+      Empty by default, so a converter that has none writes an empty file.
+    * ``source_archive`` — the files the converter archived into the
+      ``archive_dir`` it was handed (empty when it was handed none, or when
+      the adapter does not archive). Every entry must exist on disk as
+      ``<archive_dir>/<relative_path>.zst``; the use case checks that.
+    * ``blobs`` — the inline attachments the converter replaced with a
+      ``[image sha256:<hash> ...]`` placeholder, one per distinct hash. The
+      writer stores each under ``<corpus_root>/blobs/`` BEFORE it publishes
+      the session that references it.
+    """
+
+    trajectory_dict: dict[str, Any]
+    loss_report_dict: dict[str, Any]
+    edges_lines: list[str]
+    events_lines: list[str] = field(default_factory=list)
+    source_archive: tuple[ArchivedSource, ...] = ()
+    blobs: tuple[BlobOutput, ...] = ()
+
+
+class ConverterPort(Protocol):
+    """Anything that can turn one session JSONL into corpus artifacts.
+
+    Implementations may raise any exception: the materialize use case
+    records the failure against the session and continues — one broken
+    transcript must never abort a corpus sync. :class:`EmptySourceError` is
+    the one exception with its own meaning: the session is empty, not broken.
+    """
+
+    def convert(self, session_jsonl: Path, *, archive_dir: Path | None = None) -> ConversionOutput:
+        """Convert one session (main JSONL + its side-files) to artifacts.
+
+        With ``archive_dir``, also write a zstd copy of every source file of the
+        session under it, named by its path relative to ``session_jsonl``'s
+        parent plus ``.zst``, and list them in
+        :attr:`ConversionOutput.source_archive`. The copy must be of the bytes
+        the artifacts were built from; an adapter that cannot promise that
+        should archive nothing rather than something else.
+        """
+        ...
+
+
+class ArtifactProducer(Protocol):
+    """Anything that writes EXTRA per-session artifacts beside the contract ones.
+
+    The materialize use case calls :meth:`produce` once per session, inside
+    the staged session directory, after ``trajectory.json`` /
+    ``loss_report.json`` / ``edges.jsonl`` / ``session_events.jsonl`` are
+    written (so a producer may read those staged files) and before
+    ``meta.json`` is. Whatever the producer writes therefore publishes
+    atomically with the contract artifacts (the whole directory is
+    swapped into place) and is covered by the same completeness marker.
+
+    The producer returns extra keys for ``meta.json``. They must not collide
+    with the contract's own keys; the use case treats a collision as a
+    programming error and fails the session.
+
+    Implementations may raise any exception: the use case records the
+    failure against the session, never publishes the staged directory, and
+    retries the session next pass, exactly as it does for a converter
+    failure.
+    """
+
+    def produce(
+        self,
+        session_dir: Path,
+        *,
+        session_id: str,
+        trajectory: Mapping[str, Any],
+    ) -> Mapping[str, Any]:
+        """Write extra artifacts for ``session_id`` into ``session_dir``; return meta extras."""
+        ...
+
+
+class SessionSink(Protocol):
+    """Anything that keeps a copy of published sessions in step with the corpus.
+
+    atif-cli plugs atif-duck's DuckLake writer in here. The materialize use
+    case calls :meth:`sync_sessions` in the PARENT process only (never in a
+    pool worker, so an implementation need not pickle), after every session
+    in ``session_ids`` has been swapped into ``sessions/<id>/`` (or had its
+    ``meta.json`` rewritten as source-removed), one batch at a time, in plan
+    order.
+
+    Returning normally means the copy now matches those sessions' published
+    artifacts (an implementation with no copy to keep, say because none has
+    been created yet, may simply return). Raising means it may not: the use
+    case records every session of the batch, and every later batch, in
+    ``<corpus_root>/sink_pending.json`` and hands them to the sink again next
+    pass, so the copy never silently diverges from the corpus.
+    """
+
+    def sync_sessions(self, *, corpus_root: Path, agent: str, session_ids: Sequence[str]) -> None:
+        """Make the copy's rows for ``session_ids`` equal their published artifacts."""
+        ...

atif_corpus/domain/session_id.py

@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""The session id boundary: which directory names may become a session.
+
+A session id is the one piece of text from OUTSIDE this workspace that ends up
+inside a corpus path: it is derived from a transcript filename under the
+source root, becomes the ``sessions/<session_id>/`` directory name, and from
+there names the files atif-duck opens. Everything else in a statement is a
+constant. So the id is checked once, at the boundary, and a name that fails
+the check is skipped and reported rather than carried any further.
+
+The rule is deliberately narrow: an ASCII letter or digit first, then letters,
+digits, ``.``, ``_`` and ``-``, at most 255 characters (the longest name most
+filesystems accept). Every id both supported agents produce passes: Claude
+Code names a session with a version 4 UUID, Codex with a version 7 UUID
+(``019e9e38-39a6-7170-86fb-46e6f1cb1931``). Quotes, whitespace, path
+separators, ``;``, ``$``, ``?`` and anything outside ASCII fail.
+
+DELIBERATE TWIN of :mod:`atif_duck.domain.session_id`, the same way
+:mod:`atif_corpus.domain.agents` twins the converter's enum: the independence
+contract forbids the two packages from importing each other, and both sides
+must apply the SAME rule (the corpus writer before it writes, the registry
+before it reads a corpus an older version wrote). Each package's tests read
+the other's copy as source text and fail if the pattern or the limit drifts.
+
+Pure module: stdlib only.
+"""
+
+from __future__ import annotations
+
+import re
+
+#: The accepted shape, as a pattern string so the twin pin can compare it.
+SESSION_ID_PATTERN: str = r"^[A-Za-z0-9][A-Za-z0-9._-]*$"
+
+#: Longest accepted id, in characters.
+SESSION_ID_MAX_CHARS: int = 255
+
+_SESSION_ID_RE = re.compile(SESSION_ID_PATTERN)
+
+
+def session_id_rejection(candidate: str) -> str | None:
+    """Why ``candidate`` cannot be a session id, or ``None`` when it can.
+
+    The reason is one short phrase meant for a log line; a caller that only
+    needs the verdict uses :func:`is_valid_session_id`.
+    """
+    if len(candidate) > SESSION_ID_MAX_CHARS:
+        return f"longer than {SESSION_ID_MAX_CHARS} characters"
+    # fullmatch, not match: with ``$`` alone a trailing newline would pass.
+    if _SESSION_ID_RE.fullmatch(candidate) is None:
+        return (
+            "contains a character outside [A-Za-z0-9._-] or does not start with a letter or digit"
+        )
+    return None
+
+
+def is_valid_session_id(candidate: str) -> bool:
+    """Whether ``candidate`` passes the boundary."""
+    return session_id_rejection(candidate) is None
+
+
+__all__ = [
+    "SESSION_ID_MAX_CHARS",
+    "SESSION_ID_PATTERN",
+    "is_valid_session_id",
+    "session_id_rejection",
+]