atif-sql 0.2.0__py3-none-any.whl

atif_corpus/domain/sessions.py

@@ -0,0 +1,228 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Value objects describing what a materialization pass sees and decides.
+
+The decision pipeline is three pure pieces:
+
+* :class:`SessionSource` — one session's source files and their mtimes, as
+  scanned (infrastructure feeds this in; the domain never stats a file).
+* :class:`QuiescencePolicy` — the "is this session done being written?"
+  rule. A transcript is appended to many times per turn; converting a
+  session mid-write wastes a conversion and materializes a half-turn. A
+  session qualifies only once its newest source mtime is at least
+  ``quiesce_seconds`` old.
+* :class:`MaterializationPlan` / :func:`build_plan` — the deterministic
+  partition of scanned sessions into to-materialize / up-to-date /
+  skipped-live, given the recorded watermark and "now".
+
+Clock discipline: all ``*_ns`` values are **epoch nanoseconds** — mtimes from
+``os.stat().st_mtime_ns``, "now" from ``time.time_ns()`` passed in by the
+caller. The domain never reads a clock, so the same inputs always yield the
+same plan (pinned by the determinism tests).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from loguru import logger
+
+from atif_corpus.domain.watermark import diff_source_mtimes
+
+if TYPE_CHECKING:
+    from collections.abc import Collection, Mapping, Sequence
+
+#: Nanoseconds per second; policies are configured in seconds and convert
+#: once at the boundary because ``st_mtime_ns`` is the native unit.
+NANOS_PER_SECOND: int = 1_000_000_000
+
+
+@dataclass(frozen=True, slots=True)
+class SessionSource:
+    """One discovered session: its id and every source file with its mtime.
+
+    ``source_mtimes`` covers the main ``<session_id>.jsonl`` PLUS every
+    side-file under the session dir (``subagents/agent-*.jsonl``,
+    ``subagents/workflows/wf_*/agent-*.jsonl``, and any deeper future
+    nesting) — the contract requires the watermark to notice a workflow
+    side-file changing even when the main transcript did not.
+    """
+
+    #: The session UUID (the main JSONL's stem).
+    session_id: str
+    #: Absolute path of the main ``<session_id>.jsonl`` transcript.
+    session_jsonl: str
+    #: ``{absolute path: epoch-ns mtime}`` for the main file + all side-files.
+    source_mtimes: Mapping[str, int]
+
+    @property
+    def newest_mtime_ns(self) -> int:
+        """The most recent write across all of this session's source files."""
+        return max(self.source_mtimes.values())
+
+    @property
+    def source_files(self) -> tuple[str, ...]:
+        """Sorted source paths — deterministic, for meta.json and diffing."""
+        return tuple(sorted(self.source_mtimes))
+
+
+@dataclass(frozen=True, slots=True)
+class QuiescencePolicy:
+    """The rule for when a session is settled enough to convert.
+
+    A session is *quiescent* when its newest source mtime is at least
+    ``quiesce_seconds`` in the past. Nothing announces that a writer finished,
+    so silence is the only available signal.
+
+    SCOPE: the quiet period is per-SESSION, not per-file. One late write to
+    any of a session's source files holds the whole session back, which is
+    what keeps a multi-file session from being materialized half-written.
+    """
+
+    #: Seconds of source silence required before a session may materialize.
+    quiesce_seconds: int = 300
+    #: How far in the future an mtime may sit before it earns the clock warning.
+    #: The decision never uses it (a future mtime is never quiescent); it only
+    #: keeps sub-second jitter, such as a coarse filesystem timestamp or a stat
+    #: racing the clock read, from reading as a misconfigured host.
+    future_warn_seconds: float = 1.0
+
+    def is_quiescent(self, newest_mtime_ns: int, now_ns: int) -> bool:
+        """True when the newest source write is at least ``quiesce_seconds`` old.
+
+        The boundary is inclusive (``>=``): a session exactly at the
+        threshold counts as quiescent, so the edge is testable without
+        racing a clock.
+
+        A source mtime in the FUTURE (a clock step on the writing host, a
+        copied file carrying a bogus stamp) never satisfies the threshold,
+        so the session would starve in ``skipped_live`` silently. It stays
+        skipped — converting is still the wrong call — but it warns every
+        pass so the starvation is visible instead of mute. Only an mtime more
+        than ``future_warn_seconds`` ahead warns: the caller reads "now" after
+        the scan, so a smaller lead is jitter, not a clock problem.
+        """
+        age_ns = now_ns - newest_mtime_ns
+        if -age_ns > self.future_warn_seconds * NANOS_PER_SECOND:
+            logger.warning(
+                "quiescence: newest source mtime is {:.1f}s in the FUTURE; "
+                "session stays skipped until the clock catches up "
+                "(check the writing host's clock)",
+                -age_ns / NANOS_PER_SECOND,
+            )
+        return age_ns >= self.quiesce_seconds * NANOS_PER_SECOND
+
+
+@dataclass(frozen=True, slots=True)
+class MaterializationPlan:
+    """The deterministic work order for one materialization pass.
+
+    All three partitions are sorted by session id so the same scan always
+    yields the same plan (and the same log output, and the same failure
+    ordering).
+    """
+
+    #: Sessions to (re)convert: quiescent AND stale relative to the watermark.
+    to_materialize: tuple[SessionSource, ...]
+    #: Sessions whose artifacts already reflect their sources; nothing to do.
+    up_to_date: tuple[SessionSource, ...]
+    #: Sessions still being written (not quiescent); revisit next pass.
+    skipped_live: tuple[SessionSource, ...]
+
+    @property
+    def is_noop(self) -> bool:
+        """True when the pass has nothing to convert."""
+        return not self.to_materialize
+
+
+def owns_path(session_jsonl: str, path: str) -> bool:
+    """True when ``path`` is ``session_jsonl`` itself or one of its side-files.
+
+    The single definition of a session's watermark scope: the main JSONL
+    plus everything under ``<main-jsonl-stem>/``. Staleness and watermark
+    retention both key off this, so the two can never disagree about which
+    entries belong to a session. Keyed on the main path rather than a whole
+    :class:`SessionSource` so a session the scanner could NOT read — it has
+    a path but no mtimes — gets scoped by the same rule.
+    """
+    stem = session_jsonl.rsplit(".", 1)[0]
+    return path == session_jsonl or path.startswith(f"{stem}/")
+
+
+def _is_stale(session: SessionSource, watermark: Mapping[str, int]) -> bool:
+    """True when the recorded watermark fails to describe this session's sources.
+
+    Delegates the comparison to
+    :func:`atif_corpus.domain.watermark.diff_source_mtimes` over this
+    session's scope: a source file added, changed in either mtime direction,
+    or *removed* since the last pass all make the session stale. Removal
+    counts because the materialized trajectory still embeds the vanished
+    file's records.
+    """
+    recorded = {
+        path: mtime for path, mtime in watermark.items() if owns_path(session.session_jsonl, path)
+    }
+    return not diff_source_mtimes(recorded, session.source_mtimes).is_empty
+
+
+def build_plan(
+    sessions: Sequence[SessionSource],
+    *,
+    watermark: Mapping[str, int],
+    policy: QuiescencePolicy,
+    now_ns: int,
+    force: bool = False,
+    unmaterialized_session_ids: Collection[str] = (),
+    generation_stale_session_ids: Collection[str] = (),
+) -> MaterializationPlan:
+    """Partition scanned sessions into the three plan buckets.
+
+    Decision order per the contract: a session is (re)materialized when its
+    newest source mtime is quiescent AND the watermark says its sources
+    moved. ``force`` overrides ONLY the staleness check — a live session is
+    never converted even under force, because converting a half-written
+    transcript produces a wrong artifact rather than a stale one.
+
+    Parameters
+    ----------
+    sessions
+        Every session the scanner discovered under the source root.
+    watermark
+        The recorded ``{path: mtime_ns}`` map from the previous pass
+        (``watermark.json``); empty on first run.
+    policy
+        The quiescence rule.
+    now_ns
+        Epoch nanoseconds "now", read once by the caller — never here.
+    force
+        Re-materialize every quiescent session regardless of the watermark.
+    unmaterialized_session_ids
+        Sessions the watermark claims are current but whose artifact dir is
+        absent on disk (a pass killed mid-swap). Treated as stale so the
+        artifacts come back; the watermark alone cannot express this, since
+        it records SOURCE mtimes and knows nothing about the corpus dir.
+    generation_stale_session_ids
+        Sessions whose recorded provenance (``converter_schema``,
+        ``columnar_schema``) differs from what this pass would stamp, per
+        :func:`atif_corpus.domain.generation.generation_matches`. Treated as
+        stale: the source did not move, but the code that turns it into
+        artifacts did.
+    """
+    ordered = sorted(sessions, key=lambda s: s.session_id)
+    missing = set(unmaterialized_session_ids) | set(generation_stale_session_ids)
+    to_materialize: list[SessionSource] = []
+    up_to_date: list[SessionSource] = []
+    skipped_live: list[SessionSource] = []
+    for session in ordered:
+        if not policy.is_quiescent(session.newest_mtime_ns, now_ns):
+            skipped_live.append(session)
+        elif force or session.session_id in missing or _is_stale(session, watermark):
+            to_materialize.append(session)
+        else:
+            up_to_date.append(session)
+    return MaterializationPlan(
+        to_materialize=tuple(to_materialize),
+        up_to_date=tuple(up_to_date),
+        skipped_live=tuple(skipped_live),
+    )

atif_corpus/domain/slug.py

@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Map one corpus source root to a stable, human-legible directory key.
+
+The slug is an ON-DISK NAME, not an internal identifier: it is the
+``<slug>`` in ``~/.atif-sql/corpus/<slug>/``, so this function's output is
+part of every installed user's filesystem layout.
+
+Changing the algorithm therefore ORPHANS existing corpora rather than
+migrating them. ``materialize`` starts writing a fresh tree, ``status``
+reports an empty corpus, ``query`` binds zero sessions, and the watermarks
+and LanceDB embedding store at the no-longer-computed path are stranded,
+re-earned only by a full re-materialize and re-embed. ``tests/test_slug.py``
+freezes concrete outputs so such a change fails a test, not a user's install.
+
+If a change here is genuinely wanted, it needs all three of: the frozen
+values in ``tests/test_slug.py`` updated, a migration for corpora already on
+disk, and the two call sites that compose the path
+(``_default_corpus_root`` in :mod:`atif_corpus.infrastructure.settings` and
+the ``--corpus-root`` fallback in ``atif_cli.app``) reviewed together.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from pathlib import Path
+
+#: Reserved slug for the interactive corpus at ``~/.claude`` — the root nearly
+#: every user has. It is reserved rather than hashed so the common case has a
+#: dirname a human can recognize in ``ls``.
+DEFAULT_CORPUS_KEY = "default"
+
+#: Reserved slug for the Codex rollout corpus at ``~/.codex/sessions``. Same
+#: reasoning as :data:`DEFAULT_CORPUS_KEY`, and safe to add rather than a
+#: breaking change: this workspace could not read a Codex rollout before
+#: 2026-09-11, so no corpus can already occupy the hashed name this replaces
+#: (``sessions-<8 hex>``). Reserved on the SESSIONS dir rather than on
+#: ``~/.codex`` because the sessions dir is the source root a scan walks.
+CODEX_CORPUS_KEY = "codex"
+
+#: Any run of characters outside ``[a-z0-9]`` collapses to a single ``-``.
+_SLUG_SANITIZE_RE = re.compile(r"[^a-z0-9]+")
+
+#: Cap on the legible dirname component; the hash suffix carries uniqueness.
+_SLUG_MAX_NAME_LEN = 32
+
+
+def corpus_slug(corpus_root: Path | str) -> str:
+    """Map one corpus root to a stable, human-legible directory key.
+
+    Pure: no I/O beyond path resolution. The interactive corpus
+    (``~/.claude``, however it is spelled) maps to the reserved key
+    :data:`DEFAULT_CORPUS_KEY`. Every other root maps to
+    ``<sanitized-dirname>-<8-hex sha256 of the resolved path>`` — legible
+    enough to eyeball in ``ls``, hashed enough that two roots sharing a
+    dirname (``…/alice/.claude`` vs ``…/bob/.claude``) never collide.
+
+    The Codex rollout root (``~/.codex/sessions``) maps to
+    :data:`CODEX_CORPUS_KEY` for the same legibility reason.
+
+    The root is ``expanduser().resolve()``-normalized first so symlinked
+    spellings of the same corpus agree on one key.
+    """
+    resolved = Path(corpus_root).expanduser().resolve()
+    if resolved == Path("~/.claude").expanduser().resolve():
+        return DEFAULT_CORPUS_KEY
+    if resolved == Path("~/.codex/sessions").expanduser().resolve():
+        return CODEX_CORPUS_KEY
+    name = _SLUG_SANITIZE_RE.sub("-", resolved.name.lower()).strip("-")[:_SLUG_MAX_NAME_LEN]
+    digest = hashlib.sha256(str(resolved).encode("utf-8")).hexdigest()[:8]
+    return f"{name}-{digest}" if name else digest

atif_corpus/domain/source_layout.py

@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Where an agent's transcripts live, as one value object per agent.
+
+Discovery used to be a single hardcoded shape — project directories one level
+under the source root, one JSONL per session — because Claude Code was the only
+agent. Codex CLI writes nothing like it: rollouts sit three levels deep under
+``$CODEX_HOME/sessions/<YYYY>/<MM>/<DD>/`` and the session id is a uuid inside
+the FILENAME rather than the whole stem. Both shapes are now described here, as
+pure data plus pure functions, so the scanner walks one algorithm and the
+per-agent knowledge lives in exactly one place.
+
+Pure module: no filesystem access, no env, no logging. Everything it returns is
+derived from a path's TEXT.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import PurePosixPath
+
+from atif_corpus.domain.agents import AgentSource
+
+#: A Codex rollout filename's fixed prefix, and the number of hyphen-separated
+#: groups its trailing uuid occupies: ``rollout-<ts>-<8>-<4>-<4>-<4>-<12>.jsonl``.
+_ROLLOUT_PREFIX = "rollout-"
+_UUID_GROUPS = 5
+
+
+@dataclass(frozen=True, slots=True)
+class SourceLayout:
+    """How one agent's transcripts are arranged under a source root.
+
+    ``transcript_depth`` is how many directory levels sit between the source
+    root and a transcript file: 1 for Claude Code (``<project>/<session>.jsonl``)
+    and 3 for Codex (``<YYYY>/<MM>/<DD>/rollout-*.jsonl``). The scanner descends
+    exactly that many levels, listing each one explicitly, which is what keeps a
+    directory it could not OPEN distinguishable from an empty one at every level
+    rather than only at the first.
+
+    ``has_side_files`` says whether a session can own a sibling directory of
+    additional transcripts. Claude Code sessions do (subagent and workflow
+    side-files, which the watermark must notice); a Codex sub-agent writes its
+    own rollout under its own session id, so a rollout is always alone.
+    """
+
+    agent: AgentSource
+    transcript_depth: int
+    has_side_files: bool
+
+    def is_transcript(self, name: str) -> bool:
+        """Whether a filename in a transcript directory is one of this agent's."""
+        if not name.endswith(".jsonl"):
+            return False
+        if self.agent is AgentSource.CODEX:
+            return name.startswith(_ROLLOUT_PREFIX)
+        return True
+
+    def session_id(self, name: str) -> str | None:
+        """The session id a transcript filename carries, or ``None`` if it carries none.
+
+        Claude Code names the file after the session, so the stem IS the id.
+        Codex prefixes a timestamp, so the id is the stem's trailing uuid — and
+        a name whose shape does not yield one is not a session this layout can
+        materialize, which the caller must skip rather than guess at.
+        """
+        if not self.is_transcript(name):
+            return None
+        stem = name.removesuffix(".jsonl")
+        if self.agent is not AgentSource.CODEX:
+            return stem or None
+        parts = stem.split("-")
+        if len(parts) < _UUID_GROUPS + 1:
+            return None
+        return "-".join(parts[-_UUID_GROUPS:])
+
+    def session_from_watermark_path(self, source_root: str, path: str) -> tuple[str, str] | None:
+        """``(session_id, main transcript path)`` for a recorded watermark path.
+
+        Used when a directory could not be LISTED: nothing was discovered
+        there, so the watermark is the only record of which sessions lived
+        inside it, and each recorded path must resolve back to a session id
+        WITHOUT touching the filesystem.
+
+        Returns ``None`` when the path is not under ``source_root`` or its shape
+        does not name a session.
+        """
+        root_prefix = f"{source_root.rstrip('/')}/"
+        if not path.startswith(root_prefix):
+            return None
+        segments = PurePosixPath(path[len(root_prefix) :]).parts
+        if len(segments) <= self.transcript_depth:
+            return None
+        if self.agent is AgentSource.CODEX:
+            # A rollout owns no side-files, so a recorded path IS a main
+            # transcript and its own name carries the id.
+            session_id = self.session_id(segments[self.transcript_depth])
+            return (session_id, path) if session_id else None
+        # Claude Code: `<project>/<session>.jsonl`, or a side-file under
+        # `<project>/<session>/...`. Either way the second segment names the
+        # session, so the main transcript is derivable from position alone.
+        project, entry = segments[0], segments[1]
+        session_id = entry.removesuffix(".jsonl")
+        if not session_id:
+            return None
+        return session_id, f"{root_prefix}{project}/{session_id}.jsonl"
+
+
+#: Claude Code: `<source_root>/<project>/<session>.jsonl` plus a side-file tree
+#: under `<source_root>/<project>/<session>/`.
+CLAUDE_CODE_LAYOUT = SourceLayout(
+    agent=AgentSource.CLAUDE_CODE,
+    transcript_depth=1,
+    has_side_files=True,
+)
+
+#: Codex CLI: `<source_root>/<YYYY>/<MM>/<DD>/rollout-<ts>-<uuid>.jsonl`, one
+#: file per session and no side-files.
+CODEX_LAYOUT = SourceLayout(
+    agent=AgentSource.CODEX,
+    transcript_depth=3,
+    has_side_files=False,
+)
+
+_LAYOUTS: dict[AgentSource, SourceLayout] = {
+    AgentSource.CLAUDE_CODE: CLAUDE_CODE_LAYOUT,
+    AgentSource.CODEX: CODEX_LAYOUT,
+}
+
+
+def layout_for(agent: AgentSource) -> SourceLayout:
+    """The layout for ``agent``.
+
+    Every :class:`~atif_corpus.domain.agents.AgentSource` member has one, and
+    the lookup raises rather than defaulting: a new agent added to the enum
+    without a layout must fail loudly at the first scan instead of silently
+    scanning with Claude Code's shape and reporting an empty corpus.
+    """
+    try:
+        return _LAYOUTS[agent]
+    except KeyError as exc:  # pragma: no cover — unreachable while the enum is closed
+        msg = f"no source layout declared for agent {agent!r}"
+        raise ValueError(msg) from exc

atif_corpus/domain/watermark.py

@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Pure watermark math: which source files moved between two scans.
+
+Compare two ``{path: mtime_ns}`` maps and partition into added / modified /
+removed. ``touched`` (added then modified) are the sessions whose artifacts
+must be re-materialized; ``removed`` are sessions whose source vanished. Pure
+so the delta can be tested without a filesystem and so two identical scans
+always produce the same plan.
+
+Clock discipline: every ``mtime_ns`` here is a filesystem modification time
+in **epoch nanoseconds** (``os.stat().st_mtime_ns``). Nothing in this module
+reads a clock; "now" is always passed in by the caller.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+
+@dataclass(frozen=True, slots=True)
+class SourceDelta:
+    """Which source files moved between two watermark snapshots.
+
+    All three partitions are sorted tuples so a delta is deterministic and
+    comparable — the materialization plan embeds these paths, and set
+    ordering would make two identical scans plan different work orders.
+    """
+
+    #: Paths present in the current scan and absent from the previous one.
+    added: tuple[str, ...]
+    #: Paths present in both whose ``mtime_ns`` changed (either direction).
+    modified: tuple[str, ...]
+    #: Paths present in the previous scan and absent from the current one.
+    removed: tuple[str, ...]
+
+    @property
+    def touched(self) -> tuple[str, ...]:
+        """Paths whose artifacts must be re-materialized (``added`` then ``modified``)."""
+        return (*self.added, *self.modified)
+
+    @property
+    def is_empty(self) -> bool:
+        """True when nothing moved, so a materialization pass would be a no-op."""
+        return not (self.added or self.modified or self.removed)
+
+    @property
+    def changed_count(self) -> int:
+        """Total number of paths in the delta, across all three partitions."""
+        return len(self.added) + len(self.modified) + len(self.removed)
+
+
+def diff_source_mtimes(
+    previous: Mapping[str, int],
+    current: Mapping[str, int],
+) -> SourceDelta:
+    """Partition two ``{path: mtime_ns}`` maps into added / modified / removed.
+
+    A path is ``modified`` only when its mtime **changed**. An mtime that went
+    backwards (a restored backup, a clock step on the writing host, a file
+    replaced by an older copy) also counts as modified: the recorded watermark
+    no longer describes the bytes on disk, so the session must be re-read.
+    Only an exactly-equal mtime is treated as unchanged.
+    """
+    added = sorted(path for path in current if path not in previous)
+    removed = sorted(path for path in previous if path not in current)
+    modified = sorted(
+        path
+        for path, mtime_ns in current.items()
+        if path in previous and previous[path] != mtime_ns
+    )
+    return SourceDelta(added=tuple(added), modified=tuple(modified), removed=tuple(removed))

atif_corpus/infrastructure/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Infrastructure layer: filesystem scanning, atomic writes, settings, fakes.
+
+Everything that touches the world (env, clock-adjacent file mtimes, disk)
+lives here so the domain stays a pure function of its inputs.
+"""