zettcode-engine 0.3.4__py3-none-any.whl

zettcode/engine/storage/store.py

@@ -0,0 +1,1056 @@
+"""Where sessions live: the workspace layout, the JSONL writer, and the plugin.
+
+One store root is shared by every workspace the user opens, so it is grouped by
+workspace first::
+
+    ~/.zettcode/sessions/<workspace path, levels joined with ->/
+    ├── metadata.jsonl              <- the index: title and activity times
+    └── <session-id>/
+        └── data.jsonl              <- the conversation tree
+
+The folder name is the resolved workspace path with ``-`` between its levels
+(``-Users-me-api``), so a path containing separators never turns into extra
+folders; :mod:`layout` decides the name, and moves a folder an older version
+named with base64 when nothing has it open.
+
+File operations are plain blocking reads and writes: an append is one small
+line, and the caller is the single process owning the session. The last line of
+a file may be truncated by a crash, so an unparsable *final* line is ignored; a
+corrupt line anywhere else raises, because that means real damage.
+
+:class:`SessionPersistenceMixin` owns all of that — the layout, parsing and the
+torn-last-line policy, the tree walk, ``read``, ``append``, ``checkpoint``, the
+metadata log, and ``list_sessions``. :class:`SessionStore` adds the framework
+hooks on top, so saving stays a mixin concern and the agent lifecycle stays a
+plugin concern.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import threading
+from collections import Counter
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass, field, replace
+from datetime import datetime
+from pathlib import Path
+from typing import cast
+from weakref import WeakKeyDictionary
+
+from pydantic import ValidationError
+from zett_agent.agent import AgentRunContext
+from zett_agent.extensions.base import AgentExtension
+from zett_agent.extensions.compaction import CompactedMessage
+from zett_agent.extensions.events import (
+    CompactionEvent,
+    ExtensionEvent,
+    MessageAppendedEvent,
+    MessageTiming,
+    RunCancelledEvent,
+)
+from zett_agent.ids import new_uuid7
+from zett_agent.messages import AnyMessage, AssistantMessage, SystemMessage, ToolMessage
+from zett_agent.model import ModelUsage
+
+from ..dialogue.side import ORIGIN, READER, SIDE, WATCHED, is_side, is_watched_side
+from .layout import claim_workspace, open_workspace
+from .ledger import LedgerReader, SessionStats
+from .locks import prune_empty
+from .metadata import (
+    MAX_MODEL,
+    MAX_TITLE,
+    METADATA_FILE,
+    SESSION_METADATA_FILE,
+    SessionInfo,
+    SessionMetadata,
+    info_of,
+    read_metadata,
+    read_session_metadata,
+    write_session_metadata,
+)
+from .records import (
+    FORK_SUFFIX,
+    LINES,
+    MAX_AGENT_NAME,
+    SAFE_SESSION_ID,
+    STORE_FILE,
+    STORE_VERSION,
+    CompactionLine,
+    JsonValue,
+    Line,
+    MessageLine,
+    Session,
+    SessionLine,
+    now,
+)
+
+
+def message_context(
+    value: Mapping[str, object] | None, *, field: str, nonempty_keys: bool = False
+) -> dict[str, JsonValue]:
+    """Validate one metadata or tags object before it reaches the log."""
+    if value is None:
+        return {}
+    encoded: dict[str, object] = {}
+    for key, item in value.items():
+        if not isinstance(key, str):
+            raise ValueError(f"{field} keys must be strings")
+        if nonempty_keys and not key.strip():
+            raise ValueError(f"{field} keys cannot be empty")
+        encoded[key] = item
+    # Pydantic validates the value when the line is built; this only tells the
+    # checker what the runtime already guarantees.
+    return cast("dict[str, JsonValue]", encoded)
+
+
+@dataclass(slots=True)
+class _Request:
+    """Persistence bookkeeping for one request-scoped context."""
+
+    request_id: str
+    checkpoint_version: int = 0
+    #: Whether this request is a side question (see ``engine.dialogue.side``): its
+    #: messages are written for the record but never replayed into a context.
+    side: bool = False
+    #: Whether the reader watched that side question on screen, which is what
+    #: a resumed transcript restores: a plugin's own question is never drawn.
+    #: Both are set together, from the run's own question (:meth:`on_event`), and
+    #: read only for a request already known to be a side one, so the defaults are
+    #: what the dataclass requires and nothing more: a run whose question was never
+    #: seen is not a side run at all, and its lines go untagged.
+    watched: bool = False
+    #: Who asked it — ``"reader"``, or the plugin asking for itself — kept as a
+    #: tag so a stored exchange says what it was without reading its text.
+    origin: str = READER
+    #: One stored message id for each replayable non-system context position.
+    #: ``None`` marks a context-only message the store was told to skip. The
+    #: checkpoint's own position is represented by its boundary message id, so
+    #: a later compaction can name the same boundary again.
+    context_ids: list[str | None] = field(default_factory=list)
+
+
+class SessionPersistenceMixin:
+    """Standalone mixin: read and write the conversation log and its metadata.
+
+    It inherits nothing, so it can be combined with the framework's
+    ``AgentExtension`` (see :class:`SessionStore`). It owns the storage: the
+    workspace layout, parsing and the torn-last-line policy, the tree walk,
+    ``read``, ``append``, ``checkpoint``, the metadata log with
+    ``set_title``/``session_title`` and the per-session flags beside them, and
+    ``list_sessions``. It holds no framework behavior.
+
+    Args:
+        root: Directory holding one folder per workspace, and inside it one
+            ``<session_id>/`` folder per session with its own ``data.jsonl``.
+            ``None`` uses ``~/.zettcode/sessions``.
+        workspace: Project the sessions belong to; its resolved absolute path,
+            joined with ``-``, names the workspace folder. ``None`` uses the
+            current directory, which is only right for standalone use — the
+            runtime always passes the configured workspace.
+
+    Folders are created when something is first written to them, not when the store is built: a project
+    that is opened and never used leaves nothing in the store.
+    """
+
+    def __init__(self, root: str | Path | None = None, workspace: str | Path | None = None) -> None:
+        self.root = Path(root) if root is not None else Path.home() / ".zettcode" / "sessions"
+        self.workspace = Path(workspace).expanduser().resolve() if workspace is not None else Path.cwd()
+        #: Opening it moves a folder an older version named with base64 to the readable name,
+        #: or leaves it where it is when something has it open (see :mod:`.layout`). The folder is
+        #: made by the first write, not here.
+        self.workspace_dir = open_workspace(self.root, self.workspace)
+        #: The log older versions kept for the whole workspace. Read, never written: see
+        #: :mod:`zettcode.engine.storage.metadata`.
+        self.legacy_metadata_path = self.workspace_dir / METADATA_FILE
+        # Parsed sessions, keyed by the file stamp that produced them. Appending
+        # is the hot path — one call per message — so it must not re-parse the
+        # whole log; the cache makes a read O(1) and a write a tuple copy.
+        self._sessions: dict[str, tuple[tuple[int, int] | None, Session]] = {}
+        #: A read now runs on a worker thread for the framework's restore hook,
+        #: while appends still fold into the same cache on the loop. The lock
+        #: orders only the cache lookups and the publish; the parse itself runs
+        #: outside it. It does not make the file write atomic.
+        self._cache_lock = threading.RLock()
+        #: The old log folded into rows, with the stamp of the file it came from, so a
+        #: workspace with several old sessions folds it once instead of once per session.
+        self._legacy: tuple[tuple[int, int] | None, dict[str, SessionInfo]] | None = None
+        #: Whether the workspace folder has been made and marked as ours; the first write does it, once.
+        self._claimed = False
+        #: Reads the totals the sessions panel shows; made on the first ask, since most runs never list.
+        self._ledger: LedgerReader | None = None
+        super().__init__()
+
+    # -- files --------------------------------------------------------------
+    def session_dir(self, session_id: str) -> Path:
+        """Return the directory of one session, refusing ids that leave the store."""
+        if not SAFE_SESSION_ID.match(session_id) or ".." in session_id:
+            raise ValueError(f"Unsafe session id: {session_id!r}")
+        return self.workspace_dir / session_id
+
+    def session_path(self, session_id: str) -> Path:
+        """Return the conversation file of one session."""
+        return self.session_dir(session_id) / STORE_FILE
+
+    def session_ids(self) -> Iterator[str]:
+        """Yield the id of every session folder this workspace owns."""
+        for path in sorted(self.workspace_dir.glob(f"*/{STORE_FILE}")):
+            session_id = path.parent.name
+            if SAFE_SESSION_ID.match(session_id) and ".." not in session_id:
+                yield session_id
+
+    def exists(self, session_id: str) -> bool:
+        """Say whether a session was started, by reading the one line that says so.
+
+        A session file opens with its header, so that line answers the question.
+        :meth:`read` is the call that checks the whole file, and it is the wrong one
+        to ask a yes-or-no question with: on a long session it parses every message
+        to report what the first line already says, and a command line that did so
+        before resuming paid for the parse twice — once to be told the session was
+        there, and again, in a store that had not kept the first, to open it.
+
+        Args:
+            session_id: The session to look for.
+
+        Returns:
+            ``True`` when the file exists and its first record is a session header.
+        """
+        return self._header(session_id) is not None
+
+    def _header(self, session_id: str) -> SessionLine | None:
+        """Return a session's header, read from the first line of its file and nothing more."""
+        try:
+            with self.session_path(session_id).open("rb") as handle:
+                for line in handle:
+                    if line.strip():
+                        record = LINES.validate_json(line)
+                        return record if isinstance(record, SessionLine) else None
+        except (OSError, ValidationError):
+            return None
+        return None
+
+    def read(self, session_id: str) -> Session:
+        """Parse one session file, tolerating a half-written final line.
+
+        The file is parsed outside the cache lock: a worker restoring a long
+        session must not make the loop's next append wait for the whole parse.
+        The lock is taken only to look up a cached parse and to publish a new
+        one, and a parse that started before a concurrent append is discarded
+        for the newer cache entry instead of overwriting it.
+        """
+        path = self.session_path(session_id)
+        stamp = self._stamp(path) if path.is_file() else None
+        with self._cache_lock:
+            cached = self._sessions.get(session_id)
+            if cached is not None and cached[0] == stamp:
+                return cached[1]
+        if stamp is None:
+            session = Session(header=None, messages=(), compactions=(), head_id=None)
+        else:
+            session = self._parse(path)
+        with self._cache_lock:
+            cached = self._sessions.get(session_id)
+            if stamp is not None and cached is not None and cached[0] is not None and cached[0] >= stamp:
+                return cached[1]
+            self._sessions[session_id] = (stamp, session)
+        return session
+
+    def _parse(self, path: Path) -> Session:
+        """Parse one session file without touching the cache."""
+        # Bytes, not text: the parser takes them as they are, so the file is not
+        # decoded to a string only to be encoded again, and a line is only ever split
+        # where the file's own newline is. ``str.splitlines`` also breaks at U+2028,
+        # U+2029, and U+0085, which JSON writes into a string as they are — a
+        # message that held one was cut in half and the file called corrupt.
+        lines = path.read_bytes().splitlines()
+        header: SessionLine | None = None
+        messages: list[MessageLine] = []
+        compactions: list[CompactionLine] = []
+        head_id: str | None = None
+        for index, line in enumerate(lines):
+            if not line.strip():
+                continue
+            try:
+                parsed = LINES.validate_json(line)
+            except ValidationError as error:
+                if index == len(lines) - 1:
+                    break  # A crash can tear the line being appended; the rest is intact.
+                raise ValueError(f"Corrupt session file {path.name} at line {index + 1}") from error
+            match parsed:
+                case SessionLine():
+                    header = parsed
+                case MessageLine():
+                    messages.append(parsed)
+                    head_id = parsed.id
+                case CompactionLine():
+                    compactions.append(parsed)
+                case other:
+                    raise ValueError(f"Unknown session record: {type(other).__name__}")
+        self._check_links(path, messages, compactions)
+        return Session(header=header, messages=tuple(messages), compactions=tuple(compactions), head_id=head_id)
+
+    @staticmethod
+    def _stamp(path: Path) -> tuple[int, int]:
+        """Return the file's identity as mtime and size, the cache's validity key.
+
+        The log is append-only, so a larger stamp is a newer file. ``read`` uses
+        that order to discard a parse that raced an append; replacing a session
+        file with one carrying an older stamp is not a supported operation.
+        """
+        stat = path.stat()
+        return (stat.st_mtime_ns, stat.st_size)
+
+    @staticmethod
+    def _check_links(path: Path, messages: list[MessageLine], compactions: list[CompactionLine]) -> None:
+        """Reject a file whose parent links do not form the tree they claim to.
+
+        Appends only reference already-written nodes, so an unknown parent means
+        the file was edited or truncated in the middle; failing here beats
+        silently reading a broken history.
+        """
+        lines: list[MessageLine | CompactionLine] = [*messages, *compactions]
+        known = {line.id for line in lines}
+        for line in messages:
+            if line.parent is not None and line.parent not in known:
+                raise ValueError(f"Unknown parent {line.parent!r} in {path.name}")
+        for boundary in compactions:
+            if boundary.parent not in known:
+                raise ValueError(f"Unknown boundary {boundary.parent!r} in {path.name}")
+
+    def _append_line(self, session_id: str, line: Line) -> None:
+        """Append one validated line, flushing so a reader sees whole records."""
+        text = json.dumps(line.model_dump(mode="json"), ensure_ascii=False, allow_nan=False, sort_keys=True)
+        if not self._claimed:
+            claim_workspace(self.workspace_dir, self.workspace)
+            self._claimed = True
+        self.session_dir(session_id).mkdir(parents=True, exist_ok=True)
+        path = self.session_path(session_id)
+        with path.open("a", encoding="utf-8") as handle:
+            handle.write(text + "\n")
+            handle.flush()
+        self._remember(session_id, line)
+
+    def _remember(self, session_id: str, line: Line) -> None:
+        """Fold a line this process just wrote into the cache, without re-reading.
+
+        A session that was never read has no entry to update, and the next read
+        parses the file once; when there is one, the new line is spliced into the
+        parsed model and the stamp refreshed, so the append that just happened
+        never pays for the parse the cache exists to avoid.
+
+        A cache entry that does not know the line's parent is dropped rather
+        than extended: a worker may have published an empty session between the
+        append's leading read and this call, and a one-line cache would then
+        claim to be the whole history.
+        """
+        with self._cache_lock:
+            cached = self._sessions.get(session_id)
+            if cached is None:
+                return
+            _, session = cached
+            match line:
+                case SessionLine():
+                    session = replace(session, header=line)
+                case MessageLine():
+                    if (
+                        line.parent is not None
+                        and line.parent != session.head_id
+                        and session.message(line.parent) is None
+                    ):
+                        self._sessions.pop(session_id, None)
+                        return
+                    # A read on a worker may have parsed the file after this
+                    # append wrote it but before it reached this cache update.
+                    # The line is then already the tail; splicing it again
+                    # would duplicate the message. The stamp is still refreshed
+                    # below, or the next append would re-parse the whole file.
+                    if not (session.messages and session.messages[-1].id == line.id):
+                        session = replace(session, messages=(*session.messages, line), head_id=line.id)
+                case CompactionLine():
+                    if session.message(line.parent) is None:
+                        self._sessions.pop(session_id, None)
+                        return
+                    if not any(item.id == line.id for item in session.compactions):
+                        session = replace(session, compactions=(*session.compactions, line))
+            self._sessions[session_id] = (self._stamp(self.session_path(session_id)), session)
+
+    # -- the store API ------------------------------------------------------
+    async def append(
+        self,
+        session_id: str,
+        request_id: str,
+        message: AnyMessage,
+        timing: MessageTiming | None = None,
+        parent_session_id: str | None = None,
+        agent_name: str | None = None,
+        metadata: Mapping[str, object] | None = None,
+        tags: Mapping[str, object] | None = None,
+        usage: ModelUsage | None = None,
+        *,
+        parent: str | None = None,
+    ) -> str:
+        """Append one raw message and return its id, linking it into the tree.
+
+        The new message continues from ``parent`` when given, otherwise from the
+        head. Passing an older node's id forks the session: the abandoned branch
+        stays in the file and drops out of the active context, and later appends
+        continue from the fork.
+        """
+        if agent_name is not None and (not agent_name.strip() or len(agent_name) > MAX_AGENT_NAME):
+            raise ValueError(f"Agent name must contain between 1 and {MAX_AGENT_NAME} characters")
+        if usage is not None and not isinstance(message, AssistantMessage):
+            raise ValueError("Model usage belongs only to assistant messages")
+        session = self.read(session_id)
+        created_at = self._write_header(session, session_id, parent_session_id, agent_name)
+        if created_at is not None:
+            self._begin_metadata(session_id, created_at)
+        if parent is None:
+            parent = session.head_id
+        elif session.message(parent) is None:
+            raise ValueError(f"Unknown parent message: {parent!r}")
+        node_id = new_uuid7()
+        stamp = now()
+        self._append_line(
+            session_id,
+            MessageLine(
+                id=node_id,
+                parent=parent,
+                session_id=session_id,
+                request_id=request_id,
+                message=(message,),
+                timing=timing or MessageTiming.instant(),
+                usage=usage,
+                metadata=message_context(metadata, field="Message metadata"),
+                tags=message_context(tags, field="Message tags", nonempty_keys=True),
+                created_at=stamp,
+            ),
+        )
+        # Nothing is written about the session itself: the conversation file's own
+        # modification time moved with the line above, and that is the activity a list shows.
+        return node_id
+
+    async def checkpoint(
+        self,
+        session_id: str,
+        summary: CompactedMessage,
+        boundary_id: str,
+        expected_version: int,
+    ) -> CompactionLine:
+        """Append a checkpoint summarizing the branch through one message."""
+        session = self.read(session_id)
+        latest = session.latest_compaction
+        version = latest.version if latest is not None else 0
+        if version != expected_version:
+            raise ValueError("Checkpoint conflict; reload the session")
+        if session.message(boundary_id) is None:
+            raise ValueError("Checkpoint boundary must identify an existing message")
+        if latest is not None:
+            previous = session.index(latest.parent)
+            position = session.index(boundary_id)
+            if previous is not None and position is not None and position <= previous:
+                raise ValueError("Checkpoint boundary must advance beyond the previous one")
+        line = CompactionLine(
+            id=new_uuid7(),
+            parent=boundary_id,
+            version=version + 1,
+            message=(summary,),
+            created_at=now(),
+        )
+        self._append_line(session_id, line)
+        return line
+
+    async def set_title(self, session_id: str, title: str) -> None:
+        """Write one session's display title, replacing any earlier one.
+
+        The title is metadata, not conversation, so renaming rewrites the session's small
+        metadata file and never shows up in the message history. Naming a session that has
+        not been used yet creates it: the header that marks a session is written first, so a
+        name the user typed is never dropped for want of a conversation.
+        """
+        cleaned = title.strip()
+        if not cleaned or len(cleaned) > MAX_TITLE:
+            raise ValueError(f"Session title must contain between 1 and {MAX_TITLE} characters")
+        metadata = self._metadata(session_id)
+        if metadata is None:
+            created_at = self._write_header(self.read(session_id), session_id, None, None)
+            metadata = (
+                self._begin_metadata(session_id, created_at) if created_at is not None else self._metadata(session_id)
+            )
+        if metadata is not None:
+            write_session_metadata(self._metadata_file(session_id), metadata.model_copy(update={"title": cleaned}))
+
+    def fork(self, session_id: str) -> str:
+        """Copy the context one session carries into a new session, and return its id.
+
+        What is copied is the context, not the file: the newest checkpoint's summary
+        and the messages after it — the part a request would carry. A session that has
+        been compacted therefore forks into the part of it that still matters, and the
+        messages the summary replaced stay behind with the session they belong to. That
+        is what makes a fork worth having rather than a copy of the file: a long
+        conversation continues from what the model is actually working with.
+
+        The new session records where it came from and takes the source's title with
+        ``(fork)`` after it, so a picker shows the two apart. A session holding nothing
+        is refused rather than forked empty.
+
+        This is file work, like :meth:`read`, and not part of the persistence
+        extension's async contract: the shell runs it on a thread, so copying a long
+        conversation does not hold the event loop.
+
+        Args:
+            session_id: The session whose context is copied.
+
+        Raises:
+            ValueError: There is nothing to copy — the session holds no messages, or
+                never reached the store at all, which for a fork of the session the
+                reader is already in are the same answer to the same question.
+        """
+        source = self.read(session_id)
+        checkpoint, tail = source.active() if source.header is not None else (None, ())
+        if checkpoint is None and not tail:
+            raise ValueError("Nothing to fork yet: this session has no messages")
+        new_id = new_uuid7()
+        created_at = self._write_header(self.read(new_id), new_id, session_id, None)
+        if created_at is not None:
+            title = self.session_title(session_id)
+            carried = (
+                f"{title}{FORK_SUFFIX}" if title is not None and len(f"{title}{FORK_SUFFIX}") <= MAX_TITLE else None
+            )
+            # A fork carries on from where the session left off, so it uses the model that did,
+            # and keeps the choice about compaction that was made for that conversation.
+            self._begin_metadata(
+                new_id,
+                created_at,
+                title=carried,
+                model=self.session_model(session_id),
+                self_compact=self.session_self_compact(session_id),
+                permission=self.session_permission(session_id),
+            )
+        parent: str | None = None
+        if checkpoint is not None:
+            # The summary is copied as a message of its own. As a checkpoint it would
+            # need a boundary message to sit on, and the boundary it was taken at
+            # belongs to the conversation being left behind; as a message it reaches
+            # the next request exactly as the checkpoint body did.
+            parent = self._copy_message(new_id, None, cast("AnyMessage", checkpoint.message[0]), request_id="fork")
+        for line in tail:
+            parent = self._copy_message(
+                new_id,
+                parent,
+                line.message[0],
+                request_id=line.request_id,
+                timing=line.timing,
+                usage=line.usage,
+                metadata=line.metadata,
+                tags=line.tags,
+                created_at=line.created_at,
+            )
+        return new_id
+
+    def _copy_message(
+        self,
+        session_id: str,
+        parent: str | None,
+        message: AnyMessage,
+        *,
+        request_id: str,
+        timing: MessageTiming | None = None,
+        usage: ModelUsage | None = None,
+        metadata: Mapping[str, object] | None = None,
+        tags: Mapping[str, object] | None = None,
+        created_at: datetime | None = None,
+    ) -> str:
+        """Append one message copied from another session, and return its id.
+
+        Everything the line carried travels with it — its request, its timings, the
+        usage its reply reported, and the tags that mark a side question — so a forked
+        conversation reads, and counts, the way the one it came from did.
+        """
+        node_id = new_uuid7()
+        self._append_line(
+            session_id,
+            MessageLine(
+                id=node_id,
+                parent=parent,
+                session_id=session_id,
+                request_id=request_id,
+                message=(message,),
+                timing=timing or MessageTiming.instant(),
+                usage=usage,
+                metadata=dict(metadata or {}),
+                tags=dict(tags or {}),
+                created_at=created_at or now(),
+            ),
+        )
+        return node_id
+
+    def session_title(self, session_id: str) -> str | None:
+        """Return one session's stored title, or ``None`` when it is unnamed."""
+        metadata = self._metadata(session_id)
+        return metadata.title if metadata is not None else None
+
+    def session_model(self, session_id: str) -> str | None:
+        """Return the model one session last used, or ``None`` when none was recorded."""
+        metadata = self._metadata(session_id)
+        return metadata.model if metadata is not None else None
+
+    def set_model(self, session_id: str, model: str) -> bool:
+        """Record the model a session used, writing only when it is not the one already recorded.
+
+        A session is asked about once per turn and its model changes once in a long while, so
+        asking is one small read and the write is the exception. Nothing is written for a session
+        with no conversation yet — there is no folder to put the file in, and the first turn
+        records it when it ends — and a name that is empty or absurdly long is not a model.
+
+        Returns:
+            Whether the file was written.
+        """
+        cleaned = model.strip()
+        if not cleaned or len(cleaned) > MAX_MODEL:
+            return False
+        metadata = self._metadata(session_id)
+        if metadata is None or metadata.model == cleaned:
+            return False
+        return write_session_metadata(self._metadata_file(session_id), metadata.model_copy(update={"model": cleaned}))
+
+    def session_self_compact(self, session_id: str) -> bool | None:
+        """Return the choice one session made about letting the model request compaction.
+
+        ``None`` is the answer of a session that never chose — every session whose file predates the
+        setting, and every session nobody has touched it in — and means it follows the default the
+        configuration gives, which the runtime applies; the store knows nothing of the configuration.
+        """
+        metadata = self._metadata(session_id)
+        return metadata.self_compact if metadata is not None else None
+
+    def set_self_compact(self, session_id: str, choice: bool | None, *, default: bool = False) -> None:
+        """Record the choice one session made about letting the model request compaction.
+
+        The choice is metadata, not conversation: it rewrites the session's small metadata file and
+        leaves everything else in it — the title, the model, the creation time — as it was. ``None``
+        takes the choice back, so the session follows the default again. A choice made before the
+        first message is a choice about a session that does not exist yet, so a choice that *differs*
+        from the default creates it, the header going in first as it does when one is named; one that
+        matches the default, or none, records nothing, because a session that holds no choice already
+        behaves that way. A file that holds the value asked for is not rewritten.
+
+        Args:
+            session_id: The session the choice belongs to.
+            choice: ``True`` or ``False`` for the session's own choice, ``None`` to follow the default.
+            default: What a session with no choice does, to tell a choice that changes something from
+                one that does not.
+
+        Raises:
+            OSError: The metadata file could not be replaced. Nothing was saved, so a caller that
+                reported success would be reporting a setting that is not on disk.
+        """
+        metadata = self._metadata(session_id)
+        if metadata is None:
+            if choice is None or choice == default:
+                return
+            created_at = self._write_header(self.read(session_id), session_id, None, None)
+            metadata = (
+                SessionMetadata(session_id=session_id, created_at=created_at, updated_at=created_at)
+                if created_at is not None
+                else self._metadata(session_id)
+            )
+        if metadata is None or metadata.self_compact == choice:
+            return
+        updated = metadata.model_copy(update={"self_compact": choice})
+        if not write_session_metadata(self._metadata_file(session_id), updated):
+            raise OSError(f"Could not save the self-compaction setting for session {session_id!r}")
+
+    def session_permission(self, session_id: str) -> str | None:
+        """Return the permission level one session last ran at, or ``None`` when it never chose.
+
+        ``None`` is the answer of every session whose file predates the setting, and of every session nobody
+        changed the level in; a run of one of those starts at the level it was built with.
+        """
+        metadata = self._metadata(session_id)
+        return metadata.permission if metadata is not None else None
+
+    def set_permission(self, session_id: str, level: str | None, *, default: str) -> None:
+        """Record the permission level one session last ran at.
+
+        The level is metadata, not conversation: this rewrites the session's small metadata file and leaves the
+        title, the model and the times as they were. A level equal to the default records nothing — and takes back
+        a level recorded before, so a session set back to the default really does follow it — because a session
+        that holds no level behaves that way. A level recorded before the first message is about a session that
+        does not exist yet, so one that differs from the default creates it, the header going in first.
+
+        Args:
+            session_id: The session the level belongs to.
+            level: The level's name, or ``None`` to forget the choice.
+            default: What a session with no level runs at, to tell a choice that changes something from one
+                that does not.
+
+        Raises:
+            OSError: The metadata file could not be replaced, so nothing was saved.
+        """
+        stored = level if level != default else None
+        metadata = self._metadata(session_id)
+        if metadata is None:
+            if stored is None:
+                return
+            created_at = self._write_header(self.read(session_id), session_id, None, None)
+            metadata = (
+                SessionMetadata(session_id=session_id, created_at=created_at, updated_at=created_at)
+                if created_at is not None
+                else self._metadata(session_id)
+            )
+        if metadata is None or metadata.permission == stored:
+            return
+        updated = metadata.model_copy(update={"permission": stored})
+        if not write_session_metadata(self._metadata_file(session_id), updated):
+            raise OSError(f"Could not save the permission level for session {session_id!r}")
+
+    async def list_sessions(self, *, limit: int = 100, offset: int = 0) -> list[SessionInfo]:
+        """Return session metadata by latest activity, newest first.
+
+        Each row is read from the session's own metadata file, so no conversation is parsed
+        and the cost is one small file per session. A session with no file of its own is one
+        an older version made: its row is carried over from the old log, written into its
+        folder, and read from there from then on. A folder deleted by hand has no row.
+        """
+        if limit < 1:
+            raise ValueError("limit must be positive")
+        if offset < 0:
+            raise ValueError("offset cannot be negative")
+        # On a thread: the cost is a file per session, which is a few milliseconds for the
+        # hundred a person has and a fifth of a second for five thousand, and this is awaited
+        # from the interface. The first listing after an upgrade also writes a file per old
+        # session, once.
+        return await asyncio.to_thread(self._listing, limit, offset)
+
+    async def session_stats(self, session_ids: Sequence[str]) -> dict[str, SessionStats]:
+        """Return the entry count and the token totals of the given sessions.
+
+        Answered from ``sessions.json`` beside the session folders while a conversation file is as the
+        summary last saw it, so the cost of a listing does not grow with the length of the conversations
+        (see :mod:`.ledger`); a changed one is read from where it was last read. A session with no totals
+        (a subagent's, or one that cannot be read) is missing from the answer.
+        """
+        if self._ledger is None:
+            self._ledger = LedgerReader(self.root, self.workspace)
+        return await asyncio.to_thread(self._ledger.stats, tuple(session_ids))
+
+    def _listing(self, limit: int, offset: int) -> list[SessionInfo]:
+        """Read every session's row and return one page of them, newest activity first."""
+        rows: list[SessionInfo] = []
+        for session_id in self.session_ids():
+            metadata = self._metadata(session_id)
+            if metadata is not None:
+                rows.append(info_of(metadata, self.session_path(session_id)))
+        rows.sort(key=lambda info: (info.updated_at, info.session_id), reverse=True)
+        return rows[offset : offset + limit]
+
+    # -- metadata -------------------------------------------------------------
+    def _metadata_file(self, session_id: str) -> Path:
+        """Return the file that holds one session's own metadata."""
+        return self.session_dir(session_id) / SESSION_METADATA_FILE
+
+    def _begin_metadata(
+        self,
+        session_id: str,
+        created_at: datetime,
+        *,
+        title: str | None = None,
+        model: str | None = None,
+        self_compact: bool | None = None,
+        permission: str | None = None,
+    ) -> SessionMetadata:
+        """Write the metadata of a session that has just been created, and return it."""
+        metadata = SessionMetadata(
+            session_id=session_id,
+            created_at=created_at,
+            updated_at=created_at,
+            title=title,
+            model=model,
+            self_compact=self_compact,
+            permission=permission,
+        )
+        write_session_metadata(self._metadata_file(session_id), metadata)
+        return metadata
+
+    def _metadata(self, session_id: str) -> SessionMetadata | None:
+        """Return one session's metadata, from its own file when it has one.
+
+        The common case is one small read. A session with a conversation but no file — one an
+        older version made, or one whose file was lost — is carried over: the old log is asked
+        what it knew, the answer is written into the session's folder, and the next look reads
+        it from there. A session with no conversation has nothing to describe.
+
+        Returns:
+            The metadata, or ``None`` when the session has no conversation.
+        """
+        found = read_session_metadata(self._metadata_file(session_id), session_id)
+        if found is not None:
+            return found
+        header = self._header(session_id)
+        if header is None:
+            return None
+        old = self._legacy_rows().get(session_id)
+        metadata = SessionMetadata(
+            session_id=session_id,
+            created_at=old.created_at if old is not None else header.created_at,
+            updated_at=old.updated_at if old is not None else header.created_at,
+            title=old.title if old is not None else None,
+        )
+        write_session_metadata(self._metadata_file(session_id), metadata)
+        return metadata
+
+    def _legacy_rows(self) -> dict[str, SessionInfo]:
+        """Return what the old workspace log says about each session, folded once per change.
+
+        Raises:
+            ValueError: The old log has a damaged line other than its last. That is reported
+                rather than skipped: a session carried over from a log that could not be read
+                would lose its title for good.
+        """
+        path = self.legacy_metadata_path
+        try:
+            stat = path.stat()
+        except OSError:
+            self._legacy = (None, {})
+            return {}
+        stamp = (stat.st_mtime_ns, stat.st_size)
+        if self._legacy is None or self._legacy[0] != stamp:
+            self._legacy = (stamp, {info.session_id: info for info in read_metadata(path)})
+        return self._legacy[1]
+
+    async def close(self) -> None:
+        """Release nothing; every write opens and closes its own handle."""
+
+    def prune_empty(self) -> int:
+        """Remove this workspace's session folders that never held a conversation and that nobody has open.
+
+        What a launch that was killed leaves behind; see :func:`.locks.prune_empty`. Returns how many went.
+        """
+        return prune_empty(self.workspace_dir)
+
+    def _write_header(
+        self,
+        session: Session,
+        session_id: str,
+        request_parent: str | None,
+        agent_name: str | None,
+    ) -> datetime | None:
+        """Create the session line, rejecting a parent that changed after creation.
+
+        A later append that names no parent leaves the one already written alone: the
+        header's parent is set when the session is created — by a delegation, or by a
+        fork, which records the conversation it was taken from — and the runtime hands
+        it back on every append of that run. Naming a *different* parent is the mistake
+        this rejects.
+
+        Returns:
+            The new session's creation time, or ``None`` when the file already
+            had a header — which is also what tells the caller whether to start
+            the session's metadata row.
+        """
+        if session.header is not None:
+            if request_parent is not None and session.header.parent_session_id != request_parent:
+                raise ValueError("Session parent cannot change after creation")
+            return None
+        created_at = now()
+        self._append_line(
+            session_id,
+            SessionLine(
+                version=STORE_VERSION,
+                session_id=session_id,
+                parent_session_id=request_parent,
+                agent_name=agent_name,
+                created_at=created_at,
+            ),
+        )
+        return created_at
+
+
+class SessionStore(SessionPersistenceMixin, AgentExtension):
+    """The agent plugin: restore and save a conversation through the mixin.
+
+    ``SessionPersistenceMixin`` owns the files. This class adds the framework
+    hooks — restore the model context on ``on_state``, append messages and write
+    checkpoints from the event stream, and drop per-request bookkeeping on
+    success, error, or cancellation.
+
+    Args:
+        root: Store root, grouped by workspace; forwarded to the mixin.
+            ``None`` uses ``~/.zettcode/sessions``.
+        workspace: Project whose sessions this store owns; forwarded to the
+            mixin. ``None`` uses the current directory.
+    """
+
+    def __init__(self, root: str | Path | None = None, workspace: str | Path | None = None) -> None:
+        self._requests: WeakKeyDictionary[AgentRunContext, _Request] = WeakKeyDictionary()
+        super().__init__(root, workspace)
+
+    async def on_state(self, context: AgentRunContext) -> None:
+        """Restore the conversation and map every context position to a message id."""
+        checkpoint, tail = await self._restore(context)
+        ids: list[str | None] = []
+        if checkpoint is not None:
+            ids.append(checkpoint.parent)
+        ids.extend(
+            line.id
+            for line in tail
+            if line.message[0].include_in_messages and not isinstance(line.message[0], SystemMessage)
+        )
+        self._requests[context] = _Request(
+            request_id=context.config.request_id or new_uuid7(),
+            checkpoint_version=checkpoint.version if checkpoint is not None else 0,
+            context_ids=ids,
+        )
+
+    async def on_event(self, context: AgentRunContext, event: ExtensionEvent) -> None:
+        """Append freshly produced messages and checkpoint completed compactions."""
+        if isinstance(event, RunCancelledEvent):
+            self._requests.pop(context, None)
+            return
+        request = self._requests.get(context)
+        if request is None:
+            return
+        match event:
+            case MessageAppendedEvent(message=message, timing=timing, usage=usage):
+                if is_side(message):
+                    request.side = True
+                    # The question says whether the reader watched it and who asked, and every line of the run is
+                    # tagged with both: a resumed transcript restores the exchanges that were on screen and nothing
+                    # else. This reads the run's own question because the runtime publishes the input before the step's
+                    # first message is flushed. That holds for a question; a resumed run (`stream_resume`) sends a
+                    # resume point instead, so an interrupted `/btw` is stored as an ordinary turn — untagged, in the
+                    # history, and untouched by any of this. If the order ever changed for a question, the tag would
+                    # fall back to the defaults and a plugin's question would come back on screen, with nothing here to
+                    # notice.
+                    request.watched = is_watched_side(message)
+                    request.origin = str(message.attributes.get(ORIGIN, READER))
+                if request.side:
+                    # A side question is recorded, never replayed: the same flag
+                    # the runtime uses for a message that must not re-enter a
+                    # context. The live transcript is untouched — this is the
+                    # store's own copy of the message.
+                    message = replace(message, include_in_messages=False)
+                node_id: str | None = None
+                if message.persist:
+                    node_id = await self.append(
+                        cast(str, context.config.session_id),
+                        request.request_id,
+                        message,
+                        timing,
+                        parent_session_id=context.state.parent_session_id,
+                        metadata=context.metadata,
+                        # A side question's lines are tagged as well as flagged:
+                        # the tag is what a replay or an export reads, since the
+                        # flag alone would also match an instruction.
+                        tags={**context.tags, SIDE: True, ORIGIN: request.origin, WATCHED: request.watched}
+                        if request.side
+                        else context.tags,
+                        usage=usage,
+                    )
+                if message.include_in_messages and not isinstance(message, SystemMessage):
+                    request.context_ids.append(node_id)
+            case CompactionEvent():
+                await self._write_checkpoint(context, request, event)
+
+    async def on_success(self, context: AgentRunContext, result: AssistantMessage) -> None:
+        """Release the bookkeeping; messages were appended as they arrived."""
+        self._requests.pop(context, None)
+
+    async def on_error(self, context: AgentRunContext, error: Exception) -> None:
+        """Release the bookkeeping; the append-only history remains available."""
+        self._requests.pop(context, None)
+
+    async def _restore(self, context: AgentRunContext) -> tuple[CompactionLine | None, tuple[MessageLine, ...]]:
+        """Load the conversation and merge it with the current configuration.
+
+        Parsing a long JSONL file is file work with no await in it. Running it
+        on a worker leaves the loop free to finish the frame the command put up
+        before the restore began; the assignment back into the framework's
+        context stays here, where the rest of the request lifecycle lives.
+        """
+        session = await asyncio.to_thread(self.read, cast(str, context.config.session_id))
+        stored_parent = session.parent_session_id
+        configured_parent = context.state.parent_session_id
+        session_exists = session.header is not None and bool(session.messages)
+        if configured_parent is not None and session_exists and stored_parent != configured_parent:
+            raise ValueError("Stored session belongs to a different parent session")
+        context.state.parent_session_id = stored_parent or configured_parent
+        checkpoint, tail = session.active()
+        stored: list[AnyMessage] = [checkpoint.message[0]] if checkpoint is not None else []
+        # Instructions come from the current configuration; stored context
+        # contributes dialogue and checkpoints only, which is what the runtime's
+        # own ``include_in_messages`` flag means. Re-adding a persisted
+        # instruction (the system prompt, the filesystem note, the tool snippets)
+        # would duplicate it at the head of every request, bloating the prompt
+        # and breaking the provider's prefix cache from that point on.
+        stored.extend(line.message[0] for line in tail if line.message[0].include_in_messages)
+        instructions = [message for message in context.state.messages if isinstance(message, SystemMessage)]
+        context.replace_messages(
+            [*instructions, *self._provider_safe_messages(stored)],
+            emit_new=False,
+        )
+        return checkpoint, tail
+
+    @staticmethod
+    def _provider_safe_messages(messages: Sequence[AnyMessage]) -> list[AnyMessage]:
+        """Exclude interrupted tool batches from restored model context, except the last.
+
+        A process may have stopped after persisting an assistant message with
+        tool calls but before persisting every matching result; providers reject
+        that sequence. A malformed batch in the middle of the history is dropped,
+        so the rest of the restored dialogue still reaches the model. Storage
+        keeps everything.
+
+        The batch at the very end is kept as it is, results that did finish
+        included: it is the request that was interrupted, and the runtime closes
+        it when the next request starts by adding a result for each call that has
+        none. Dropping it would forget the tools that did run and whatever they
+        changed. This follows zett-agent's own persistence extension, which is
+        where the rule is written; a batch there with an orphan result (a result
+        naming no call of its own) is still dropped.
+        """
+        restored: list[AnyMessage] = []
+        index = 0
+        while index < len(messages):
+            message = messages[index]
+            if isinstance(message, ToolMessage):
+                index += 1  # an orphan result has no meaningful provider context
+                continue
+            if not isinstance(message, AssistantMessage) or not message.tool_calls:
+                restored.append(message)
+                index += 1
+                continue
+            tool_messages: list[ToolMessage] = []
+            cursor = index + 1
+            while cursor < len(messages) and isinstance(current := messages[cursor], ToolMessage):
+                tool_messages.append(current)
+                cursor += 1
+            expected = Counter(call.id for call in message.tool_calls)
+            observed = Counter(result.tool_call_id for result in tool_messages)
+            at_end = cursor == len(messages)
+            if expected == observed or (at_end and not observed - expected):
+                restored.extend((message, *tool_messages))
+            index = cursor
+        return restored
+
+    async def _write_checkpoint(
+        self,
+        context: AgentRunContext,
+        request: _Request,
+        event: CompactionEvent,
+    ) -> None:
+        """Store the summary and remap its context position to the compacted prefix."""
+        if event.compressed_from != 1 or event.compressed_to > len(request.context_ids):
+            raise ValueError("Compaction range does not match the restored context")
+        compacted = [node_id for node_id in request.context_ids[: event.compressed_to] if node_id is not None]
+        if not compacted:
+            raise ValueError("Compaction must represent at least one stored message")
+        boundary_id = compacted[-1]
+        line = await self.checkpoint(
+            cast(str, context.config.session_id),
+            CompactedMessage(content=event.summary),
+            boundary_id,
+            request.checkpoint_version,
+        )
+        request.checkpoint_version = line.version
+        request.context_ids[:] = [boundary_id, *request.context_ids[event.compressed_to :]]