zettcode-engine 0.3.4__py3-none-any.whl

zettcode/engine/server_tools.py

@@ -0,0 +1,179 @@
+"""What a provider-run tool left behind: the rows for a hosted search, live and replayed.
+
+A hosted tool is not called by the agent: the endpoint runs it, and the framework reports the call and its result as
+events. Those events are not messages, so the store never sees them; but the provider's own blocks are kept as the
+answer's opaque ``replay_blocks``, which the store writes and reads back. This module is the one place that turns
+either of those into the words of a row, so a resumed session shows what the live one showed — except a hosted call
+the provider closed without a result: the runtime strips that call's block so the next request is accepted, so the row
+that explains it exists only in the live transcript.
+
+The result of a search is a list of pages, each with a title, a URL and an opaque encrypted payload that must never be
+put in front of the reader or sent back in a prompt as text: only the title and the URL are shown here.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from zett_agent.model import NOT_REPORTED_STATUS
+
+#: The most pages one row lists. A row is a note that a search happened, not the search's own page.
+MAX_RESULTS = 5
+
+#: Result blocks whose payload is the provider's own and is never shown as text.
+_SEARCH_RESULT_TYPES = frozenset({"web_search_result", "search_result"})
+
+#: Block types that record a hosted call rather than text.
+_CALL_TYPES = frozenset({"server_tool_use"})
+
+
+@dataclass(frozen=True, slots=True)
+class ServerToolRun:
+    """One hosted tool call as the transcript needs it.
+
+    Attributes:
+        call_id: The provider's identifier for the call.
+        name: The tool's name, such as ``web_search``.
+        arguments: What it was asked, when the provider said; ``{"query": ...}`` for a search.
+        output: What it returned, as the provider reported it.
+        error: The provider's error code when the tool failed.
+    """
+
+    call_id: str
+    name: str
+    arguments: Mapping[str, Any]
+    output: Any = None
+    error: str | None = None
+
+
+def _results_in(output: Any) -> list[Mapping[str, Any]]:
+    """Return the page blocks of a search result, wherever the provider put them.
+
+    The adapter hands a hosted tool's output over as the block's own ``content``: a list of pages for a search, with
+    the provider's encrypted payload beside each title and URL.
+    """
+    match output:
+        case bool() | str() | int() | float() | None:
+            return []
+        case Mapping():
+            content = output.get("content")
+            if isinstance(content, Iterable) and not isinstance(content, str):
+                return [item for item in content if isinstance(item, Mapping)]
+            if output.get("type") in _SEARCH_RESULT_TYPES:
+                return [output]
+            return []
+        case Iterable():
+            return [item for item in output if isinstance(item, Mapping)]
+        case _:
+            return []
+
+
+def describe_search(output: Any) -> str:
+    """Return a search result as the few lines a row shows: how many pages, and which.
+
+    Only a page's title and URL are shown; the encrypted payload the provider returns beside them is its own business
+    and is neither printed nor kept.
+    """
+    pages = _results_in(output)
+    if not pages:
+        return "No pages"
+    lines = [f"{len(pages)} page{'s' if len(pages) != 1 else ''}"]
+    for page in pages[:MAX_RESULTS]:
+        title = page.get("title")
+        url = page.get("url")
+        named = title.strip() if isinstance(title, str) and title.strip() else ""
+        address = url.strip() if isinstance(url, str) and url.strip() else ""
+        lines.append(f"- {named} \u2014 {address}" if named and address else f"- {named or address}")
+    if len(pages) > MAX_RESULTS:
+        lines.append(f"\u2026 and {len(pages) - MAX_RESULTS} more")
+    return "\n".join(lines)
+
+
+def not_reported(output: Any) -> bool:
+    """Return whether a hosted call's output is the adapter's marker that no result was reported.
+
+    ``zett-agent`` closes a call the provider left open with ``output={"status": NOT_REPORTED_STATUS}``. The status is
+    the adapter's own, never a provider result shape, so that one key is read as the marker here and in the event the
+    live row is closed with.
+    """
+    return isinstance(output, Mapping) and output.get("status") == NOT_REPORTED_STATUS
+
+
+def describe_output(run: ServerToolRun) -> str:
+    """Return what one hosted tool said, in the words of its own kind.
+
+    A call the provider left open is closed by the runtime with ``output={"status": "not_reported"}``: the call
+    ended, but nothing about what it found was reported, so the row says that rather than claiming a result (or, as it
+    once did, showing the word ``Completed`` for a search that never answered).
+    """
+    if run.error:
+        return run.error
+    if not_reported(run.output):
+        return "The provider did not report what this call returned"
+    if run.name == "web_search" or _results_in(run.output):
+        return describe_search(run.output)
+    # Another provider's hosted tool: its payload is already text or a plain structure.
+    return run.output if isinstance(run.output, str) else ""
+
+
+def runs_in_blocks(blocks: Iterable[Mapping[str, Any]]) -> list[ServerToolRun]:
+    """Return the hosted tool calls one answer's ``replay_blocks`` record, results paired with their calls.
+
+    A block that is not a hosted call is skipped: a thinking block, or the answer's text. A call whose result is not
+    in the same answer keeps an empty output, which reads as a search that said nothing.
+    """
+    calls: dict[str, ServerToolRun] = {}
+    order: list[str] = []
+    for block in blocks:
+        match block.get("type"):
+            case "server_tool_use":
+                call_id = block.get("id")
+                name = block.get("name")
+                if not isinstance(call_id, str) or not isinstance(name, str):
+                    continue
+                call_input = block.get("input")
+                arguments = dict(call_input) if isinstance(call_input, Mapping) else {}
+                calls[call_id] = ServerToolRun(call_id, name, arguments)
+                order.append(call_id)
+            case "web_search_tool_result" | "server_tool_result" | "web_fetch_tool_result":
+                found = block.get("tool_use_id") or block.get("call_id")
+                if not isinstance(found, str) or found not in calls:
+                    continue
+                error = block.get("error_code")
+                calls[found] = ServerToolRun(
+                    found,
+                    calls[found].name,
+                    calls[found].arguments,
+                    output=block,
+                    error=str(error) if isinstance(error, str) and error else None,
+                )
+    return [calls[call_id] for call_id in order]
+
+
+def answer_after_tools(blocks: Iterable[Mapping[str, Any]]) -> bool:
+    """Return whether the answer's text came after its hosted tool, as the provider streamed them.
+
+    Anthropic runs a hosted search before it writes the answer, so the row belongs above it; a provider that writes
+    first and searches after (DeepSeek does, when it announces the search) has it below. The blocks record which.
+    """
+    first_tool: int | None = None
+    for index, block in enumerate(blocks):
+        kind = block.get("type")
+        if first_tool is None and kind in _CALL_TYPES:
+            first_tool = index
+        elif kind == "text" and isinstance(block.get("text"), str) and block["text"].strip():
+            return first_tool is not None
+    return False
+
+
+__all__ = [
+    "MAX_RESULTS",
+    "ServerToolRun",
+    "answer_after_tools",
+    "describe_output",
+    "describe_search",
+    "not_reported",
+    "runs_in_blocks",
+]

zettcode/engine/storage/__init__.py

@@ -0,0 +1,51 @@
+"""Session storage: the conversation log, its metadata, and the agent plugin.
+
+One folder per workspace, one folder per session, and everything about a session in its own::
+
+    ~/.zettcode/sessions/<workspace path, levels joined with ->/
+    ├── workspace.txt               <- the path the folder belongs to (see :mod:`layout`)
+    ├── metadata.jsonl              <- the old index; read for sessions that predate their own file
+    ├── usage.json                  <- what ``/usage`` has counted so far, so it reads only what is new (see :mod:`ledger`)
+    ├── sessions.json               <- each session's entry count and token totals, so ``/resume`` reads no conversation
+    └── <session-id>/
+        ├── data.jsonl              <- the append-only message tree
+        ├── metadata.json           <- title and times: written on creation and rename only
+        ├── session.lock            <- held by the process that has the session open (see :mod:`locks`)
+        └── tool-output/            <- long output any tool produced, kept for the model to read
+
+:mod:`records` owns the conversation's record types and the parsed read model,
+:mod:`metadata` owns the per-session file and the old log, and :mod:`store` puts them
+behind one API: the persistence mixin and the :class:`SessionStore` plugin. :mod:`layout` names the
+workspace folder, and renames one that an older version named with base64.
+"""
+
+from .layout import claim_workspace, find_workspace, legacy_workspace_key, open_workspace, workspace_key
+from .ledger import SessionStats
+from .locks import LOCK_FILE, LOCKED_MESSAGE, NoLocks, SessionGuard, SessionLocked, SessionLocks, prune_empty
+from .metadata import MAX_TITLE, SessionInfo
+from .records import STORE_FILE, MessageLine, Session, now
+from .store import SessionPersistenceMixin, SessionStore
+
+__all__ = [
+    "LOCKED_MESSAGE",
+    "LOCK_FILE",
+    "MAX_TITLE",
+    "MessageLine",
+    "STORE_FILE",
+    "NoLocks",
+    "Session",
+    "SessionGuard",
+    "SessionInfo",
+    "SessionLocked",
+    "SessionLocks",
+    "SessionPersistenceMixin",
+    "SessionStats",
+    "SessionStore",
+    "claim_workspace",
+    "find_workspace",
+    "legacy_workspace_key",
+    "now",
+    "open_workspace",
+    "prune_empty",
+    "workspace_key",
+]

zettcode/engine/storage/layout.py

@@ -0,0 +1,161 @@
+"""Which folder under the store root holds one workspace's sessions.
+
+A workspace's folder is named after its path, the levels joined with ``-``::
+
+    /Users/me/projects/api   ->   ~/.zettcode/sessions/-Users-me-projects-api/
+
+so ``ls ~/.zettcode/sessions`` says which folder is which project. Older versions named it with the
+URL-safe base64 of the path (``L1VzZXJzL21l...``); a store written that way is **moved** to the new
+name the first time it is opened, with one rename, which is atomic: a reader sees the folder under
+the old name or under the new one and never half of each.
+
+The move is a courtesy, not a requirement, and it is skipped whenever it is not safe:
+
+* a session in the folder is open in another process (its ``session.lock`` is held), or the system
+  refuses the rename because a file in the folder is open (Windows does), and
+* the rename fails for any other reason.
+
+In every one of those the workspace **keeps its old name** and is used from there, exactly as before;
+the next start tries again. Nothing is copied, so there is never a second copy to reconcile.
+
+Joining with ``-`` is not injective (``/a-b/c`` and ``/a/b-c`` both read ``-a-b-c``), so each folder
+records the path it belongs to in :data:`OWNER_FILE`. A folder that names a different path is not
+shared: the workspace that finds it taken falls back to its base64 name, which is unique.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import os
+import re
+from pathlib import Path
+
+from zettcode.filelock import is_held
+
+from .locks import LOCK_FILE
+
+#: Longest workspace folder name; a longer path keeps a hash instead of its tail.
+MAX_KEY_CHARS = 120
+
+#: The file in a workspace folder that says which path the folder belongs to.
+OWNER_FILE = "workspace.txt"
+
+#: Everything a folder name keeps as it is; any other character becomes ``-``.
+_UNSAFE = re.compile(r"[^\w.-]")
+
+
+def _resolve(workspace: str | Path) -> Path:
+    return Path(workspace).expanduser().resolve()
+
+
+def legacy_workspace_key(workspace: str | Path) -> str:
+    """Return the name older versions gave the workspace's folder: the base64 of its path.
+
+    The URL-safe base64 of the resolved absolute path without padding. A path too long for one file
+    name keeps a short digest instead of its tail.
+    """
+    encoded = base64.urlsafe_b64encode(str(_resolve(workspace)).encode()).decode().rstrip("=")
+    if len(encoded) <= MAX_KEY_CHARS:
+        return encoded
+    digest = hashlib.sha256(encoded.encode()).hexdigest()[:12]
+    return f"{encoded[: MAX_KEY_CHARS - 13]}-{digest}"
+
+
+def workspace_key(workspace: str | Path) -> str:
+    """Return the folder name that groups one workspace's sessions: its path with ``-`` between the levels.
+
+    Every character that is not a letter, a digit, ``_``, ``.`` or ``-`` becomes ``-`` (so the path
+    separators, a drive's colon and a space all do), which keeps the name one folder and legal on every
+    platform. A name too long for one file keeps a short digest of the whole instead of its tail.
+    """
+    text = _UNSAFE.sub("-", str(_resolve(workspace)))
+    if len(text) <= MAX_KEY_CHARS:
+        return text
+    digest = hashlib.sha256(str(_resolve(workspace)).encode()).hexdigest()[:12]
+    return f"{text[: MAX_KEY_CHARS - 13]}-{digest}"
+
+
+def _owner(folder: Path) -> str | None:
+    """Return the path a folder says it belongs to, or ``None`` when it does not say."""
+    try:
+        return (folder / OWNER_FILE).read_text(encoding="utf-8").strip() or None
+    except (OSError, UnicodeError):
+        return None
+
+
+def _is_ours(folder: Path, path: Path) -> bool:
+    """Say whether a folder under the readable name may hold this workspace's sessions.
+
+    A folder that records no owner is taken as ours: it was made by a run that stopped before it wrote
+    the note, and only a path joined to the same name could be the other claimant.
+    """
+    owner = _owner(folder)
+    return owner is None or owner == str(path)
+
+
+def _claim(folder: Path, path: Path) -> None:
+    """Write the note that says the folder is this path's, if it has none. Never fails."""
+    if _owner(folder) is not None:
+        return
+    try:
+        (folder / OWNER_FILE).write_text(f"{path}\n", encoding="utf-8")
+    except OSError:
+        pass
+
+
+def _in_use(folder: Path) -> bool:
+    """Say whether a process has a session in the folder open (its lock is held)."""
+    try:
+        locks = sorted(folder.glob(f"*/{LOCK_FILE}"))
+    except OSError:
+        return True
+    return any(is_held(lock) for lock in locks)
+
+
+def _choose(root: Path, path: Path, *, move: bool) -> Path:
+    """Return the folder to use, moving the old one to the new name first when ``move`` allows."""
+    readable = root / workspace_key(path)
+    legacy = root / legacy_workspace_key(path)
+    if readable.is_dir():
+        return readable if _is_ours(readable, path) else legacy
+    if legacy == readable or not legacy.is_dir():
+        return readable
+    if not move or _in_use(legacy):
+        return legacy
+    try:
+        os.rename(legacy, readable)
+    except OSError:
+        # Another process may have moved it a moment ago; otherwise the system said no (a file is
+        # open), and the workspace stays where it was.
+        return readable if readable.is_dir() and _is_ours(readable, path) else legacy
+    _claim(readable, path)
+    return readable
+
+
+def find_workspace(root: str | Path, workspace: str | Path) -> Path:
+    """Return the folder a workspace's sessions are in now, changing nothing on disk.
+
+    For a reader that must not create anything: it names the readable folder, the old one when that
+    is where the sessions still are, and the readable one when there are none yet.
+    """
+    return _choose(Path(root), _resolve(workspace), move=False)
+
+
+def open_workspace(root: str | Path, workspace: str | Path) -> Path:
+    """Return the folder a workspace's sessions are written in, moving an old one to the readable name.
+
+    An old base64 folder is renamed to the readable name when nothing has it open, and stays where it is
+    when something does (see the module's notes). **Nothing is created**: the folder may not exist yet, and
+    a project that is opened and never used leaves no trace in the store. :func:`claim_workspace` makes it
+    when the first record is written.
+    """
+    return _choose(Path(root), _resolve(workspace), move=True)
+
+
+def claim_workspace(folder: Path, workspace: str | Path) -> None:
+    """Create a workspace's folder if it is not there, and write the note saying whose it is."""
+    path = _resolve(workspace)
+    folder.mkdir(parents=True, exist_ok=True)
+    if folder.name == workspace_key(path):
+        _claim(folder, path)