nexor-runner 0.2.0__py3-none-any.whl

nexor_runner/__init__.py

@@ -0,0 +1,3 @@
+"""nexor-runner: hosts and runs AI coding agents in tmux sessions."""
+
+__version__ = "0.2.0"

nexor_runner/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

nexor_runner/activity.py

@@ -0,0 +1,477 @@
+"""Per-session activity: is the agent working, idle, or blocked on an approval prompt?
+
+Everything here is read-only and cheap. Per runtime:
+
+- Claude Code: the transcript `<claude>/projects/<cwd-slug>/<session-id>.jsonl` is append-only;
+  only its tail is read, incrementally (a stored offset, so an unchanged file costs one `stat`).
+- Codex: the rollout `<codex>/sessions/YYYY/MM/DD/rollout-<ts>-<thread-id>.jsonl`, same method.
+- OpenCode: three GETs to the local engine (`/api/session/active`, `/api/session/<id>`,
+  `/api/session/<id>/permission`), answered from a short cache.
+- A tmux pane capture (one per session that looks like it is working, plus the sessions that have
+  no readable source) adds what the files cannot say: a visible approval menu means `waiting`,
+  `esc to interrupt` means `working`.
+
+The result per session is `activity` (`working` | `idle` | `waiting` | `unknown`),
+`activity_since` (ISO 8601 UTC, or null) and `last_turn_end` (ISO 8601 UTC, or null).
+Nothing here raises into the caller: a source that cannot be read is `unknown`.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import re
+import sys
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+from . import opencode_service
+from .tmux import Tmux, TmuxError
+
+WORKING, IDLE, WAITING, UNKNOWN = "working", "idle", "waiting", "unknown"
+ACTIVITIES = (WORKING, IDLE, WAITING, UNKNOWN)
+
+WINDOWS = (256 << 10, 2 << 20)  # first look at a file: its last 256 KiB, then 2 MiB if undecided
+MAX_CHUNK = 1 << 20  # never read more than this from one file in one go
+PANE_TTL = 2.0  # seconds a pane capture is reused
+ENGINE_TTL = 3.0  # seconds the engine's answers are reused (active set, permissions)
+SESSION_TTL = 10.0  # seconds an engine session record (time.idle) is reused
+URL_TTL = 60.0  # seconds the engine URL is reused (`opencode service status` is a subprocess)
+URL_FAIL_TTL = 10.0
+MISS_TTL = 15.0  # seconds before looking again for a transcript/rollout that was not found
+LIST_TTL = 30.0  # seconds the engine session list (title lookup) is reused
+ENGINE_TIMEOUT = 2.0
+PANE_LINES = 30
+
+WORKING_PANE_RE = re.compile(r"esc to interrupt", re.IGNORECASE)
+# The numbered approval menu of Claude Code and Codex: `❯ 1. Yes ...`, `› 1. Yes, proceed (y)`.
+WAITING_PANE_RE = re.compile(r"^[ \t]*(?:[❯›>][ \t]*)?1\.[ \t]+Yes\b", re.MULTILINE)
+SKIP_PREFIXES = ("<local-command-", "<command-name>", "<command-message>")
+
+
+@dataclass
+class Act:
+    """What a source says. `since` None = the source cannot say when it began."""
+    activity: str = UNKNOWN
+    since: str | None = None
+    last_end: str | None = None
+    compacted: bool = False  # reader scratch: a compaction boundary was just seen
+
+
+def iso(value: object) -> str | None:
+    """ISO 8601 UTC with milliseconds from an ISO string or an epoch in milliseconds."""
+    try:
+        if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0:
+            dt = datetime.fromtimestamp(value / 1000, UTC)
+        elif isinstance(value, str) and value:
+            dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
+            if dt.tzinfo is None:
+                dt = dt.replace(tzinfo=UTC)
+        else:
+            return None
+    except (ValueError, OverflowError, OSError):
+        return None
+    return dt.astimezone(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def classify_pane(text: str) -> str | None:
+    """`waiting` when an approval menu is on screen, `working` when the client says it can be
+    interrupted, else None. Only the bottom of the pane counts (the menu sits there)."""
+    tail = "\n".join(text.splitlines()[-PANE_LINES:])
+    if WAITING_PANE_RE.search(tail):
+        return WAITING
+    if WORKING_PANE_RE.search(tail):
+        return WORKING
+    return None
+
+
+# --- transcript / rollout readers ---------------------------------------------------------------
+
+def _text_of(content: object) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "".join(b.get("text", "") for b in content
+                       if isinstance(b, dict) and b.get("type") == "text")
+    return ""
+
+
+def _end(st: Act, ts: str | None) -> None:
+    """A turn ended. Several records announce it (the closing assistant entry, then the turn
+    summary): the first one counts."""
+    if st.activity != IDLE:
+        st.activity, st.since, st.last_end = IDLE, ts, ts
+
+
+def claude_step(st: Act, o: dict) -> None:
+    """Fold one transcript entry into the state. Only conversation entries count: the file also
+    holds titles, hook output, attachments, cost and bridge records."""
+    if o.get("isSidechain"):  # a subagent's own thread, not this session's turn
+        return
+    ts = iso(o.get("timestamp"))
+    kind, msg = o.get("type"), o.get("message")
+    compacted, st.compacted = st.compacted, False
+    if kind == "system" and o.get("subtype") == "compact_boundary":
+        st.compacted = True  # the summary entry that follows is not a prompt
+    elif kind == "system" and o.get("subtype") == "turn_duration":
+        _end(st, ts)  # written right after the closing assistant entry
+    elif kind == "assistant" and isinstance(msg, dict):
+        stop = msg.get("stop_reason")
+        if stop in ("end_turn", "stop_sequence", "max_tokens", "refusal"):
+            _end(st, ts)
+        else:  # tool_use, or a block written before the stop reason is known
+            if st.activity != WORKING:
+                st.since = ts
+            st.activity = WORKING
+    elif kind == "user" and isinstance(msg, dict):
+        text = _text_of(msg.get("content")).lstrip()
+        if text.startswith("[Request interrupted by user"):
+            _end(st, ts)
+        elif (text.startswith(SKIP_PREFIXES) or o.get("isMeta") or o.get("isCompactSummary")
+              or compacted):
+            return  # slash-command and compaction bookkeeping: not a turn
+        else:  # a prompt or a tool result: the model has the floor
+            if st.activity != WORKING:
+                st.since = ts
+            st.activity = WORKING
+
+
+def codex_step(st: Act, o: dict) -> None:
+    """Fold one rollout line into the state: turns open with `task_started` and close with
+    `task_complete` or `turn_aborted` (event_msg payloads)."""
+    if o.get("type") != "event_msg" or not isinstance(o.get("payload"), dict):
+        return
+    ts, kind = iso(o.get("timestamp")), o["payload"].get("type")
+    if kind == "task_started":
+        if st.activity != WORKING:
+            st.since = ts
+        st.activity = WORKING
+    elif kind in ("task_complete", "turn_aborted"):
+        _end(st, ts)
+
+
+class Follower:
+    """Folds an append-only JSONL file into an `Act`, reading only what was appended.
+
+    The first look takes the tail of the file (a bigger window when the tail holds no turn
+    boundary yet); after that only bytes past the stored offset are read. A partial last line
+    is left for the next poll, and a file that shrank is read again from its tail."""
+
+    def __init__(self, path: Path, step: Callable[[Act, dict], None]):
+        self.path, self.step = path, step
+        self.offset: int | None = None
+        self.state = Act()
+
+    def _read(self, start: int, end: int, drop_first: bool) -> tuple[list[bytes], int]:
+        with self.path.open("rb") as fh:
+            fh.seek(start)
+            data = fh.read(end - start)
+        if drop_first:  # `start` may be mid-line
+            cut = data.find(b"\n")
+            if cut < 0:
+                return [], end if end - start >= MAX_CHUNK else start
+            data, start = data[cut + 1:], start + cut + 1
+        last = data.rfind(b"\n")
+        if last < 0:  # no complete line yet; a pathological one is skipped
+            return [], end if len(data) >= MAX_CHUNK else start
+        return data[:last].split(b"\n"), start + last + 1
+
+    def _fold(self, lines: list[bytes]) -> None:
+        for raw in lines:
+            if not raw.strip():
+                continue
+            try:
+                obj = json.loads(raw)
+            except ValueError:
+                continue
+            if isinstance(obj, dict):
+                self.step(self.state, obj)
+
+    def poll(self) -> Act:
+        size = self.path.stat().st_size
+        if self.offset is not None and size < self.offset:
+            self.offset, self.state = None, Act()  # truncated or replaced
+        if self.offset is None:
+            for window in WINDOWS:
+                start = max(0, size - window)
+                lines, self.offset = self._read(start, size, drop_first=start > 0)
+                self.state = Act()
+                self._fold(lines)
+                if start == 0 and self.state.activity == UNKNOWN:
+                    self.state.activity = IDLE  # the whole file, and nobody has said anything
+                if self.state.activity != UNKNOWN:
+                    break
+        elif size > self.offset:
+            start = max(self.offset, size - MAX_CHUNK)
+            lines, self.offset = self._read(start, size, drop_first=start > self.offset)
+            self._fold(lines)
+        return Act(self.state.activity, self.state.since, self.state.last_end)
+
+
+# --- the tracker --------------------------------------------------------------------------------
+
+class _Gone(Exception):
+    """The engine does not know this session (404)."""
+
+
+def _slug(cwd: str) -> str:
+    return re.sub(r"[^A-Za-z0-9]", "-", cwd)
+
+
+class ActivityTracker:
+    """Keeps the per-session caches and answers `annotate(rows)`. Thread-safe."""
+
+    def __init__(self, tmux: Tmux | None, env: Mapping[str, str],
+                 clock: Callable[[], float] = time.time,
+                 opener: Callable = urllib.request.urlopen,
+                 log: Callable[[str], None] | None = None):
+        self.tmux, self.env, self.clock, self.opener = tmux, env, clock, opener
+        self.log = log or (lambda m: print(f"[activity] {m}", file=sys.stderr, flush=True))
+        self.lock = threading.RLock()
+        self.followers: dict[tuple[str, str], Follower | None] = {}  # (client, id) -> follower
+        self.missed: dict[tuple[str, str], float] = {}  # (client, id) -> when it was not found
+        self.seen: dict[str, tuple[str, str | None]] = {}  # name -> (activity, since)
+        self.panes: dict[str, tuple[float, str | None]] = {}
+        self.engine: dict[str, tuple[float, object]] = {}  # cache key -> (when, value)
+        self.warned: set[str] = set()
+
+    # -- public ----------------------------------------------------------------------------
+
+    def annotate(self, rows: list[dict]) -> None:
+        """Add `activity`, `activity_since` and `last_turn_end` to every row (`name`, `client`,
+        `session_id`, `handle`, `project`). Never raises."""
+        with self.lock:
+            now = self.clock()
+            for r in rows:
+                try:
+                    act = self._one(r, now)
+                except (OSError, ValueError, KeyError, TypeError, AttributeError,
+                        opencode_service.EngineError, TmuxError) as exc:
+                    self._warn(f"{r.get('name')}: {type(exc).__name__}")
+                    act = Act()
+                self._stamp(r, act, now)
+            alive = {r["name"] for r in rows}
+            for gone in [n for n in self.seen if n not in alive]:
+                del self.seen[gone]
+            self._prune({(r.get("client"), r.get("session_id")) for r in rows}, now)
+
+    # -- internals -------------------------------------------------------------------------
+
+    def _warn(self, what: str) -> None:
+        if what not in self.warned and len(self.warned) < 50:  # once per distinct problem
+            self.warned.add(what)
+            self.log(f"cannot read activity: {what}")
+
+    def _prune(self, live: set[tuple[object, object]], now: float) -> None:
+        for cache in (self.followers, self.missed):
+            for key in [k for k in cache if k not in live]:
+                del cache[key]
+        for cache in (self.engine, self.panes):  # entries nobody asked for in a while
+            for key in [k for k, v in cache.items() if now - v[0] > 600]:
+                del cache[key]
+
+    def _stamp(self, row: dict, act: Act, now: float) -> None:
+        since = act.since
+        if act.activity == UNKNOWN:
+            since = None
+        elif since is None:  # the source cannot say: keep what was seen, else when it changed
+            prev = self.seen.get(row["name"])
+            since = prev[1] if prev and prev[0] == act.activity and prev[1] else iso(now * 1000)
+        self.seen[row["name"]] = (act.activity, since)
+        row["activity"], row["activity_since"], row["last_turn_end"] = (
+            act.activity, since, act.last_end)
+
+    def _one(self, row: dict, now: float) -> Act:
+        client = row.get("client")
+        if client == "cld":
+            act = self._file_act(row, now, "cld")
+        elif client == "cdx":
+            act = self._file_act(row, now, "cdx")
+        elif client == "oc":
+            return self._oc_act(row, now)
+        else:
+            return Act()
+        if act is None:  # no transcript/rollout: the screen is all there is
+            pane = self._pane(row["name"], now)
+            return Act(pane) if pane in (WORKING, WAITING, IDLE) else Act()
+        if act.activity == WORKING and self._pane(row["name"], now) == WAITING:
+            return Act(WAITING, None, act.last_end)
+        return act
+
+    def _pane(self, name: str, now: float) -> str | None:
+        """`working` / `waiting` / `idle` (a readable pane that shows neither) / None (no pane)."""
+        hit = self.panes.get(name)
+        if hit and now - hit[0] < PANE_TTL:
+            return hit[1]
+        value = None
+        if self.tmux is not None:
+            try:
+                text = self.tmux.capture_tail(name, PANE_LINES)
+            except TmuxError:
+                text = ""
+            value = classify_pane(text) or ("idle" if text.strip() else None)
+        self.panes[name] = (now, value)
+        return value
+
+    # -- Claude Code and Codex: files ------------------------------------------------------
+
+    def _claude_path(self, sid: str, cwd: str | None) -> Path | None:
+        home = Path(self.env.get("CLAUDE_CONFIG_DIR") or Path.home() / ".claude")
+        projects = home / "projects"
+        if cwd:
+            direct = projects / _slug(cwd) / f"{sid}.jsonl"
+            if direct.is_file():
+                return direct
+        return next(iter(projects.glob(f"*/{sid}.jsonl")), None)
+
+    def _codex_path(self, sid: str) -> Path | None:
+        home = Path(self.env.get("CODEX_HOME") or Path.home() / ".codex")
+        for f in (home / "sessions").glob(f"*/*/*/rollout-*-{sid}.jsonl"):
+            if self._guardian(f):
+                continue
+            return f
+        return None
+
+    @staticmethod
+    def _guardian(path: Path) -> bool:
+        """A guardian_review rollout is the approval reviewer's own thread, not a session."""
+        try:
+            with path.open("rb") as fh:
+                head = fh.read(65536).split(b"\n", 1)[0]
+            meta = json.loads(head).get("payload") or {}
+        except (OSError, ValueError, AttributeError):
+            return False
+        return meta.get("thread_source") == "guardian_review"
+
+    def _file_act(self, row: dict, now: float, client: str) -> Act | None:
+        sid = row.get("session_id")
+        if not isinstance(sid, str) or not sid or not re.fullmatch(r"[A-Za-z0-9_-]{8,64}", sid):
+            return None
+        key = (client, sid)
+        if key not in self.followers:
+            if now - self.missed.get(key, -1e9) < MISS_TTL:
+                return None
+            path = (self._claude_path(sid, row.get("project")) if client == "cld"
+                    else self._codex_path(sid))
+            if path is None:
+                self.missed[key] = now
+                return None
+            self.followers[key] = Follower(path, claude_step if client == "cld" else codex_step)
+        follower = self.followers[key]
+        try:
+            return follower.poll()
+        except FileNotFoundError:  # the file went away: look it up again next time
+            del self.followers[key]
+            self.missed[key] = now
+            return None
+
+    # -- OpenCode: the engine --------------------------------------------------------------
+
+    def _cached(self, key: str, ttl: float, now: float, fetch: Callable[[], object]) -> object:
+        hit = self.engine.get(key)
+        if hit and now - hit[0] < ttl:
+            return hit[1]
+        value = fetch()
+        self.engine[key] = (now, value)
+        return value
+
+    def _engine_url(self, now: float) -> str:
+        hit = self.engine.get("url")
+        if hit and now - hit[0] < (URL_TTL if hit[1] else URL_FAIL_TTL):
+            if hit[1]:
+                return hit[1]
+            raise opencode_service.EngineError("the OpenCode engine is not running")
+        try:
+            url = opencode_service.status_url(self.env)
+        except opencode_service.EngineError:
+            url = None
+        self.engine["url"] = (now, url)
+        if not url:
+            raise opencode_service.EngineError("the OpenCode engine is not running")
+        return url
+
+    def _get(self, path: str, now: float) -> object:
+        down = self.engine.get("down")
+        if down and now - down[0] < URL_FAIL_TTL:  # one timeout per window, not one per row
+            raise opencode_service.EngineError("the engine is not reachable")
+        url = self._engine_url(now)
+        pw = opencode_service.engine_password(self.env)
+        auth = "Basic " + base64.b64encode(f"opencode:{pw}".encode()).decode()
+        req = urllib.request.Request(url.rstrip("/") + path, headers={"Authorization": auth},
+                                     method="GET")
+        try:
+            with self.opener(req, timeout=ENGINE_TIMEOUT) as resp:
+                body = json.loads(resp.read() or b"null")
+        except urllib.error.HTTPError as exc:
+            if exc.code == 404:
+                raise _Gone from None
+            raise opencode_service.EngineError(f"the engine answered HTTP {exc.code}") from None
+        except (urllib.error.URLError, OSError, ValueError) as exc:
+            self.engine.pop("url", None)  # it may have moved (the engine restarted)
+            self.engine["down"] = (now, True)
+            raise opencode_service.EngineError(
+                f"the engine is not reachable: {type(exc).__name__}") from None
+        return body.get("data") if isinstance(body, dict) else None
+
+    def _oc_resolve(self, row: dict, now: float) -> str | None:
+        """The engine session id: the row's own, else the newest engine session titled with the
+        handle (the runner titles its sessions with it; see oc_map)."""
+        sid = row.get("session_id")
+        if isinstance(sid, str) and sid.startswith("ses_") and re.fullmatch(r"[\w-]+", sid):
+            try:
+                self._cached(f"s:{sid}", SESSION_TTL, now,
+                             lambda: self._get(f"/api/session/{sid}", now))
+                return sid
+            except _Gone:
+                self.engine.pop(f"s:{sid}", None)
+        handle = row.get("handle")
+        if not handle:
+            return None
+        listing = self._cached("list", LIST_TTL, now, lambda: opencode_service.list_sessions(
+            self.env, self._engine_url(now), timeout=ENGINE_TIMEOUT, opener=self.opener,
+            page=100, max_pages=1))
+        titled = [s for s in listing if s["title"] == handle]
+        return max(titled, key=lambda s: s["stamp"])["id"] if titled else None
+
+    def _oc_act(self, row: dict, now: float) -> Act:
+        sid = self._oc_resolve(row, now)
+        if sid is None:
+            return Act()
+        try:
+            rec = self._cached(f"s:{sid}", SESSION_TTL, now,
+                               lambda: self._get(f"/api/session/{sid}", now))
+            active = self._cached("active", ENGINE_TTL, now,
+                                  lambda: self._get("/api/session/active", now))
+            blocked = self._cached(f"p:{sid}", ENGINE_TTL, now, lambda: self._blocked(sid, now))
+        except _Gone:
+            return Act()
+        idle = rec.get("time", {}).get("idle") if isinstance(rec, dict) else None
+        last_end = iso(idle)
+        if blocked:
+            return Act(WAITING, None, last_end)
+        if isinstance(active, dict) and sid in active:
+            return Act(WORKING, None, last_end)
+        if isinstance(active, dict):
+            if idle is not None and self.seen.get(row["name"], ("",))[0] in (WORKING, WAITING):
+                # just finished: the cached record predates the turn end
+                self.engine.pop(f"s:{sid}", None)
+                rec = self._cached(f"s:{sid}", SESSION_TTL, now,
+                                   lambda: self._get(f"/api/session/{sid}", now))
+                idle = rec.get("time", {}).get("idle") if isinstance(rec, dict) else idle
+                last_end = iso(idle)
+            return Act(IDLE, last_end, last_end)
+        return Act()
+
+    def _blocked(self, sid: str, now: float) -> bool:
+        for what in ("permission", "form"):  # an approval request, or a question to the user
+            pending = self._get(f"/api/session/{sid}/{what}", now)
+            if isinstance(pending, list) and pending:
+                return True
+        return False
+