keepwatch 0.1.1__py3-none-any.whl

keepwatch/__init__.py

@@ -0,0 +1,11 @@
+"""keepwatch — keep work alive when the process that started it is not.
+
+A task is a JSON file that moves between three folders. Whoever holds it holds a
+lease; a dead holder's lease expires and the task comes back with a trail. A task
+is only ``done`` when the ledger holds a ``verify`` record with ``rc == 0``.
+
+Protocol: docs/PROTOCOL.md. Story: examples/two_workers.py.
+"""
+
+__version__ = "0.1.1"
+__all__ = ["__version__"]

keepwatch/cli.py

@@ -0,0 +1,242 @@
+"""Command line interface. Every command maps to one function in ``core``."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+from . import __version__, core
+
+
+def _default_root() -> str:
+    return os.environ.get("KEEPWATCH_ROOT", ".")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="keepwatch",
+        description="Keep agent work alive when the process that started it is not.",
+    )
+    parser.add_argument("--root", default=_default_root(), help="work root (default: $KEEPWATCH_ROOT or .)")
+    parser.add_argument("--json", action="store_true", help="machine readable output")
+    parser.add_argument("--version", action="version", version=f"keepwatch {__version__}")
+
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_init = sub.add_parser("init", help="create the layout for some agents")
+    p_init.add_argument("agents", nargs="+", metavar="AGENT")
+
+    p_send = sub.add_parser("send", help="hand a task to an agent")
+    p_send.add_argument("--from", dest="sender", required=True)
+    p_send.add_argument("--to", required=True)
+    p_send.add_argument("--subject", required=True)
+    p_send.add_argument("--body", default="")
+    p_send.add_argument("--body-file")
+
+    p_check = sub.add_parser("check", help="list tasks you can pick up")
+    p_check.add_argument("--who", required=True)
+
+    p_claim = sub.add_parser("claim", help="take a task (and hold a lease on it)")
+    p_claim.add_argument("--who", required=True)
+    p_claim.add_argument("--id", required=True)
+    p_claim.add_argument("--ttl", type=int, default=core.DEFAULT_TTL_SECONDS)
+
+    p_renew = sub.add_parser("renew", help="extend your own lease on a task")
+    p_renew.add_argument("--who", required=True)
+    p_renew.add_argument("--id", required=True)
+    p_renew.add_argument("--ttl", type=int, default=core.DEFAULT_TTL_SECONDS)
+
+    p_done = sub.add_parser("done", help="close a task (requires a passing verify)")
+    p_done.add_argument("--who", required=True)
+    p_done.add_argument("--id", required=True)
+    p_done.add_argument("--note", default="")
+    p_done.add_argument("--force", action="store_true", help="close without a verify record")
+
+    p_verify = sub.add_parser("verify", help="record a check result against a task")
+    p_verify.add_argument("--id", required=True)
+    p_verify.add_argument("--name", required=True)
+    p_verify.add_argument("--rc", type=int, required=True)
+
+    p_run = sub.add_parser(
+        "run",
+        help="claim a task, run a command, record the exit code and close or release it",
+        usage="keepwatch run --who AGENT [--id ID] [--ttl SECONDS] [--max-tries N] -- COMMAND ...",
+    )
+    p_run.add_argument("--who", required=True)
+    p_run.add_argument("--id", help="task id (default: the oldest pending task)")
+    p_run.add_argument("--ttl", type=int, default=core.DEFAULT_TTL_SECONDS)
+    p_run.add_argument("--max-tries", type=int, default=3)
+    p_run.add_argument("cmd", nargs=argparse.REMAINDER)
+
+    sub.add_parser("reap", help="return expired leases to pending")
+
+    p_status = sub.add_parser("status", help="what is stuck, who is dead, what is next")
+    p_status.add_argument("--stale", type=int, default=core.DEFAULT_STALE_SECONDS)
+
+    return parser
+
+
+def _emit(args: argparse.Namespace, payload: Any, human: list[str]) -> None:
+    if args.json:
+        print(json.dumps(payload, ensure_ascii=False, indent=2))
+    else:
+        for line in human:
+            print(line)
+
+
+def _short(text: str, width: int = 40) -> str:
+    text = (text or "").replace("\n", " ")
+    return text if len(text) <= width else text[: width - 1] + "…"
+
+
+def run(args: argparse.Namespace) -> int:
+    store = core.Store(args.root)
+
+    if args.command == "init":
+        store.init(args.agents)
+        _emit(args, {"root": str(store.root), "agents": args.agents},
+              [f"initialised {store.root}", "agents: " + ", ".join(args.agents)])
+        return core.EXIT_OK
+
+    if args.command == "send":
+        body = args.body
+        if args.body_file:
+            body = Path(args.body_file).read_text(encoding="utf-8")
+        task = core.send(store, args.sender, args.to, args.subject, body)
+        _emit(args, task, [f"sent {task['id']} -> {args.to}: {args.subject}"])
+        return core.EXIT_OK
+
+    if args.command == "check":
+        core.heartbeat(store, args.who)
+        tasks = core.check(store, args.who)
+        human = [f"{len(tasks)} task(s) for {args.who}"]
+        for t in tasks:
+            age = int(core.age_seconds(t["created_at"]))
+            human.append(f"  {t['id']}  {age:>6}s  from {t['from']:<12} {_short(t['subject'])}")
+        _emit(args, {"who": args.who, "tasks": tasks}, human)
+        return core.EXIT_OK
+
+    if args.command == "claim":
+        core.heartbeat(store, args.who)
+        task = core.claim(store, args.who, args.id, args.ttl)
+        lease = task["_lease"]
+        _emit(args, task, [
+            f"claimed {task['id']} (try {task['tries']}) lease until {lease['expires_at']}",
+        ])
+        return core.EXIT_OK
+
+    if args.command == "renew":
+        core.heartbeat(store, args.who)
+        task = core.renew(store, args.who, args.id, args.ttl)
+        _emit(args, task, [f"renewed {task['id']} until {task['_lease']['expires_at']}"])
+        return core.EXIT_OK
+
+    if args.command == "done":
+        core.heartbeat(store, args.who)
+        task = core.done(store, args.who, args.id, args.note, args.force)
+        _emit(args, task, [f"closed {task['id']}"])
+        return core.EXIT_OK
+
+    if args.command == "verify":
+        record = core.verify(store, args.id, args.name, args.rc)
+        _emit(args, record, [f"recorded {args.name} rc={args.rc} for {args.id}"])
+        return core.EXIT_OK
+
+    if args.command == "run":
+        return run_task(store, args)
+
+    if args.command == "reap":
+        tasks = core.reap(store)
+        human = [f"reaped {len(tasks)} task(s)"]
+        for t in tasks:
+            human.append(f"  {t['id']}  back to pending (tries={t['tries']})")
+        _emit(args, {"reaped": tasks}, human)
+        return core.EXIT_OK
+
+    if args.command == "status":
+        state = core.status(store, args.stale)
+        human = [f"root: {state['root']}"]
+        for name, row in state["agents"].items():
+            if row["heartbeat_age_s"] is None:
+                hb, flag = "never seen", "unknown"
+            elif row["alive"]:
+                hb, flag = f"hb {row['heartbeat_age_s']}s", "alive"
+            else:
+                hb, flag = f"hb {row['heartbeat_age_s']}s", "STALE"
+            human.append(
+                f"  {name:<14} pending {row['pending']}  processing {row['processing']}"
+                f"  done {row['done']}  {hb} ({flag})"
+            )
+        if state["next_steps"]:
+            human.append("next:")
+            human.extend(f"  - {line}" for line in state["next_steps"])
+        else:
+            human.append("next: nothing to do")
+        _emit(args, state, human)
+        return core.EXIT_OK if state["ok"] else core.EXIT_BLOCKED
+
+    raise core.keepwatchError(f"unknown command: {args.command}")
+
+
+def run_task(store: core.Store, args: argparse.Namespace) -> int:
+    """Claim -> run -> verify -> close or release. The one command an agent needs."""
+    cmd = list(args.cmd)
+    if cmd and cmd[0] == "--":
+        cmd = cmd[1:]
+    if not cmd:
+        raise core.keepwatchError(
+            "no command given; use: keepwatch run --who AGENT -- COMMAND ..."
+        )
+
+    core.heartbeat(store, args.who)
+    task_id = args.id
+    if not task_id:
+        task = core.oldest_pending(store, args.who)
+        if task is None:
+            raise core.NotFound(f"no pending task for {args.who}")
+        task_id = task["id"]
+
+    claimed = core.claim(store, args.who, task_id, args.ttl)
+    label = " ".join(cmd)
+    print(f"running {task_id} (try {claimed['tries']}): {label}", flush=True)
+
+    # stdio is inherited on purpose: the agent's output must reach the human.
+    rc = subprocess.run(cmd, check=False).returncode
+    core.verify(store, task_id, label, rc)
+
+    if rc == 0:
+        core.done(store, args.who, task_id, note=label)
+        print(f"closed {task_id} (rc=0)", flush=True)
+        return core.EXIT_OK
+
+    if claimed["tries"] >= args.max_tries:
+        core.ledger_append(store, "suspect", id=task_id, tries=claimed["tries"], rc=rc)
+        print(
+            f"left {task_id} in processing: rc={rc}, tries={claimed['tries']} >= {args.max_tries}. "
+            f"Check `status`, then `reap` if it should be retried anyway.",
+            flush=True,
+        )
+        return rc if 0 < rc < 256 else core.EXIT_ERROR
+
+    core.release(store, args.who, task_id, reason=f"rc={rc}")
+    print(f"released {task_id} back to pending (rc={rc})", flush=True)
+    return rc if 0 < rc < 256 else core.EXIT_ERROR
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return run(args)
+    except core.keepwatchError as exc:
+        print(f"keepwatch: {exc}", file=sys.stderr)
+        return exc.code
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

keepwatch/core.py

@@ -0,0 +1,554 @@
+"""keepwatch core.
+
+Three ideas, nothing more:
+
+1. A task is a JSON file that lives in exactly one of three folders:
+   ``pending`` -> ``processing`` -> ``done``.
+2. Whoever is working on a task holds a *lease*. If that worker dies, the lease
+   expires and ``reap`` puts the task back. The task keeps a hand-over trail.
+3. A task is only ``done`` when a ``verify`` record with ``rc == 0`` exists.
+   The model can say "it passed"; the ledger is what proves it.
+
+Everything is files. No database, no daemon, no third-party dependency.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import socket
+import time
+import uuid
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Iterable
+
+SCHEMA_VERSION = 1
+PENDING = "pending"
+PROCESSING = "processing"
+DONE = "done"
+BOXES = (PENDING, PROCESSING, DONE)
+
+AGENT_RE = re.compile(r"^[a-z0-9_-]{1,32}$")
+DEFAULT_TTL_SECONDS = 300
+DEFAULT_STALE_SECONDS = 1200
+
+EXIT_OK = 0
+EXIT_ERROR = 1
+EXIT_NOT_FOUND = 2
+EXIT_HELD = 3
+EXIT_BLOCKED = 13
+
+
+class keepwatchError(Exception):
+    """Base error. ``code`` is part of the CLI contract."""
+
+    code = EXIT_ERROR
+
+
+class NotFound(keepwatchError):
+    code = EXIT_NOT_FOUND
+
+
+class Held(keepwatchError):
+    code = EXIT_HELD
+
+
+class Blocked(keepwatchError):
+    code = EXIT_BLOCKED
+
+
+# --------------------------------------------------------------------------
+# time helpers (always UTC, always aware)
+# --------------------------------------------------------------------------
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def stamp(dt: datetime | None = None) -> str:
+    # Milliseconds, not seconds: two tasks created in the same second must still
+    # have a deterministic order when we pick "the oldest pending one".
+    return (dt or utcnow()).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def parse_stamp(text: str) -> datetime:
+    return datetime.fromisoformat(text.replace("Z", "+00:00"))
+
+
+def age_seconds(text: str) -> float:
+    return (utcnow() - parse_stamp(text)).total_seconds()
+
+
+# --------------------------------------------------------------------------
+# layout
+# --------------------------------------------------------------------------
+class Store:
+    """The directory layout. Construct once, pass around."""
+
+    def __init__(self, root: str | Path):
+        self.root = Path(root).expanduser().resolve()
+
+    # -- paths ----------------------------------------------------------
+    def agent_dir(self, agent: str) -> Path:
+        if not AGENT_RE.match(agent):
+            raise keepwatchError(f"bad agent id: {agent!r} (want [a-z0-9_-]{{1,32}})")
+        return self.root / "agents" / agent
+
+    def box(self, agent: str, box: str) -> Path:
+        return self.agent_dir(agent) / box
+
+    def task_path(self, agent: str, box: str, task_id: str) -> Path:
+        return self.box(agent, box) / f"{task_id}.json"
+
+    def heartbeat_path(self, agent: str) -> Path:
+        return self.root / "state" / f"{agent}.heartbeat.json"
+
+    def audit_path(self) -> Path:
+        return self.root / "audit" / f"{utcnow():%Y-%m}.jsonl"
+
+    # -- lifecycle ------------------------------------------------------
+    def init(self, agents: Iterable[str]) -> Store:
+        for agent in agents:
+            self.agent_dir(agent)
+            for box in BOXES:
+                self.box(agent, box).mkdir(parents=True, exist_ok=True)
+        (self.root / "state").mkdir(parents=True, exist_ok=True)
+        (self.root / "audit").mkdir(parents=True, exist_ok=True)
+        return self
+
+    def require(self) -> None:
+        if not (self.root / "agents").is_dir():
+            raise keepwatchError(f"{self.root} is not a keepwatch root; run `init` first")
+
+    def agents(self) -> list[str]:
+        base = self.root / "agents"
+        if not base.is_dir():
+            return []
+        return sorted(p.name for p in base.iterdir() if p.is_dir())
+
+
+# --------------------------------------------------------------------------
+# files
+# --------------------------------------------------------------------------
+def _read(path: Path) -> dict[str, Any]:
+    for attempt in range(6):
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            raise NotFound(f"task not found: {path.name}") from None
+        except PermissionError:
+            # Windows may refuse a read while another process renames the file.
+            if attempt == 5:
+                raise Held(f"task is busy (locked by another process): {path.name}") from None
+            time.sleep(0.005)
+        except json.JSONDecodeError as exc:
+            raise keepwatchError(f"corrupt task file {path.name}: {exc}") from exc
+    raise Held(f"task is busy: {path.name}")
+
+
+def _replace(src: Path, dst: Path, retries: int = 10, delay: float = 0.005) -> None:
+    """os.replace with a short retry loop.
+
+    On Windows a rename fails with PermissionError (WinError 32) while another
+    process has the file open, even for a moment. Retrying turns a spurious
+    failure into a correct outcome: whoever wins the retry race owns the file.
+    """
+    last: Exception | None = None
+    for _ in range(retries):
+        try:
+            os.replace(src, dst)
+            return
+        except FileNotFoundError:
+            raise
+        except PermissionError as exc:
+            last = exc
+            time.sleep(delay)
+    assert last is not None
+    raise last
+
+
+def _write_atomic(path: Path, payload: dict[str, Any]) -> None:
+    """Write to a uniquely named temp file, then rename. No half files, no shared temps."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex[:8]}.tmp")
+    tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    try:
+        _replace(tmp, path)
+    finally:
+        if tmp.exists():
+            tmp.unlink(missing_ok=True)
+
+
+def _move(src: Path, dst: Path) -> None:
+    """Atomic move. If src disappeared, someone else won the race."""
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        _replace(src, dst)
+    except FileNotFoundError:
+        raise NotFound(f"task already taken: {dst.name}") from None
+    except PermissionError:
+        raise Held(f"task is busy (locked by another process): {dst.name}") from None
+
+
+@contextmanager
+def _exclusive(path: Path, stale_after: float = 30.0):
+    """A mutex built on ``O_CREAT | O_EXCL``.
+
+    Do **not** use rename as a lock. On at least one Windows filesystem two
+    concurrent ``os.replace()`` calls on the same source both report success,
+    which silently lets two workers claim one task. Exclusive create was the one
+    primitive that stayed atomic under contention, so it is the one we use.
+    """
+    acquired = False
+    for _ in range(4):
+        try:
+            fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+            os.write(fd, str(os.getpid()).encode())
+            os.close(fd)
+            acquired = True
+            break
+        except FileExistsError:
+            try:
+                if time.time() - path.stat().st_mtime > stale_after:
+                    path.unlink(missing_ok=True)
+                    continue
+            except FileNotFoundError:
+                continue
+            raise Held(f"task is busy (someone is working on it): {path.name}") from None
+    if not acquired:
+        raise Held(f"task is busy: {path.name}")
+    try:
+        yield
+    finally:
+        path.unlink(missing_ok=True)
+
+
+def heartbeat(store: Store, agent: str) -> dict[str, Any]:
+    payload = {"agent": agent, "at": stamp(), "pid": os.getpid(), "host": socket.gethostname()}
+    _write_atomic(store.heartbeat_path(agent), payload)
+    return payload
+
+
+def ledger_append(store: Store, event: str, **fields: Any) -> dict[str, Any]:
+    record = {"at": stamp(), "event": event, "schema": SCHEMA_VERSION, **fields}
+    path = store.audit_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(record, ensure_ascii=False) + "\n")
+    return record
+
+
+def ledger_records(store: Store) -> list[dict[str, Any]]:
+    records: list[dict[str, Any]] = []
+    audit = store.root / "audit"
+    if not audit.is_dir():
+        return records
+    for path in sorted(audit.glob("*.jsonl")):
+        for line in path.read_text(encoding="utf-8").splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+    return records
+
+
+# --------------------------------------------------------------------------
+# commands
+# --------------------------------------------------------------------------
+def new_task_id() -> str:
+    return f"{utcnow():%Y%m%dT%H%M%S}-{uuid.uuid4().hex[:6]}"
+
+
+def send(store: Store, sender: str, to: str, subject: str, body: str = "") -> dict[str, Any]:
+    store.require()
+    if to not in store.agents():
+        raise keepwatchError(f"unknown agent: {to!r}")
+    task = {
+        "id": new_task_id(),
+        "from": sender,
+        "to": to,
+        "subject": subject,
+        "body": body,
+        "created_at": stamp(),
+        "state": PENDING,
+        "tries": 0,
+        "prev_owners": [],
+        "_lease": None,
+    }
+    _write_atomic(store.task_path(to, PENDING, task["id"]), task)
+    ledger_append(store, "send", id=task["id"], from_=sender, to=to, subject=subject)
+    return task
+
+
+def list_tasks(store: Store, agent: str, box: str = PENDING) -> list[dict[str, Any]]:
+    store.require()
+    folder = store.box(agent, box)
+    if not folder.is_dir():
+        return []
+    tasks = []
+    for path in sorted(folder.glob("*.json")):
+        try:
+            tasks.append(_read(path))
+        except keepwatchError:
+            continue
+    return tasks
+
+
+def check(store: Store, agent: str) -> list[dict[str, Any]]:
+    """What can I pick up right now?"""
+    return list_tasks(store, agent, PENDING)
+
+
+def claim(store: Store, agent: str, task_id: str, ttl: int = DEFAULT_TTL_SECONDS) -> dict[str, Any]:
+    store.require()
+    token = store.box(agent, PROCESSING) / f".{task_id}.claim"
+    with _exclusive(token):
+        return _claim_locked(store, agent, task_id, ttl)
+
+
+def _claim_locked(store: Store, agent: str, task_id: str, ttl: int) -> dict[str, Any]:
+    now = utcnow()
+    pending = store.task_path(agent, PENDING, task_id)
+    processing = store.task_path(agent, PROCESSING, task_id)
+
+    if pending.exists():
+        task = _read(pending)
+        _move(pending, processing)
+        task["state"] = PROCESSING
+        task["tries"] = int(task.get("tries", 0)) + 1
+        task["_lease"] = {
+            "owner": agent,
+            "acquired_at": stamp(now),
+            "expires_at": stamp(now + timedelta(seconds=ttl)),
+        }
+        _write_atomic(processing, task)
+        ledger_append(store, "claim", id=task_id, by=agent, tries=task["tries"], ttl=ttl)
+        return task
+
+    if processing.exists():
+        task = _read(processing)
+        lease = task.get("_lease") or {}
+        owner = lease.get("owner")
+        expires = lease.get("expires_at")
+        if not expires:
+            # The file is in processing but has no lease yet: another process is
+            # between "moved the file" and "wrote the lease". Never steal that.
+            raise Held(f"task {task_id} is being claimed right now (no lease yet)")
+        if parse_stamp(expires) > now:
+            # Strict on purpose: a second claim means two workers would run the
+            # same task. Use `renew` to extend your own lease instead.
+            raise Held(f"task {task_id} is already claimed (owner={owner}, until {expires})")
+        # Lease expired: take it over, and leave a trail.
+        task.setdefault("prev_owners", []).append(owner)
+        task["tries"] = int(task.get("tries", 0)) + 1
+        task["_lease"] = {
+            "owner": agent,
+            "acquired_at": stamp(now),
+            "expires_at": stamp(now + timedelta(seconds=ttl)),
+        }
+        _write_atomic(processing, task)
+        ledger_append(store, "steal", id=task_id, by=agent, from_=owner, tries=task["tries"])
+        return task
+
+    raise NotFound(f"no such task in {agent}: {task_id}")
+
+
+def renew(store: Store, agent: str, task_id: str, ttl: int = DEFAULT_TTL_SECONDS) -> dict[str, Any]:
+    """Extend your own lease. Only the current owner may call this."""
+    store.require()
+    path = store.task_path(agent, PROCESSING, task_id)
+    with _exclusive(store.box(agent, PROCESSING) / f".{task_id}.claim"):
+        task = _read(path)
+        lease = task.get("_lease") or {}
+        if lease.get("owner") != agent:
+            raise Held(f"task {task_id} is not yours (owner={lease.get('owner')})")
+        lease["expires_at"] = stamp(utcnow() + timedelta(seconds=ttl))
+        task["_lease"] = lease
+        _write_atomic(path, task)
+    ledger_append(store, "renew", id=task_id, by=agent, ttl=ttl)
+    return task
+
+
+def release(store: Store, agent: str, task_id: str, reason: str = "") -> dict[str, Any]:
+    """Put a task back to pending without closing it (failed run, or giving up)."""
+    store.require()
+    path = store.task_path(agent, PROCESSING, task_id)
+    token = store.box(agent, PROCESSING) / f".{task_id}.claim"
+    with _exclusive(token):
+        task = _read(path)
+        lease = task.get("_lease") or {}
+        if lease.get("owner") not in (None, agent):
+            raise Held(f"task {task_id} is held by {lease.get('owner')}")
+        task["state"] = PENDING
+        task["prev_owners"] = list(task.get("prev_owners", [])) + [lease.get("owner")]
+        task["_lease"] = None
+        task["released_at"] = stamp()
+        _write_atomic(path, task)
+        _move(path, store.task_path(agent, PENDING, task_id))
+    ledger_append(store, "release", id=task_id, by=agent, reason=reason)
+    return task
+
+
+def oldest_pending(store: Store, agent: str) -> dict[str, Any] | None:
+    tasks = list_tasks(store, agent, PENDING)
+    if not tasks:
+        return None
+    return min(tasks, key=lambda t: (t.get("created_at", ""), t["id"]))
+
+
+def verify(store: Store, task_id: str, name: str, rc: int) -> dict[str, Any]:
+    """Record a verification result. The task is not done until this returns 0."""
+    store.require()
+    return ledger_append(store, "verify", id=task_id, name=name, rc=int(rc))
+
+
+def has_ok_verify(store: Store, task_id: str) -> bool:
+    return any(
+        r.get("event") == "verify" and r.get("id") == task_id and r.get("rc") == 0
+        for r in ledger_records(store)
+    )
+
+
+def done(
+    store: Store,
+    agent: str,
+    task_id: str,
+    note: str = "",
+    force: bool = False,
+) -> dict[str, Any]:
+    store.require()
+    processing = store.task_path(agent, PROCESSING, task_id)
+    with _exclusive(store.box(agent, PROCESSING) / f".{task_id}.claim"):
+        task = _read(processing)
+        lease = task.get("_lease") or {}
+        if lease.get("owner") not in (None, agent):
+            raise Held(f"task {task_id} is held by {lease.get('owner')}")
+
+        if not force and not has_ok_verify(store, task_id):
+            raise Blocked(
+                f"refusing to close {task_id}: no verify record with rc=0. "
+                f"Run `verify --id {task_id} --name <check> --rc 0` first, or pass --force."
+            )
+
+        task["state"] = DONE
+        task["_lease"] = None
+        task["closed_at"] = stamp()
+        task["closed_by"] = agent
+        task["note"] = note
+        _write_atomic(processing, task)
+        _move(processing, store.task_path(agent, DONE, task_id))
+    ledger_append(store, "done", id=task_id, by=agent, note=note)
+    return task
+
+
+def reap(store: Store, stale_after: int = 0) -> list[dict[str, Any]]:
+    """Return expired leases to pending. This is what runs after a worker dies."""
+    store.require()
+    now = utcnow()
+    reaped: list[dict[str, Any]] = []
+    for agent in store.agents():
+        for task in list_tasks(store, agent, PROCESSING):
+            lease = task.get("_lease") or {}
+            path = store.task_path(agent, PROCESSING, task["id"])
+            expires = lease.get("expires_at")
+            if expires:
+                deadline = parse_stamp(expires)
+            else:
+                # Orphan: crashed between moving the file and writing the lease.
+                deadline = datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc)
+            if deadline + timedelta(seconds=stale_after) > now:
+                continue
+            task_id = task["id"]
+            token = store.box(agent, PROCESSING) / f".{task_id}.claim"
+            try:
+                with _exclusive(token):
+                    task["state"] = PENDING
+                    task["reaped_at"] = stamp(now)
+                    task["prev_owners"] = list(task.get("prev_owners", [])) + [lease.get("owner")]
+                    task["_lease"] = None
+                    _write_atomic(store.task_path(agent, PROCESSING, task_id), task)
+                    _move(
+                        store.task_path(agent, PROCESSING, task_id),
+                        store.task_path(agent, PENDING, task_id),
+                    )
+            except Held:
+                continue  # someone is mid-claim; leave it for the next sweep
+            ledger_append(store, "reap", id=task_id, agent=agent, owner=lease.get("owner"))
+            reaped.append(task)
+    return reaped
+
+
+def status(store: Store, stale_after: int = DEFAULT_STALE_SECONDS) -> dict[str, Any]:
+    """Everything a human needs to answer: what is stuck, who is dead, what is next."""
+    store.require()
+    now = utcnow()
+    agents: dict[str, Any] = {}
+    expired: list[dict[str, Any]] = []
+    next_steps: list[str] = []
+
+    for agent in store.agents():
+        hb_path = store.heartbeat_path(agent)
+        hb_age = None
+        if hb_path.exists():
+            try:
+                hb_age = int(age_seconds(json.loads(hb_path.read_text(encoding="utf-8"))["at"]))
+            except (KeyError, ValueError, json.JSONDecodeError):
+                hb_age = None
+        alive = hb_age is not None and hb_age <= stale_after
+
+        pending = list_tasks(store, agent, PENDING)
+        processing = list_tasks(store, agent, PROCESSING)
+        finished = list_tasks(store, agent, DONE)
+        agents[agent] = {
+            "pending": len(pending),
+            "processing": len(processing),
+            "done": len(finished),
+            "heartbeat_age_s": hb_age,
+            "alive": alive,
+        }
+
+        for task in processing:
+            lease = task.get("_lease") or {}
+            expires = lease.get("expires_at")
+            if expires:
+                deadline = parse_stamp(expires)
+            else:
+                deadline = datetime.fromtimestamp(
+                    store.task_path(agent, PROCESSING, task["id"]).stat().st_mtime,
+                    tz=timezone.utc,
+                )
+            if deadline <= now:
+                expired.append(
+                    {
+                        "agent": agent,
+                        "id": task["id"],
+                        "held_by": lease.get("owner"),
+                        "overdue_s": int((now - deadline).total_seconds()),
+                    }
+                )
+
+        if pending and not processing and not alive:
+            next_steps.append(
+                f"{agent}: {len(pending)} task(s) waiting, and no live heartbeat - is the worker dead?"
+            )
+        elif pending and alive:
+            next_steps.append(f"{agent}: {len(pending)} task(s) waiting to be claimed")
+
+    for item in expired:
+        next_steps.append(
+            f"{item['agent']}: {item['id']} held by {item['held_by'] or 'nobody'}, "
+            f"{item['overdue_s']}s past its lease - run `reap`"
+        )
+
+    return {
+        "root": str(store.root),
+        "agents": agents,
+        "expired_leases": expired,
+        "next_steps": next_steps,
+        "ok": not expired,
+    }

keepwatch-0.1.1.dist-info/METADATA

@@ -0,0 +1,192 @@
+Metadata-Version: 2.4
+Name: keepwatch
+Version: 0.1.1
+Summary: Reliability layer for unattended agent mailboxes: leases, heartbeats, and silent-failure detection.
+License: MIT
+Project-URL: Documentation, https://github.com/OWNER/keepwatch#readme
+Project-URL: Issues, https://github.com/OWNER/keepwatch/issues
+Keywords: agents,multi-agent,reliability,mailbox,unattended
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Dynamic: license-file
+
+# keepwatch
+
+> **Keep work alive when the process that started it is not.**
+> A 300-line, zero-dependency ledger for agents that run unattended.
+
+[![ci](https://github.com/ReZeroStarting/keepwatch/actions/workflows/ci.yml/badge.svg)](https://github.com/ReZeroStarting/keepwatch/actions/workflows/ci.yml)
+
+---
+
+## The problem
+
+An unattended pipeline does not fail loudly. It goes quiet:
+
+- a task sits in `pending/` and nobody claims it
+- a worker dies mid-task and the task is stuck in `processing/` forever
+- a worker finishes but never marks it done, so the next round re-dispatches it
+- the agent says "all tests pass" and you have nothing to check that against
+
+All four look identical from the outside: **nothing happened, and the logs are clean.**
+
+## What it does
+
+Three folders, one ledger, four verbs.
+
+```
+<root>/agents/<agent>/pending/      tasks a worker can pick up
+<root>/agents/<agent>/processing/   tasks someone is holding (with a lease)
+<root>/agents/<agent>/done/         closed tasks, never deleted
+<root>/audit/YYYY-MM.jsonl          append-only record of everything
+```
+
+| Verb | Meaning |
+|---|---|
+| `check` | what can I pick up |
+| `claim` | take one, and hold a lease on it |
+| `done` | close it — **only if a passing `verify` record exists** |
+| `verify` | record a check result (`--rc 0`) against a task |
+| **`run`** | **claim → run a command → record its exit code → close or release** |
+
+Plus `send`, `renew`, `release`, `reap`, `status`.
+
+## The one command you actually need
+
+```bash
+keepwatch run --who builder -- python -m pytest
+```
+
+That does the whole loop in one go:
+
+1. claims the oldest pending task (or `--id <id>`)
+2. runs the command with inherited stdio, so you still see the output
+3. writes `verify ... --rc <real exit code>` into the ledger
+4. `rc=0` → `done`; `rc!=0` → back to `pending` for another try
+5. after `--max-tries` (default 3) failures → left in `processing` with a `suspect` record
+
+If the process is killed mid-command, nothing is lost: the lease expires and `reap` puts the
+task back. **You do not have to remember to write the ledger — the wrapper does it.**
+
+## Why it matters
+
+`claim` starts a lease. If the worker dies, the lease expires and `reap` puts the task
+back with a trail (`tries`, `prev_owners`). No task is ever stranded, and no task is
+silently done twice.
+
+`done` refuses to close a task unless the ledger holds a `verify ... rc=0` for it.
+That is the whole point: **the model can say "it passed"; the ledger is what proves it.**
+
+```console
+$ keepwatch done --who builder --id 20261023T083729-8c4fae
+keepwatch: refusing to close 20261023T083729-8c4fae: no verify record with rc=0.
+Run `verify --id ... --name <check> --rc 0` first, or pass --force.
+$ echo $?
+13
+```
+
+## Install
+
+Python 3.9+, no dependencies.
+
+```bash
+pip install -e .
+# or just run it from a checkout:
+PYTHONPATH=src python -m keepwatch.cli --help
+```
+
+## 60-second demo
+
+```bash
+keepwatch --root /tmp/ns init builder reviewer
+
+keepwatch --root /tmp/ns send \
+  --from reviewer --to builder --subject "batch 3: add login page"
+
+keepwatch --root /tmp/ns check --who builder
+keepwatch --root /tmp/ns claim --who builder --id <id> --ttl 60
+```
+
+Now pretend the worker dies. Wait for the lease, then:
+
+```console
+$ keepwatch --root /tmp/ns status
+root: /tmp/ns
+  builder        pending 0  processing 1  done 0  hb 61s (alive)
+  reviewer       pending 0  processing 0  done 0  never seen (unknown)
+next:
+  - builder: 20261023T083750-ef2c5d held by builder, 1s past its lease - run `reap`
+
+$ keepwatch --root /tmp/ns reap
+reaped 1 task(s)
+  20261023T083750-ef2c5d  back to pending (tries=1)
+
+$ keepwatch --root /tmp/ns claim --who builder --id 20261023T083750-ef2c5d
+claimed 20261023T083750-ef2c5d (try 2) lease until 2026-10-23T08:42:31Z
+```
+
+The task came back with `tries=1` and the previous owner recorded. Close it, but only
+after proof:
+
+```bash
+keepwatch --root /tmp/ns verify --id <id> --name pytest --rc 0
+keepwatch --root /tmp/ns done   --who builder --id <id> --note "e2e green"
+```
+
+## Exit codes
+
+Exit codes are the interface; agents read them instead of parsing prose.
+
+| Code | Meaning |
+|---|---|
+| 0 | ok |
+| 1 | bad arguments, IO error |
+| 2 | task not found / already taken |
+| 3 | task held by another worker (or busy) |
+| 13 | blocked: no passing verify, or `status` found something that needs attention |
+
+## Design notes
+
+**Files are the source of truth.** No database, no daemon, no network. `ls` and `cat`
+are the debugging tools.
+
+**Exclusive create is the mutex, not rename.** On at least one Windows filesystem two
+concurrent `os.replace()` calls on the same source *both report success*. Claiming is
+therefore guarded by `O_CREAT|O_EXCL` on a short-lived token file. This is documented in
+[`docs/PROTOCOL.md`](docs/PROTOCOL.md) because it cost a real bug to find.
+
+**One worker per agent box.** Leases are per agent directory, so `builder` running two
+processes is a configuration error, not a supported mode.
+
+**`status` exits 13 when something needs attention** — so `keepwatch status || alert`
+works in cron.
+
+## What it is not
+
+- not a mailbox (see [docs/COMPETITION.md](docs/COMPETITION.md) for what exists)
+- not an orchestrator: it never decides *what* to do next
+- not an agent: it makes no model calls
+- not durable execution: it will not replay your side effects
+
+## Docs
+
+| File | Contents |
+|---|---|
+| [docs/PROTOCOL.md](docs/PROTOCOL.md) | layout, task format, lease rules, exit codes |
+| [docs/COMPETITION.md](docs/COMPETITION.md) | why this exists next to agent-mailbox / A2A / Temporal |
+| [docs/ROADMAP.md](docs/ROADMAP.md) | v0.1 → v1.0 and the release checklist |
+| [docs/zh/](docs/zh/) | 中文原文 (Chinese originals) |
+
+Tests: `PYTHONPATH=src python -m unittest discover -s tests -v`
+
+## License
+
+MIT. See [LICENSE](LICENSE).

keepwatch-0.1.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+keepwatch/__init__.py,sha256=BSckz3Ju9p8d17DnoE5TMLqLhdYuRu6Iu4litEjq-BQ,429
+keepwatch/cli.py,sha256=puT2EKicKcbe0mMIRJrVBbA4_5XAtSIyucGByqKQGb8,9290
+keepwatch/core.py,sha256=_nNQOZYNwahMuzAaaCtUEK-exuHFPzbKqWvkO01fVrQ,20162
+keepwatch-0.1.1.dist-info/licenses/LICENSE,sha256=9IP-h92D0ka3PdvhJtELf0-B4aSEGPGLfTx-m6oA9L8,1080
+keepwatch-0.1.1.dist-info/METADATA,sha256=MpOLzgMA5uBu-IyoiAqL4DBepv2NiSZqwRy6kBLgSac,6624
+keepwatch-0.1.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+keepwatch-0.1.1.dist-info/entry_points.txt,sha256=JEzd6ch5sCcBtTy24E3tCAZAXStW6sSTQdwt3nPNtPI,49
+keepwatch-0.1.1.dist-info/top_level.txt,sha256=M0z5U1DUcrRuGXqzBow4QXHGgNIIXpps_gAD1wPOYec,10
+keepwatch-0.1.1.dist-info/RECORD,,

keepwatch-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

keepwatch-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+keepwatch = keepwatch.cli:main

keepwatch-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 nightshift contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

keepwatch-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+keepwatch