handcode 0.3.0rc1__py3-none-any.whl

agentctl/control/proxyenv.py

@@ -0,0 +1,309 @@
+"""A LiteLLM proxy that agentctl installs, starts and stops for you.
+
+`docs/0043` Phase 2, part B. Before this, the pool took three commands and two
+flags (`proxy --out`, `start.sh` or `start.ps1`, then `--model openai/pool
+--base-url ...`), plus a two-step install, because `litellm[proxy]` declares
+`mcp<2` and the README said the SDK needed `mcp>=2` (`docs/0028`).
+
+The proxy is a separate PROCESS already, so it can have a separate
+ENVIRONMENT, and then the conflict cannot exist:
+
+    ~/.agentctl/proxy-env/     a venv holding litellm[proxy] and a COPY of
+                               agentctl's pure-Python package (the hook needs
+                               its kernel), and never the OpenHands SDK
+    ~/.agentctl/proxy/         config, hook, pid file, log
+
+The user's own install never contains `litellm[proxy]`. Nothing is put on the
+proxy's PYTHONPATH: the generated start scripts used to point it at the main
+environment's site-packages, which would bring every package -- and the
+conflict -- back in (`docs/0047`).
+
+    agentctl proxy up        create the env if needed, verify, start, wait
+    agentctl proxy status    is it running, and does it answer
+    agentctl proxy down      stop it
+    agentctl run --pool      `up` if needed, then route through it
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+LITELLM_SPEC = "litellm[proxy]>=1.100.0"
+DEFAULT_PORT = 4000
+
+
+def home() -> Path:
+    return Path(os.environ.get("AGENTCTL_HOME") or Path.home() / ".agentctl")
+
+
+def env_dir() -> Path:
+    """`~/.agentctl/proxy-env`, or `AGENTCTL_PROXY_ENV`.
+
+    The override is for the container image (`docs/0051` Stage 2), which
+    builds the environment in at `/opt/handcode/proxy-env`. Under
+    `~/.agentctl` it would be hidden by the volume that keeps `status` and
+    `resume` working across containers.
+    """
+    if (p := os.environ.get("AGENTCTL_PROXY_ENV")):
+        return Path(p)
+    return home() / "proxy-env"
+
+
+def run_dir() -> Path:
+    return home() / "proxy"
+
+
+def _env_python() -> Path:
+    d = env_dir()
+    return d / ("Scripts/python.exe" if sys.platform == "win32" else "bin/python")
+
+
+def _env_litellm() -> Path:
+    d = env_dir()
+    return d / ("Scripts/litellm.exe" if sys.platform == "win32" else "bin/litellm")
+
+
+def _say(msg: str) -> None:
+    print(f"  {msg}", flush=True)
+
+
+# ── the environment ────────────────────────────────────────────────────
+def ensure_env(log=_say) -> Path:
+    """Create the proxy's own venv, install litellm[proxy], copy agentctl in.
+
+    The first call downloads litellm and its proxy extras, which takes
+    minutes; later calls only refresh the agentctl copy, which takes well
+    under a second and keeps the hook in step with the CLI that started it.
+    """
+    py = _env_python()
+    if not py.exists():
+        log(f"creating the proxy environment at {env_dir()} (once)")
+        r = subprocess.run([sys.executable, "-m", "venv", str(env_dir())],
+                           capture_output=True, text=True)
+        if r.returncode != 0:
+            raise SystemExit(
+                f"could not create a venv for the proxy:\n{r.stderr.strip()[-600:]}\n"
+                f"  on Ubuntu: sudo apt install python3-venv")
+    if not _env_litellm().exists():
+        log(f"installing {LITELLM_SPEC} into it -- a few minutes, once")
+        r = subprocess.run([str(py), "-m", "pip", "install", "-q",
+                            "--disable-pip-version-check", LITELLM_SPEC],
+                           capture_output=True, text=True)
+        if r.returncode != 0 or not _env_litellm().exists():
+            raise SystemExit(f"installing {LITELLM_SPEC} failed:\n"
+                             f"{(r.stdout + r.stderr).strip()[-1200:]}")
+    _copy_agentctl(py)
+    return py
+
+
+def _copy_agentctl(py: Path) -> Path:
+    """Put this exact agentctl package into the proxy env's site-packages.
+
+    A copy, not a PYTHONPATH entry and not a pip install: the hook must be the
+    same code as the CLI that generated its config, it must come without the
+    main environment's other packages, and it must work whether agentctl was
+    installed from PyPI, from a wheel, or editable from a clone.
+    """
+    import agentctl
+
+    src = Path(agentctl.__file__).resolve().parent
+    purelib = subprocess.run(
+        [str(py), "-c", "import sysconfig; print(sysconfig.get_paths()['purelib'])"],
+        capture_output=True, text=True, check=True).stdout.strip()
+    dst = Path(purelib) / "agentctl"
+    if dst.exists():
+        shutil.rmtree(dst)
+    shutil.copytree(src, dst, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
+    return dst
+
+
+# ── the process ────────────────────────────────────────────────────────
+def _pidfile() -> Path:
+    return run_dir() / "proxy.json"
+
+
+def _read_pid() -> dict | None:
+    try:
+        return json.loads(_pidfile().read_text(encoding="utf-8"))
+    except Exception:                                   # noqa: BLE001
+        return None
+
+
+def url(port: int = DEFAULT_PORT) -> str:
+    return f"http://127.0.0.1:{port}"
+
+
+def answers(port: int, timeout: float = 2.0) -> bool:
+    """Does a proxy on this port answer its unauthenticated liveness check?"""
+    try:
+        with urllib.request.urlopen(f"{url(port)}/health/liveliness",
+                                    timeout=timeout) as r:
+            return r.status == 200
+    except Exception:                                   # noqa: BLE001
+        return False
+
+
+def status() -> dict:
+    """{'state': 'running' | 'stopped' | 'dead' | 'foreign', ...}
+
+    `foreign` is something answering on the port that agentctl did not start
+    (a proxy you ran by hand, say). It is reported, never stopped.
+    """
+    from agentctl.runtime.lease import pid_alive
+
+    rec = _read_pid()
+    port = (rec or {}).get("port", DEFAULT_PORT)
+    alive = bool(rec) and pid_alive(int(rec["pid"]))
+    up = answers(port)
+    if rec and alive:
+        state = "running" if up else "starting"
+    elif rec:
+        state = "dead"
+    else:
+        state = "foreign" if up else "stopped"
+    return {"state": state, "port": port, "pid": (rec or {}).get("pid"),
+            "answers": up, "log": str(run_dir() / "proxy.log"),
+            "env": str(env_dir())}
+
+
+def up(port: int = DEFAULT_PORT, verify: bool = True, log=_say,
+       wait_s: float = 120.0) -> dict:
+    """Start the managed proxy, or reuse it if it is already running."""
+    s = status()
+    if s["state"] in ("running", "starting"):
+        if s["port"] == port:
+            log(f"proxy already running (pid {s['pid']}) at {url(port)}")
+            return s
+        raise SystemExit(f"a managed proxy is running on port {s['port']}, "
+                         f"not {port}. `agentctl proxy down` first.")
+    # Only the port being asked for matters. `status()` with no pid file
+    # probes the DEFAULT port, and refusing port N because something answers
+    # on 4000 was a bug a live proxy on 4000 exposed in the tests.
+    if answers(port):
+        raise SystemExit(f"something not started by agentctl answers on port "
+                         f"{port}. Stop it, or use --port.")
+
+    ensure_env(log)
+
+    from agentctl.control.proxy import available, write
+
+    only, drop = None, set()
+    if verify:
+        from agentctl.control.providers import BY_NAME
+        from agentctl.control.proxy import verified_providers
+
+        log("verifying providers (one completion per model id; paid skipped) ...")
+        only, report = verified_providers()
+        for name, r in sorted(report.items()):
+            log(f"  {'ok' if name in only else '--'}  {name:<11} {r.status}")
+        drop = {f"{BY_NAME[n].prefix}{m}" for n, r in report.items() for m in r.gone}
+        for m in sorted(drop):
+            log(f"  left out   {m}  (the provider no longer serves it)")
+    entries = [e for e in available()
+               if (only is None or _provider_name(e[0]) in only)
+               and e[1] not in drop]
+    if not entries:
+        raise SystemExit("no provider can serve right now -- nothing to pool.\n"
+                         "  `agentctl keys --check` says why.")
+    log(f"{len(entries)} deployment(s) in the pool")
+
+    d = run_dir()
+    d.mkdir(parents=True, exist_ok=True)
+    cfg, _hook = write(d, only=only, drop_models=drop)
+
+    env = {**os.environ, "PYTHONUTF8": "1", "PYTHONIOENCODING": "utf-8",
+           "AGENTCTL_TELEMETRY": str(d / "hook_telemetry.json")}
+    env.pop("PYTHONPATH", None)            # never the main env's packages
+    logf = open(d / "proxy.log", "ab")
+    kw: dict = {"stdout": logf, "stderr": subprocess.STDOUT, "env": env,
+                "cwd": str(d), "stdin": subprocess.DEVNULL}
+    # The env's own python, not the `litellm.exe` console-script wrapper:
+    # launched DETACHED, the wrapper exited during startup with 0xC000013A
+    # (STATUS_CONTROL_C_EXIT) and an empty log (`docs/0047`).
+    argv = [str(_env_python()), "-c", "from litellm import run_server; run_server()",
+            "--config", str(cfg), "--port", str(port)]
+    if sys.platform == "win32":
+        # No window, its own group, and OUT of the launcher's job object, so
+        # it outlives the `agentctl` process that started it. A job that does
+        # not allow breakaway refuses the flag; then it runs without it.
+        base = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP
+        try:
+            p = subprocess.Popen(argv, creationflags=base | 0x01000000, **kw)
+        except OSError:
+            p = subprocess.Popen(argv, creationflags=base, **kw)
+    else:
+        p = subprocess.Popen(argv, start_new_session=True, **kw)
+    logf.close()
+    _pidfile().write_text(json.dumps({"pid": p.pid, "port": port,
+                                      "started": time.time()}), encoding="utf-8")
+
+    log(f"starting litellm (pid {p.pid}) on {url(port)} ...")
+    deadline = time.time() + wait_s
+    while time.time() < deadline:
+        if p.poll() is not None:
+            _pidfile().unlink(missing_ok=True)
+            raise SystemExit(f"the proxy exited during startup (code {p.returncode}). "
+                             f"Last lines of {d / 'proxy.log'}:\n"
+                             + _tail(d / "proxy.log"))
+        if answers(port):
+            log(f"proxy up at {url(port)}")
+            return status()
+        time.sleep(1)
+    raise SystemExit(f"the proxy did not answer within {wait_s:.0f}s; it is "
+                     f"still running (pid {p.pid}). See {d / 'proxy.log'}")
+
+
+def _provider_name(env_var: str) -> str:
+    """`OPENROUTER_API_KEY_2` -> `openrouter`."""
+    from agentctl.control.providers import BY_KEY
+    return BY_KEY[env_var.split("_API_KEY")[0] + "_API_KEY"].name
+
+
+def down(log=_say) -> bool:
+    """Stop the managed proxy. True if something was stopped."""
+    from agentctl.runtime.lease import pid_alive
+
+    rec = _read_pid()
+    if not rec:
+        log("no managed proxy is running")
+        return False
+    pid = int(rec["pid"])
+    if pid_alive(pid):
+        if sys.platform == "win32":
+            subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"],
+                           capture_output=True)
+        else:
+            import signal
+            # The group only when it is the proxy's OWN. `up` starts it in a
+            # new session, but a pid file can name anything, and signalling a
+            # group we belong to kills the caller: it SIGTERMed pytest, and the
+            # CI step with it, on every Linux job (`docs/0047`).
+            try:
+                pgid = os.getpgid(pid)
+                if pgid != os.getpgid(0):
+                    os.killpg(pgid, signal.SIGTERM)
+                else:
+                    os.kill(pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
+        for _ in range(20):
+            if not pid_alive(pid):
+                break
+            time.sleep(0.25)
+    _pidfile().unlink(missing_ok=True)
+    log(f"proxy stopped (pid {pid})")
+    return True
+
+
+def _tail(p: Path, n: int = 15) -> str:
+    try:
+        lines = p.read_text(encoding="utf-8", errors="replace").splitlines()
+        return "\n".join("    " + l for l in lines[-n:])
+    except Exception:                                   # noqa: BLE001
+        return "    (no log)"

agentctl/control/replay/__init__.py

@@ -0,0 +1,14 @@
+"""Record and replay real LLM sessions offline, at zero cost. M6.
+
+    from agentctl.control.replay import Cassette, ReplayServer
+
+`cassette.py` is pure and has no dependencies; `server.py` is stdlib only.
+The recorder lives in `adapters/litellm/` because it is vendor-specific,
+which is the same split the rest of the package uses.
+"""
+from .cassette import (Cassette, Miss, Turn, current_env, fingerprint,
+                       incompatible, summarise)
+from .server import ReplayServer
+
+__all__ = ["Cassette", "Miss", "Turn", "ReplayServer", "current_env",
+           "fingerprint", "incompatible", "summarise"]

agentctl/control/replay/cassette.py

@@ -0,0 +1,281 @@
+r"""A recorded LLM session, and the rule for matching a request to a response.
+
+The point of M6 (`docs/0012` §7): **re-run a real session offline at zero
+cost.** No API key, no network, no tokens, and — the part that matters —
+no sampling. The same code produces the same run every time.
+
+## The matching rule is the whole design
+
+A cassette is keyed by a fingerprint of what actually determines the response:
+
+    model + messages + tool schemas
+
+and deliberately *not* by `temperature`, `max_tokens`, `api_key`, `base_url`,
+request id, or timestamp. Those either do not change the mapping or change on
+every run, and folding them in would mean a cassette that never hits twice.
+
+A **miss is the product, not a failure.** Replaying a recorded session against
+changed code and getting a miss means the agent asked something different —
+which is precisely the regression signal M6 exists to produce. So a miss is
+reported with the turn number and a diff of what changed, rather than being
+papered over.
+
+## What determinism hides
+
+Replay pins `tool_call_id`, because the recorded response carries the one the
+model minted. That is convenient and it is also a trap: `docs/0023` found that
+`tool_call_id` is **not** stable across a real model pool, and a suite that
+only ever ran under replay would never have found it. Replay is for testing
+*our* logic against fixed inputs. It cannot test how the world varies.
+
+## Cassettes contain prompts
+
+Every recorded request holds the full message history — source code, file
+contents, whatever the agent was working on. Credentials are never recorded
+(they live in headers, which this never sees), but a cassette is as sensitive
+as the workspace it was recorded in. Treat it like a log, not like a fixture.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Iterator
+
+# THE list. It decides both what the recorder captures and what the
+# fingerprint is built from, and it is one list on purpose.
+#
+# It began as two: an allow-list in the recorder and a deny-list here. They
+# drifted, and a cassette recorded from a Python callback could never match the
+# same request arriving as HTTP, because the wire carries fields the callback
+# never saw -- `prompt_cache_key` (the conversation id, different every run by
+# definition) and `usage: {include: true}`. Every replay missed on turn 0.
+#
+# A deny-list cannot work here. The two sides are different representations of
+# one call, and any provider may add a field to the wire form at any time; a
+# deny-list would have to predict them all. An allow-list is closed, and the
+# cost of being closed is explicit: **a field outside this list is asserted not
+# to determine the response.** Adding a sampling parameter that does would
+# produce a wrong match rather than a miss, so the list is short and additions
+# belong in review.
+SIGNIFICANT_FIELDS = ("model", "messages", "tools", "tool_choice",
+                      "response_format", "functions", "function_call")
+
+
+def fingerprint(request: dict) -> str:
+    """Stable hash of the parts of a request that decide the response."""
+    return hashlib.sha256(canonical(request).encode("utf-8")).hexdigest()
+
+
+def canonical(request: dict) -> str:
+    """The exact text that gets hashed. Exposed so a miss can be diffed."""
+    kept = {k: request[k] for k in SIGNIFICANT_FIELDS
+            if request.get(k) is not None}
+    return json.dumps(kept, sort_keys=True, separators=(",", ":"), default=str)
+
+
+@dataclass
+class Turn:
+    """One request and the response it produced."""
+    index: int
+    fingerprint: str
+    request: dict
+    response: dict
+    model: str = ""
+    usage: dict = field(default_factory=dict)
+    # What the CALLER configured, e.g. "openrouter/vendor/model:free".
+    # `model` is what litellm passed on after stripping the provider prefix,
+    # and that stripped form is what the request carries and what the
+    # fingerprint is built from. Replay needs the routable name to reconfigure
+    # the run; matching needs the stripped one. They are not the same string
+    # and using one for both breaks replay (`docs/0029` §3).
+    provider_model: str = ""
+    # The environment the recording was made in. A cassette is NOT portable:
+    # the SDK builds a platform-dependent system prompt -- a Windows recording
+    # says "powershell" in message 0 -- so a cassette recorded on one OS misses
+    # on turn 0 everywhere else. Recording this is what lets a replay say so
+    # instead of looking like a behaviour change (`docs/0029` §6).
+    env: dict = field(default_factory=dict)
+
+    def to_json(self) -> str:
+        return json.dumps({
+            "index": self.index, "fingerprint": self.fingerprint,
+            "model": self.model, "provider_model": self.provider_model,
+            "usage": self.usage, "env": self.env,
+            "request": self.request, "response": self.response,
+        }, default=str)
+
+    @classmethod
+    def from_json(cls, line: str) -> "Turn":
+        d = json.loads(line)
+        return cls(index=d["index"], fingerprint=d["fingerprint"],
+                   request=d["request"], response=d["response"],
+                   model=d.get("model", ""), usage=d.get("usage") or {},
+                   provider_model=d.get("provider_model", ""),
+                   env=d.get("env") or {})
+
+
+@dataclass
+class Miss:
+    """A replayed request that the cassette has no recording for."""
+    turn: int
+    request: dict
+    nearest: Turn | None = None
+    recorded_turns: int = 0
+
+    def describe(self) -> str:
+        """Say what diverged, not merely that something did."""
+        if self.nearest is None:
+            return (f"turn {self.turn}: the recording ended after "
+                    f"{self.recorded_turns} turns — this run wanted more")
+        want = _messages(self.nearest.request)
+        got = _messages(self.request)
+        if len(want) != len(got):
+            return (f"turn {self.turn}: {len(got)} messages, recorded run had "
+                    f"{len(want)} — the conversation took a different shape")
+        for i, (a, b) in enumerate(zip(want, got)):
+            if a != b:
+                return (f"turn {self.turn}: message {i} ({b.get('role')}) "
+                        f"differs from the recording")
+        return f"turn {self.turn}: same messages, different tools or model"
+
+
+def current_env() -> dict:
+    """What this machine is, for the parts a recording depends on."""
+    import platform
+    try:
+        from importlib.metadata import version
+        sdk = version("openhands-sdk")
+    except Exception:                                   # noqa: BLE001
+        sdk = "?"
+    return {"platform": sys.platform, "python": platform.python_version(),
+            "openhands_sdk": sdk}
+
+
+def incompatible(cassette: "Cassette") -> str | None:
+    """Why this cassette cannot replay here, or None if it can.
+
+    Only `platform` is fatal, and it is fatal for a concrete reason rather than
+    caution: the SDK writes the shell name into the system prompt, so message 0
+    differs and every turn misses. The SDK version is reported when it differs
+    but not refused -- a prompt change would show up as an honest miss.
+    """
+    if not cassette.turns:
+        return "the cassette is empty"
+    rec = cassette.turns[0].env or {}
+    here = current_env()
+    if rec.get("platform") and rec["platform"] != here["platform"]:
+        return (f"recorded on {rec['platform']}, replaying on "
+                f"{here['platform']} -- the SDK puts the shell name in the "
+                f"system prompt, so turn 0 cannot match")
+    return None
+
+
+def _messages(request: dict) -> list[dict]:
+    m = request.get("messages")
+    return m if isinstance(m, list) else []
+
+
+class Cassette:
+    """Recorded turns, addressed by fingerprint.
+
+    Duplicate fingerprints are kept in order and served in order: an agent that
+    genuinely asks the same question twice must get both recorded answers, not
+    the first one twice.
+    """
+
+    def __init__(self, turns: list[Turn] | None = None):
+        self.turns: list[Turn] = list(turns or [])
+        self.misses: list[Miss] = []
+        self._served: set[int] = set()
+        self._index: dict[str, list[int]] = {}
+        for i, t in enumerate(self.turns):
+            self._index.setdefault(t.fingerprint, []).append(i)
+
+    # ── recording ──────────────────────────────────────────────────────
+    def append(self, request: dict, response: dict, *, model: str = "",
+               usage: dict | None = None, provider_model: str = "",
+               env: dict | None = None) -> Turn:
+        t = Turn(index=len(self.turns), fingerprint=fingerprint(request),
+                 request=request, response=response, model=model,
+                 usage=usage or {}, provider_model=provider_model,
+                 env=env if env is not None else current_env())
+        self.turns.append(t)
+        self._index.setdefault(t.fingerprint, []).append(t.index)
+        return t
+
+    # ── replaying ──────────────────────────────────────────────────────
+    def match(self, request: dict) -> Turn | None:
+        """The recorded response for this request, or None on a divergence."""
+        fp = fingerprint(request)
+        for i in self._index.get(fp, ()):
+            if i not in self._served:
+                self._served.add(i)
+                return self.turns[i]
+
+        self.misses.append(Miss(turn=len(self._served), request=request,
+                                nearest=self._nearest(),
+                                recorded_turns=len(self.turns)))
+        return None
+
+    def _nearest(self) -> Turn | None:
+        """The turn the recorded run would have been at by now.
+
+        Not a similarity search -- just position. It is what makes a miss
+        legible: 'at this point the recording asked X, you asked Y'.
+        """
+        n = len(self._served)
+        return self.turns[n] if n < len(self.turns) else None
+
+    @property
+    def exhausted(self) -> bool:
+        return len(self._served) >= len(self.turns)
+
+    def unplayed(self) -> list[Turn]:
+        """Recorded turns never reached. A short run is a divergence too."""
+        return [t for i, t in enumerate(self.turns) if i not in self._served]
+
+    # ── storage ────────────────────────────────────────────────────────
+    def save(self, path: str | Path) -> Path:
+        p = Path(path)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        with p.open("w", encoding="utf-8", newline="\n") as fh:
+            for t in self.turns:
+                fh.write(t.to_json() + "\n")
+        return p
+
+    @classmethod
+    def load(cls, path: str | Path) -> "Cassette":
+        p = Path(path)
+        if not p.exists():
+            raise FileNotFoundError(f"no cassette at {p}")
+        turns = [Turn.from_json(line) for line in _lines(p)]
+        return cls(turns)
+
+    def __len__(self) -> int:
+        return len(self.turns)
+
+    def __iter__(self) -> Iterator[Turn]:
+        return iter(self.turns)
+
+
+def _lines(p: Path) -> Iterator[str]:
+    with p.open(encoding="utf-8") as fh:
+        for line in fh:
+            if line.strip():
+                yield line
+
+
+def summarise(cassette: Cassette) -> dict[str, Any]:
+    """What a replay run actually did. The evaluator's output."""
+    return {
+        "turns_recorded": len(cassette.turns),
+        "turns_replayed": len(cassette.turns) - len(cassette.unplayed()),
+        "misses": len(cassette.misses),
+        "unplayed": len(cassette.unplayed()),
+        "diverged": bool(cassette.misses) or bool(cassette.unplayed()),
+        "first_divergence": (cassette.misses[0].describe()
+                             if cassette.misses else None),
+    }