coding-os 0.3.2__py3-none-any.whl

scripts/bench_sdk_dispatcher.py

@@ -0,0 +1,177 @@
+"""
+Real-world benchmark for the claude-sdk dispatcher.
+
+Spawns 3 formula-agents (F2/F5/F7) across 2 scenarios and records:
+  - latency_ms (wall clock)
+  - status (ok/timeout/error)
+  - output_json non-empty
+  - token cost if provided by ResultMessage
+
+Run: uv run --extra claude-sdk python scripts/bench_sdk_dispatcher.py
+
+NOTE: This calls the real Claude Code CLI — requires claude to be installed
+and authenticated on the local machine.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import importlib.util
+import json
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+sys.path.insert(0, str(ROOT / "core" / "thinking_os"))
+
+from dispatcher import DispatchRequest
+
+# Load the claude-sdk dispatcher by path (core/ can't import it directly)
+spec = importlib.util.spec_from_file_location(
+    "claude_sdk_dispatcher",
+    ROOT / "adapters" / "claude" / "sdk_dispatcher.py",
+)
+mod = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mod)
+
+
+SCENARIOS = [
+    {
+        "name": "F2-decompose-small",
+        "formula": "analyst",
+        "agent_file": str(ROOT / "src/core/thinking_os/agents/analyst.md"),
+        "prompt": (
+            "Decompose this task: 'Add a health-check endpoint /healthz to "
+            "a FastAPI service that returns {status: ok, version: X}.' "
+            "Keep scope trivial. Emit a small JSON slice."
+        ),
+        "input_slice": {"task_description": "add /healthz endpoint"},
+        "timeout_s": 90,
+    },
+    {
+        "name": "F7-debug-syntax",
+        "formula": "debugger",
+        "agent_file": str(ROOT / "src/core/thinking_os/agents/debugger.md"),
+        "prompt": (
+            "Debug this Python error: "
+            "`TypeError: '<' not supported between instances of str and int` "
+            "raised in `sorted(items, key=lambda x: x['priority'])` where some "
+            "items have int priority and others have string. "
+            "Emit the 5-phase debug JSON."
+        ),
+        "input_slice": {"task_description": "fix sort TypeError"},
+        "timeout_s": 90,
+    },
+    {
+        "name": "F5-implement-trivial",
+        "formula": "implementer",
+        "agent_file": str(ROOT / "src/core/thinking_os/agents/implementer.md"),
+        "prompt": (
+            "Implement plan: add a Python function `slugify(s: str) -> str` "
+            "that lowercases, strips, and replaces spaces with '-'. "
+            "Do not actually write files; just emit the implementation JSON slice."
+        ),
+        "input_slice": {"task_description": "add slugify helper"},
+        "timeout_s": 90,
+    },
+]
+
+
+async def run_one(d, scenario) -> dict:
+    req = DispatchRequest(
+        formula_id=scenario["formula"],
+        agent_file=scenario["agent_file"],
+        prompt=scenario["prompt"],
+        input_slice=scenario["input_slice"],
+        intensity="light",
+        timeout_s=scenario["timeout_s"],
+    )
+    t0 = time.monotonic()
+    result = await d.dispatch(req)
+    wall_ms = int((time.monotonic() - t0) * 1000)
+    return {
+        "scenario": scenario["name"],
+        "formula": scenario["formula"],
+        "status": result.status,
+        "latency_ms": result.latency_ms,
+        "wall_ms": wall_ms,
+        "output_keys": sorted(result.output_json.keys())[:6],
+        "output_size": len(json.dumps(result.output_json)),
+        "error": result.error,
+        "transcript_chars": len(result.raw_transcript or ""),
+    }
+
+
+async def main():
+    d = mod.ClaudeSDKDispatcher()
+    if not d.available():
+        print("claude-agent-sdk not available; aborting", file=sys.stderr)
+        return 1
+
+    # Narration → stderr; the final JSON document is the only thing on stdout
+    # so a caller can pipe stdout straight into a JSON parser.
+    print(
+        f"\n=== Benchmark: claude-sdk dispatcher × {len(SCENARIOS)} scenarios ===\n",
+        file=sys.stderr,
+    )
+    results = []
+    for sc in SCENARIOS:
+        print(f"→ running {sc['name']} ...", file=sys.stderr)
+        try:
+            r = await run_one(d, sc)
+        except Exception as exc:
+            r = {
+                "scenario": sc["name"],
+                "formula": sc["formula"],
+                "status": "crashed",
+                "error": f"{type(exc).__name__}: {exc}",
+            }
+        results.append(r)
+        print(
+            f"  status={r.get('status')} latency={r.get('latency_ms')}ms "
+            f"output_keys={r.get('output_keys')}",
+            file=sys.stderr,
+        )
+
+    print("\n=== Sequential summary ===", file=sys.stderr)
+    ok = sum(1 for r in results if r["status"] == "ok")
+    total_ms = sum(r.get("latency_ms", 0) for r in results)
+    print(f"ok: {ok}/{len(results)}   total_latency: {total_ms}ms", file=sys.stderr)
+
+    # ---- Parallel scenario: F7 + F5 concurrently ----
+    print("\n=== Parallel scenario: F7+F5 via asyncio.gather ===", file=sys.stderr)
+    t0 = time.monotonic()
+    par_results = await asyncio.gather(
+        run_one(d, SCENARIOS[1]),  # F7
+        run_one(d, SCENARIOS[2]),  # F5
+        return_exceptions=True,
+    )
+    par_wall_ms = int((time.monotonic() - t0) * 1000)
+    par_ok = sum(1 for r in par_results if isinstance(r, dict) and r.get("status") == "ok")
+    seq_equivalent = results[1].get("latency_ms", 0) + results[2].get("latency_ms", 0)
+    print(
+        f"parallel wall: {par_wall_ms}ms   "
+        f"vs sequential equivalent: {seq_equivalent}ms   "
+        f"speedup: {seq_equivalent / par_wall_ms:.2f}x",
+        file=sys.stderr,
+    )
+
+    print("\n=== Full results ===", file=sys.stderr)
+    print(
+        json.dumps(
+            {
+                "sequential": results,
+                "parallel": par_results,
+                "parallel_wall_ms": par_wall_ms,
+                "sequential_equivalent_ms": seq_equivalent,
+            },
+            indent=2,
+            default=str,
+        )
+    )
+    return 0 if ok == len(results) and par_ok == 2 else 2
+
+
+if __name__ == "__main__":
+    sys.exit(asyncio.run(main()))

scripts/capture_golden.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+"""Capture golden snapshots of `cos init` output for every (agent × stack).
+
+Writes to `tests/golden/<section_id>/` where section_id mirrors the
+manifest naming (claude_base, claude_django, claude_nextjs, codex_*).
+
+Runtime state files (listed below) are excluded so that repeated captures
+produce byte-identical output.
+
+Usage:
+    uv run python scripts/capture_golden.py           # refresh all sections
+    uv run python scripts/capture_golden.py --section claude_base
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import click
+
+from scripts.golden_sections import SECTIONS
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+GOLDEN_DIR = REPO_ROOT / "tests" / "golden"
+
+# Files created at runtime — never tracked in golden.
+# Must match cli/doctor.py::RUNTIME_PATHS.
+RUNTIME_PATHS = {
+    ".coding-os/coding-os.db",
+    ".coding-os/coding-os.db-shm",
+    ".coding-os/coding-os.db-wal",
+    ".coding-os/session-id",
+    ".coding-os/.thinking_os-gate",
+    ".coding-os/.task-current",
+    ".coding-os/.zoom-checkpoint",
+    ".coding-os/.last-verify",
+    ".coding-os/.fresh-init",
+}
+IGNORED_PREFIXES = (".git/", "node_modules/", ".venv/", ".build/", ".ruff_cache/", "__pycache__/")
+
+FIXTURE_NAME = "cos-golden-fixture"
+FROZEN_DATE = "2026-01-01"  # passed to `cos init --today` for determinism
+
+
+def _scaffold(agent: str, templates: list[str], target: Path) -> None:
+    cmd = [
+        sys.executable,
+        "-m",
+        "cli.main",
+        "init",
+        "--agent",
+        agent,
+        "--project-dir",
+        str(target.parent),
+        "--name",
+        target.name,
+        "--no-git",
+        "--force",
+        "--no-register",
+        # Pin the full module surface so a default-profile flip never silently
+        # shrinks the goldens (TASK-509); per-profile scaffolds are tested apart.
+        "--profile",
+        "full",
+        # Golden tracks scaffold structure only; the doc index lives in the
+        # gitignored runtime DB. Skipping it drops the ~15s embedding-model
+        # load per section (8× faster capture, no perceived hang).
+        "--no-index",
+        "--today",
+        FROZEN_DATE,
+    ]
+    for t in templates:
+        cmd.extend(["--template", t])
+    env = os.environ.copy()
+    env["PYTHONPATH"] = str(REPO_ROOT)
+    subprocess.run(
+        cmd,
+        cwd=str(REPO_ROOT),
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=180,
+        check=True,
+    )
+
+
+# Some adapters (codex) bake the absolute install path into generated configs.
+# Normalise it to a sandbox placeholder AT CAPTURE so the committed golden is
+# deterministic regardless of the temp dir — otherwise the tree is perpetually
+# dirty (audit F14). Must match test_golden_parity.py's compare-time regex.
+_SANDBOX_RE = re.compile(rb"/[^\s\"'`]*?/" + re.escape(FIXTURE_NAME.encode()) + rb"/")
+
+
+def _copy_filtered(src_root: Path, dst_root: Path) -> int:
+    """Copy files from src to dst, skipping runtime/ignored paths. Return count."""
+    if dst_root.exists():
+        shutil.rmtree(dst_root)
+    dst_root.mkdir(parents=True)
+    count = 0
+    for f in sorted(src_root.rglob("*")):
+        if not f.is_file():
+            continue
+        rel = f.relative_to(src_root).as_posix()
+        if rel in RUNTIME_PATHS:
+            continue
+        if any(rel.startswith(p) for p in IGNORED_PREFIXES):
+            continue
+        dst = dst_root / rel
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        raw = f.read_bytes()
+        normalised = _SANDBOX_RE.sub(b"__SANDBOX__/", raw)
+        if normalised != raw:
+            dst.write_bytes(normalised)
+        else:
+            shutil.copy2(f, dst)
+        count += 1
+    return count
+
+
+@click.command()
+@click.option("--section", default=None, help="Only capture a specific section id.")
+def main(section: str | None) -> None:
+    targets = [s for s in SECTIONS if section is None or s[0] == section]
+    if not targets:
+        click.echo(f"No matching section: {section}", err=True)
+        sys.exit(2)
+
+    total = len(targets)
+    click.echo(
+        f"[golden] capturing {total} section(s) — each runs a full `cos init` "
+        f"(~10-20s each, no output while it scaffolds)."
+    )
+    with tempfile.TemporaryDirectory(prefix="cos-golden-") as tmp:
+        tmp_dir = Path(tmp)
+        for i, (section_id, agent, templates) in enumerate(targets, 1):
+            stack = ", ".join(templates) if templates else "base"
+            # Emit BEFORE the slow scaffold so the user sees which section is
+            # running (and that it's not hung). Flush — output is block-buffered
+            # when piped through `make`.
+            click.echo(f"  → [{i}/{total}] {section_id} ({agent}/{stack}) — scaffolding…")
+            sys.stdout.flush()
+            sandbox_parent = tmp_dir / section_id
+            sandbox_parent.mkdir(parents=True)
+            sandbox = sandbox_parent / FIXTURE_NAME
+            try:
+                _scaffold(agent, templates, sandbox)
+            except subprocess.CalledProcessError as exc:
+                click.echo(
+                    f"[golden] {section_id}: init failed\n{exc.stderr}",
+                    err=True,
+                )
+                sys.exit(1)
+            dst = GOLDEN_DIR / section_id
+            count = _copy_filtered(sandbox, dst)
+            click.echo(
+                f"  ✓ [{i}/{total}] {section_id}: {count} files → {dst.relative_to(REPO_ROOT)}"
+            )
+            sys.stdout.flush()
+
+    click.echo(f"[golden] done — wrote {total} section(s) to {GOLDEN_DIR.relative_to(REPO_ROOT)}")
+
+
+if __name__ == "__main__":
+    main()

scripts/check_graph_phantoms.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Graph phantom-regression gate (TASK-461 / audit A2).
+
+The graph the meta-graph-first rule tells agents to TRUST went healthy:false with
+~70 phantom `cursor-*` nodes (adapter-removal staleness) and nothing caught it.
+A raw `healthy:false` gate is the wrong signal — it is persistently false from
+benign `orphaned_external_unresolved` (unresolved third-party imports) and
+`files_with_parse_errors`, so it would be permanently red. The real corruption
+signal is `orphaned_phantom`: a code node referenced but never defined.
+
+This checker reads `cos graph-doctor` and fails when the phantom count EXCEEDS a
+baseline ceiling — catching a regression (a spike like the 70 cursor nodes)
+without demanding the graph first be cleaned to zero. Ratchet the baseline DOWN
+as phantoms are triaged; never up without a reason.
+
+Cost decision (acceptance #3): a full `cos graph-reindex` of the meta-repo
+(~44K nodes) takes minutes and the graph DB is gitignored (absent in CI), so this
+is wired into the NIGHTLY slow suite (45-min budget), not the per-PR gate. Per-PR
+graph building was rejected as too slow for the value.
+
+Usage:
+    python src/scripts/check_graph_phantoms.py [--baseline N]
+Exit 0 if phantom_count <= baseline, else 1.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+
+DEFAULT_BASELINE = 3  # current live count (2026-06-19); ratchet down, never up.
+
+
+def _phantom_count() -> tuple[int, list[dict]]:
+    proc = subprocess.run(
+        ["cos", "graph-doctor"],
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    data = json.loads(proc.stdout).get("data", {})
+    issues = data.get("issues", [])
+    phantom = sum(int(i.get("count", 0)) for i in issues if i.get("category") == "orphaned_phantom")
+    samples = [i for i in issues if i.get("category") == "orphaned_phantom"]
+    return phantom, samples
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Graph phantom-regression gate")
+    parser.add_argument("--baseline", type=int, default=DEFAULT_BASELINE)
+    args = parser.parse_args()
+
+    try:
+        phantom, samples = _phantom_count()
+    except (subprocess.SubprocessError, json.JSONDecodeError, OSError) as exc:
+        print(f"check_graph_phantoms: could not read graph-doctor: {exc}", file=sys.stderr)
+        return 1
+
+    if phantom > args.baseline:
+        print(
+            f"GRAPH REGRESSION: {phantom} orphaned_phantom nodes > baseline {args.baseline}.",
+            file=sys.stderr,
+        )
+        print(json.dumps(samples, indent=2)[:2000], file=sys.stderr)
+        print("Run `cos graph-doctor --fix` / triage extractor staleness.", file=sys.stderr)
+        return 1
+
+    print(f"graph phantom check OK: {phantom} <= baseline {args.baseline}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())