agentforge-py 0.2.1__py3-none-any.whl

agentforge/cli/debug_cmd.py

@@ -0,0 +1,168 @@
+"""`agentforge debug` — interactive replay REPL (feat-017 chunk 6).
+
+    agentforge debug --replay <RUN_ID> [--path agentforge.yaml]
+
+Loads `category="__step"` claims for the run, exposes a `cmd.Cmd`
+prompt with:
+
+    step / s        advance to the next step
+    back / b        rewind one step
+    state           print the current step's payload
+    inspect FIELD   print payload[FIELD] (dotted-path supported)
+    steps           list all step kinds + indices
+    quit / q        exit
+
+No external dependencies — uses stdlib `cmd.Cmd`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import cmd
+import json
+import sys
+from pathlib import Path
+from typing import IO, Any
+
+from agentforge.cli._build import build_memory_from_config
+from agentforge.recording import STEP_CATEGORY
+
+
+def register_debug_cmd(sub: argparse._SubParsersAction) -> None:  # type: ignore[type-arg]
+    parser = sub.add_parser(
+        "debug",
+        help="Interactive REPL to step through a recorded run.",
+    )
+    parser.add_argument("--replay", required=True, metavar="RUN_ID")
+    parser.add_argument("--path", type=Path, default=None)
+    parser.add_argument("--env", default=None)
+    parser.add_argument("--override", action="append", default=[])
+    parser.set_defaults(_handler=_debug_handler)
+
+
+def _debug_handler(args: argparse.Namespace) -> int:
+    return asyncio.run(_dispatch(args))
+
+
+async def _dispatch(args: argparse.Namespace) -> int:
+    from agentforge_core.config.loader import load_config  # noqa: PLC0415
+
+    config = load_config(args.path, env=args.env, overrides=list(args.override) or None)
+    memory = build_memory_from_config(config)
+    if memory is None:
+        sys.stderr.write("agentforge debug: modules.memory must be configured.\n")
+        return 1
+    steps = await memory.query(category=STEP_CATEGORY, run_id=args.replay, limit=10_000)
+    if not steps:
+        sys.stderr.write(f"agentforge debug: no steps recorded for run_id={args.replay!r}.\n")
+        return 1
+    repl = _ReplayREPL([s.payload for s in steps])
+    repl.cmdloop()
+    return 0
+
+
+class _ReplayREPL(cmd.Cmd):
+    """Interactive replay stepper. Output is plain text — no Rich."""
+
+    intro = "agentforge debug — recorded-run replay. Type 'help' for commands."
+    prompt = "(agentforge) "
+
+    def __init__(
+        self,
+        steps: list[dict[str, Any]],
+        *,
+        stdin: IO[str] | None = None,
+        stdout: IO[str] | None = None,
+    ) -> None:
+        super().__init__(stdin=stdin, stdout=stdout)
+        self._steps = steps
+        self._cursor = 0
+
+    def do_step(self, arg: str) -> bool:
+        del arg
+        if self._cursor >= len(self._steps):
+            self._w("END of recording.\n")
+            return False
+        self._w(_format_step(self._cursor, self._steps[self._cursor]))
+        self._cursor += 1
+        return False
+
+    do_s = do_step
+
+    def do_back(self, arg: str) -> bool:
+        del arg
+        if self._cursor <= 0:
+            self._w("at start.\n")
+            return False
+        self._cursor -= 1
+        self._w(_format_step(self._cursor, self._steps[self._cursor]))
+        return False
+
+    do_b = do_back
+
+    def do_state(self, arg: str) -> bool:
+        del arg
+        if self._cursor == 0:
+            self._w("no step entered yet.\n")
+            return False
+        idx = self._cursor - 1
+        self._w(json.dumps(self._steps[idx], indent=2) + "\n")
+        return False
+
+    def do_inspect(self, arg: str) -> bool:
+        if self._cursor == 0:
+            self._w("no step entered yet.\n")
+            return False
+        idx = self._cursor - 1
+        payload: Any = self._steps[idx]
+        for part in arg.split("."):
+            if not part:
+                continue
+            if isinstance(payload, dict) and part in payload:
+                payload = payload[part]
+            else:
+                self._w(f"no such field: {arg}\n")
+                return False
+        self._w(json.dumps(payload, indent=2) + "\n")
+        return False
+
+    def do_steps(self, arg: str) -> bool:
+        del arg
+        for i, s in enumerate(self._steps):
+            self._w(f"  {i:3d}  {s['kind']:<8}  iter={s['iteration']}\n")
+        return False
+
+    def do_quit(self, arg: str) -> bool:
+        del arg
+        return True
+
+    do_q = do_quit
+    do_EOF = do_quit  # noqa: N815 — cmd.Cmd protocol uses this exact name
+
+    def _w(self, text: str) -> None:
+        out = self.stdout or sys.stdout
+        out.write(text)
+        out.flush()
+
+
+_CONTENT_PREVIEW_LEN = 80
+_CONTENT_TRUNCATE_AT = 77
+
+
+def _format_step(idx: int, payload: dict[str, Any]) -> str:
+    line = f"[{idx:3d}] kind={payload.get('kind')} iter={payload.get('iteration')}"
+    content = payload.get("content")
+    if isinstance(content, str):
+        preview = (
+            content
+            if len(content) <= _CONTENT_PREVIEW_LEN
+            else content[:_CONTENT_TRUNCATE_AT] + "..."
+        )
+        line += f"  content={preview!r}"
+    elif content is not None:
+        line += f"  content={type(content).__name__}"
+    return line + "\n"
+
+
+__all__ = ["register_debug_cmd"]

agentforge/cli/docs_cmd.py

@@ -0,0 +1,217 @@
+"""`agentforge docs` — open and audit runbooks (feat-019 chunk 7).
+
+Four subcommands:
+
+- `docs` — interactive picker (lists every runbook, prompts for
+  the one to open).
+- `docs <topic>` — open by name. Matches filename stem (e.g.
+  `02-add-a-tool`), bare number (`2`), or alias (`add-tool`,
+  `add-mcp`).
+- `docs check` — diff local runbook content against the
+  framework's current bundle; report drift; suggest
+  `agentforge upgrade`.
+- `docs serve` — local HTTP browser of the runbook tree.
+
+The runbooks live under `docs/runbooks/` relative to the
+project's working directory (overrideable via
+`agentforge.yaml > docs.runbooks_path`). The framework's bundled
+copies live inside the `agentforge` wheel — `docs check`
+compares the two.
+"""
+
+from __future__ import annotations
+
+import argparse
+import http.server
+import os
+import re
+import socketserver
+import subprocess  # nosec B404 — opens user-chosen $EDITOR; argv list, no shell
+import sys
+from importlib import resources
+from pathlib import Path
+
+from agentforge.cli._scaffold_state import _strip_marker_for_hash, hash_content
+
+
+def register_docs_cmd(sub: argparse._SubParsersAction) -> None:  # type: ignore[type-arg]
+    """Attach `agentforge docs` to the parent subparser action."""
+    parser = sub.add_parser(
+        "docs",
+        help="Open / list / audit project runbooks.",
+        description="Open and audit AgentForge runbooks shipped into this project.",
+    )
+    parser.add_argument(
+        "topic",
+        nargs="?",
+        default=None,
+        help=(
+            "Runbook to open. Matches filename stem (02-add-a-tool), "
+            "bare number (2), or alias (add-tool / add-memory)."
+        ),
+    )
+    parser.add_argument(
+        "--path",
+        type=Path,
+        default=None,
+        help="Override the runbooks directory (default: ./docs/runbooks).",
+    )
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="Compare local runbooks against the framework's bundle; report drift.",
+    )
+    parser.add_argument(
+        "--serve",
+        action="store_true",
+        help="Start a local HTTP browser of the runbook tree on port 8765.",
+    )
+    parser.set_defaults(_handler=_run_docs)
+
+
+def _run_docs(args: argparse.Namespace) -> int:
+    runbooks_dir = args.path if args.path is not None else Path.cwd() / "docs" / "runbooks"
+    if not runbooks_dir.exists():
+        sys.stderr.write(
+            f"agentforge docs: {runbooks_dir} does not exist. "
+            "Scaffold via `agentforge new` to install runbooks.\n"
+        )
+        return 1
+    if args.check:
+        return _do_check(runbooks_dir)
+    if args.serve:
+        return _do_serve(runbooks_dir)
+    if args.topic is None:
+        return _do_list(runbooks_dir)
+    return _do_open(runbooks_dir, args.topic)
+
+
+def _do_list(runbooks_dir: Path) -> int:
+    """Print every runbook in numeric order."""
+    for runbook in _scan(runbooks_dir):
+        print(f"  {runbook.stem:<40}  {runbook}")
+    return 0
+
+
+def _do_open(runbooks_dir: Path, topic: str) -> int:
+    """Resolve `topic` to a single runbook and open it via $EDITOR / less."""
+    match = _resolve_topic(runbooks_dir, topic)
+    if match is None:
+        sys.stderr.write(
+            f"agentforge docs: no runbook matches {topic!r}. Try `agentforge docs` to list.\n"
+        )
+        return 1
+    editor = os.environ.get("EDITOR")
+    if editor:
+        return subprocess.run(  # noqa: S603  # nosec B603 — $EDITOR is user's own
+            [editor, str(match)],
+            check=False,
+        ).returncode
+    # No EDITOR — print to stdout so the developer can pipe to less.
+    sys.stdout.write(match.read_text(encoding="utf-8"))
+    return 0
+
+
+def _do_check(runbooks_dir: Path) -> int:
+    """Diff local runbook hashes against the framework's bundle."""
+    bundled = _bundled_runbooks_dir()
+    if bundled is None:
+        sys.stderr.write(
+            "agentforge docs check: framework bundle not found — "
+            "running from a non-standard install?\n"
+        )
+        return 1
+    drift: list[str] = []
+    for local in _scan(runbooks_dir):
+        rel = local.relative_to(runbooks_dir)
+        # Bundled file may carry `.tmpl` suffix; check both.
+        candidates = [bundled / rel, bundled / (str(rel) + ".tmpl")]
+        bundled_path = next((c for c in candidates if c.exists()), None)
+        if bundled_path is None:
+            drift.append(f"  +local  {rel}")
+            continue
+        local_hash = hash_content(_strip_marker_for_hash(local.read_text(encoding="utf-8")))
+        bundled_hash = hash_content(bundled_path.read_text(encoding="utf-8"))
+        if local_hash != bundled_hash:
+            drift.append(f"  ~drift  {rel}")
+    if drift:
+        print("Runbook drift detected:")
+        for line in drift:
+            print(line)
+        print("\nRun `agentforge upgrade` to merge framework updates.")
+        return 1
+    print("All runbooks in sync with framework bundle.")
+    return 0
+
+
+def _do_serve(runbooks_dir: Path, port: int = 8765) -> int:
+    """Start a basic HTTP server over the runbooks directory."""
+    handler_cls = http.server.SimpleHTTPRequestHandler
+    cwd = os.getcwd()
+    os.chdir(runbooks_dir)
+    try:
+        with socketserver.TCPServer(("127.0.0.1", port), handler_cls) as httpd:
+            sys.stdout.write(
+                f"agentforge docs: serving {runbooks_dir} at http://127.0.0.1:{port}/\n"
+                "Press Ctrl-C to stop.\n"
+            )
+            httpd.serve_forever()
+    except KeyboardInterrupt:
+        sys.stdout.write("\nstopped.\n")
+    finally:
+        os.chdir(cwd)
+    return 0
+
+
+def _scan(runbooks_dir: Path) -> list[Path]:
+    """Walk the runbooks directory and return numbered runbooks in order."""
+    return sorted(p for p in runbooks_dir.glob("*.md") if _RUNBOOK_RE.match(p.name))
+
+
+_RUNBOOK_RE = re.compile(r"^\d{2}-[a-z0-9-]+\.md$")
+
+
+def _resolve_topic(runbooks_dir: Path, topic: str) -> Path | None:
+    """Resolve `topic` to a runbook path.
+
+    Match precedence:
+    1. Exact filename stem (`02-add-a-tool`).
+    2. Bare number (`2` → `02-...`).
+    3. Alias (`add-tool` → matches any runbook whose body
+       (after the leading number) contains the alias).
+    """
+    if topic.endswith(".md"):
+        candidate = runbooks_dir / topic
+        if candidate.exists():
+            return candidate
+    candidate = runbooks_dir / f"{topic}.md"
+    if candidate.exists():
+        return candidate
+    if topic.isdigit():
+        num = f"{int(topic):02d}-"
+        for p in _scan(runbooks_dir):
+            if p.name.startswith(num):
+                return p
+    for p in _scan(runbooks_dir):
+        # Drop the `NN-` prefix when comparing aliases.
+        body = p.stem.split("-", 1)[1] if "-" in p.stem else p.stem
+        if topic in body:
+            return p
+    return None
+
+
+def _bundled_runbooks_dir() -> Path | None:
+    """Return the on-disk path of the framework's bundled runbooks."""
+    try:
+        traversable = resources.files("agentforge.templates").joinpath(
+            "_shared", "docs", "runbooks"
+        )
+    except ModuleNotFoundError:
+        return None
+    with resources.as_file(traversable) as path:
+        if not path.exists() or not path.is_dir():
+            return None
+        return Path(path)
+
+
+__all__ = ["register_docs_cmd"]

agentforge/cli/eval_cmd.py

@@ -0,0 +1,181 @@
+"""`agentforge eval` — run an agent against JSONL fixtures (feat-017 chunk 5).
+
+Each fixture line:
+
+    {"task": "...", "expected": "...", "metadata": {...}}
+
+The command builds the agent once, iterates fixtures, runs the agent
+on each, aggregates per-evaluator scores, and threshold-checks the
+mean. Output formats: rich (default), json, junit. Exit 5 on
+threshold failure.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from pathlib import Path
+from typing import Any
+from xml.etree import (
+    ElementTree as ET,  # nosec B405 — output only; never parses untrusted XML
+)
+
+from pydantic import ValidationError
+
+from agentforge.cli._build import load_and_build
+from agentforge.cli.run_cmd import (
+    EXIT_CONFIG_INVALID,
+    EXIT_GENERIC,
+    EXIT_OK,
+)
+
+EXIT_THRESHOLD = 5
+
+
+def register_eval_cmd(sub: argparse._SubParsersAction) -> None:  # type: ignore[type-arg]
+    parser = sub.add_parser(
+        "eval",
+        help="Run an agent against JSONL fixtures and apply evaluators.",
+    )
+    parser.add_argument("--fixtures", type=Path, required=True, help="Path to JSONL fixtures.")
+    parser.add_argument(
+        "--threshold",
+        type=float,
+        default=None,
+        help="Minimum mean score across all evaluators. Exit 5 if below.",
+    )
+    parser.add_argument(
+        "--output-format",
+        choices=("rich", "json", "junit"),
+        default="rich",
+    )
+    parser.add_argument("--path", type=Path, default=None)
+    parser.add_argument("--env", default=None)
+    parser.add_argument("--override", action="append", default=[])
+    parser.set_defaults(_handler=_eval_handler)
+
+
+def _eval_handler(args: argparse.Namespace) -> int:
+    return asyncio.run(_dispatch(args))
+
+
+async def _dispatch(args: argparse.Namespace) -> int:
+    try:
+        fixtures = _load_fixtures(args.fixtures)
+    except (OSError, json.JSONDecodeError) as exc:
+        sys.stderr.write(f"agentforge eval: failed to read fixtures: {exc}\n")
+        return EXIT_GENERIC
+
+    try:
+        agent = await load_and_build(
+            path=args.path,
+            env=args.env,
+            overrides=list(args.override) or None,
+        )
+    except ValidationError as exc:
+        sys.stderr.write(f"agentforge eval: config invalid:\n{exc}\n")
+        return EXIT_CONFIG_INVALID
+
+    results: list[dict[str, Any]] = []
+    for fix in fixtures:
+        run_result = await agent.run(fix["task"])
+        results.append(
+            {
+                "task": fix["task"],
+                "expected": fix.get("expected"),
+                "output": run_result.output,
+                "scores": [score.model_dump(mode="json") for score in run_result.eval_scores],
+                "run_id": run_result.run_id,
+            }
+        )
+
+    mean = _mean_score(results)
+    fail = args.threshold is not None and mean < args.threshold
+
+    _emit(results, mean, args.threshold, args.output_format, fail=fail)
+    return EXIT_THRESHOLD if fail else EXIT_OK
+
+
+def _load_fixtures(path: Path) -> list[dict[str, Any]]:
+    fixtures: list[dict[str, Any]] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        stripped = line.strip()
+        if not stripped:
+            continue
+        fixtures.append(json.loads(stripped))
+    return fixtures
+
+
+def _mean_score(results: list[dict[str, Any]]) -> float:
+    values: list[float] = []
+    for r in results:
+        for s in r["scores"]:
+            score = s.get("score")
+            if isinstance(score, int | float):
+                values.append(float(score))
+    return sum(values) / len(values) if values else 0.0
+
+
+def _emit(
+    results: list[dict[str, Any]],
+    mean: float,
+    threshold: float | None,
+    fmt: str,
+    *,
+    fail: bool,
+) -> None:
+    if fmt == "json":
+        print(
+            json.dumps(
+                {
+                    "fixtures": len(results),
+                    "mean_score": mean,
+                    "threshold": threshold,
+                    "passed": not fail,
+                    "results": results,
+                },
+                indent=2,
+            )
+        )
+        return
+    if fmt == "junit":
+        print(_to_junit(results, mean, fail=fail))
+        return
+    # Rich-or-plain summary.
+    print(f"fixtures: {len(results)}")
+    print(f"mean_score: {mean:.4f}")
+    if threshold is not None:
+        print(f"threshold: {threshold:.4f}  →  {'FAIL' if fail else 'PASS'}")
+
+
+def _to_junit(results: list[dict[str, Any]], mean: float, *, fail: bool) -> str:
+    suite = ET.Element(
+        "testsuite",
+        attrib={
+            "name": "agentforge-eval",
+            "tests": str(len(results)),
+            "failures": "1" if fail else "0",
+        },
+    )
+    for i, r in enumerate(results):
+        case = ET.SubElement(
+            suite,
+            "testcase",
+            attrib={"name": f"fixture[{i}]", "classname": "agentforge.eval"},
+        )
+        for score in r["scores"]:
+            if score.get("score", 1.0) < 1.0:
+                f = ET.SubElement(case, "failure", attrib={"type": "score"})
+                f.text = json.dumps(score)
+    if fail:
+        f = ET.SubElement(
+            suite,
+            "system-err",
+        )
+        f.text = f"mean_score {mean:.4f} below threshold"
+    return ET.tostring(suite, encoding="unicode")
+
+
+__all__ = ["register_eval_cmd"]

agentforge/cli/health_cmd.py

@@ -0,0 +1,139 @@
+"""`agentforge health` — preflight checks (feat-017 chunk 8).
+
+Renamed from the spec's `agentforge status` to avoid colliding with
+the feat-011 scaffolding-state `agentforge status`. Checks:
+
+1. Config loads + validates.
+2. Every installed module resolvable via `Resolver.list_installed`.
+3. Every backend declared under `modules.{memory,graph,retriever}`
+   reachable (instantiate, `__aenter__`/`close()`).
+4. Provider construction is exercised as a no-API probe.
+
+Exit codes: 0 all OK, 1 any FAIL, 2 config invalid.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from agentforge_core.config.loader import load_config
+from agentforge_core.config.schema import AgentForgeConfig
+from agentforge_core.production.exceptions import ModuleError
+from agentforge_core.resolver import Resolver
+from pydantic import ValidationError
+
+from agentforge.cli._build import (
+    build_memory_from_config,
+)
+
+
+def register_health_cmd(sub: argparse._SubParsersAction) -> None:  # type: ignore[type-arg]
+    parser = sub.add_parser(
+        "health",
+        help="Preflight: config valid, modules loadable, backends reachable.",
+    )
+    parser.add_argument("--path", type=Path, default=None)
+    parser.add_argument("--env", default=None)
+    parser.add_argument("--override", action="append", default=[])
+    parser.add_argument(
+        "--output-format",
+        choices=("rich", "plain", "json"),
+        default="plain",
+    )
+    parser.set_defaults(_handler=_health_handler)
+
+
+def _health_handler(args: argparse.Namespace) -> int:
+    return asyncio.run(_dispatch(args))
+
+
+async def _dispatch(args: argparse.Namespace) -> int:
+    checks: list[dict[str, Any]] = []
+
+    try:
+        config = load_config(args.path, env=args.env, overrides=list(args.override) or None)
+        checks.append({"name": "config", "kind": "config", "ok": True, "detail": "valid"})
+    except ValidationError as exc:
+        _emit([{"name": "config", "kind": "config", "ok": False, "detail": str(exc)}], args)
+        return 2
+    except ModuleError as exc:
+        _emit([{"name": "config", "kind": "config", "ok": False, "detail": str(exc)}], args)
+        return 2
+
+    checks.extend(_check_modules())
+    checks.extend(await _check_backends(config))
+
+    ok = all(c["ok"] for c in checks)
+    _emit(checks, args)
+    return 0 if ok else 1
+
+
+def _check_modules() -> list[dict[str, Any]]:
+    """Walk Resolver.list_installed and assert each module resolvable."""
+    out: list[dict[str, Any]] = []
+    resolver = Resolver.global_()
+    for info in resolver.list_installed():
+        try:
+            resolver.resolve(info.category, info.name)
+        except ModuleError as exc:
+            out.append(
+                {
+                    "name": f"{info.category}:{info.name}",
+                    "kind": "module",
+                    "ok": False,
+                    "detail": str(exc),
+                }
+            )
+        else:
+            out.append(
+                {
+                    "name": f"{info.category}:{info.name}",
+                    "kind": "module",
+                    "ok": True,
+                    "detail": "resolvable",
+                }
+            )
+    return out
+
+
+async def _check_backends(config: AgentForgeConfig) -> list[dict[str, Any]]:
+    """For each configured backend, attempt to instantiate + close."""
+    out: list[dict[str, Any]] = []
+
+    if config.modules.memory is not None:
+        out.append(await _probe("memory", lambda: build_memory_from_config(config)))
+    return out
+
+
+async def _probe(label: str, factory: Any) -> dict[str, Any]:
+    try:
+        instance = factory()
+        if instance is None:
+            return {"name": label, "kind": "backend", "ok": True, "detail": "none configured"}
+        init = getattr(instance, "init_schema", None)
+        if callable(init):
+            await init()
+        close = getattr(instance, "close", None)
+        if callable(close):
+            await close()
+    except (ModuleError, OSError) as exc:
+        return {"name": label, "kind": "backend", "ok": False, "detail": str(exc)}
+    return {"name": label, "kind": "backend", "ok": True, "detail": "reachable"}
+
+
+def _emit(checks: list[dict[str, Any]], args: argparse.Namespace) -> None:
+    if args.output_format == "json":
+        ok = all(c["ok"] for c in checks)
+        print(json.dumps({"checks": checks, "ok": ok}, indent=2))
+        return
+    for c in checks:
+        status = "OK  " if c["ok"] else "FAIL"
+        sys.stdout.write(f"{status}  {c['kind']:<8}  {c['name']:<32}  {c['detail']}\n")
+
+
+__all__ = ["register_health_cmd"]