doesitfire 0.1.2__py3-none-any.whl

doesitfire/__init__.py

@@ -0,0 +1,3 @@
+"""doesitfire: test whether agent skills (SKILL.md) actually activate in real coding agents."""
+
+__version__ = "0.1.2"

doesitfire/agents.py

@@ -0,0 +1,310 @@
+"""How to run each agent CLI headless and how to tell, from its JSON events,
+whether it activated a skill.
+
+- Claude Code: a ``Skill`` tool_use in ``claude -p --output-format stream-json``.
+- OpenCode: its native ``skill`` tool in ``opencode run --format json``.
+- Codex: no activation event exists, so we look for the agent reading
+  ``<skill>/SKILL.md`` in the ``command_execution`` items of ``codex exec --json``.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+
+from .config import AgentConfig
+from .providers import Provider, container_url, resolve_key
+
+CLI_VERSIONS = {"claude-code": "2.1.317", "codex": "0.166.0", "opencode": "1.19.5"}
+SKILLS_DIRS = {"claude-code": ".claude/skills", "codex": ".agents/skills", "opencode": ".opencode/skills"}
+EXPERIMENTAL = {"codex"}  # activation is inferred from a file read, not reported by the CLI
+
+
+@dataclass
+class Invocation:
+    argv: list[str]
+    env: dict[str, str] = field(default_factory=dict)  # plain values (no secrets)
+    secret_env: dict[str, str] = field(default_factory=dict)  # passed to docker by name only
+    allow: list[str] = field(default_factory=list)  # base URLs the egress proxy must allow
+
+
+@dataclass
+class Observation:
+    activated: list[str] = field(default_factory=list)  # skill names, in order, deduplicated
+    input_tokens: int = 0
+    output_tokens: int = 0
+    reported_cost: float | None = None
+    error: str | None = None
+    tool_calls: int = 0
+    usage_seen: bool = False
+
+    def add(self, skill: str) -> None:
+        if skill not in self.activated:
+            self.activated.append(skill)
+
+
+class Agent:
+    name = ""
+
+    def __init__(self, cfg: AgentConfig, provider: Provider, max_turns: int):
+        self.cfg = cfg
+        self.provider = provider
+        self.max_turns = max_turns
+
+    @property
+    def skills_dir(self) -> str:
+        return SKILLS_DIRS[self.name]
+
+    def build(self, prompt: str) -> Invocation:  # pragma: no cover - abstract
+        raise NotImplementedError
+
+    def feed(self, line: str, obs: Observation, skills: set[str]) -> None:
+        """Update the observation with one line of the agent's stdout."""
+        line = line.strip()
+        if not line.startswith("{"):
+            return
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            return
+        if isinstance(event, dict):
+            self.on_event(event, obs, skills)
+
+    def on_event(self, event: dict, obs: Observation, skills: set[str]) -> None:  # pragma: no cover
+        raise NotImplementedError
+
+    def parse(self, stdout: str, skills: set[str]) -> Observation:
+        obs = Observation()
+        for line in stdout.splitlines():
+            self.feed(line, obs, skills)
+        return obs
+
+    def should_stop(self, obs: Observation, skills: set[str]) -> bool:
+        """Stop early once the outcome is known: a tested skill fired, or the turn cap was hit.
+
+        Only for local models; with paid providers we let the CLI finish so it reports usage.
+        """
+        if not self.provider.local:
+            return False
+        return any(s in skills for s in obs.activated) or obs.tool_calls > self.max_turns
+
+
+def _norm_skill(value) -> str:
+    name = str(value or "").strip().lstrip("/").lower()
+    return name.split(":")[-1].split()[0] if name else ""
+
+
+class ClaudeCode(Agent):
+    name = "claude-code"
+
+    def build(self, prompt: str) -> Invocation:
+        p = self.provider
+        base = container_url(p.anthropic_base or "")
+        env = {
+            "ANTHROPIC_BASE_URL": base,
+            "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
+            "DISABLE_AUTOUPDATER": "1",
+            "DISABLE_TELEMETRY": "1",
+            "DISABLE_ERROR_REPORTING": "1",
+            "API_TIMEOUT_MS": "900000",
+        }
+        secret: dict[str, str] = {}
+        key = resolve_key(self.name, p)
+        if p.name == "ollama":
+            env["ANTHROPIC_AUTH_TOKEN"] = "ollama"
+        elif p.name == "openrouter" and key:
+            secret["ANTHROPIC_AUTH_TOKEN"] = key[1]
+            env["ANTHROPIC_API_KEY"] = ""
+        elif key:
+            secret[key[0]] = key[1]
+        if "claude" not in self.cfg.model.lower():
+            # Non-Claude models: route every internal model slot to the same model, and give
+            # the skill listing the budget a 200k-context Claude model would get. Without it
+            # Claude Code lists skills by name only, which no real user would see.
+            for slot in (
+                "ANTHROPIC_DEFAULT_HAIKU_MODEL",
+                "ANTHROPIC_DEFAULT_SONNET_MODEL",
+                "ANTHROPIC_DEFAULT_OPUS_MODEL",
+                "CLAUDE_CODE_SUBAGENT_MODEL",
+            ):
+                env[slot] = self.cfg.model
+            env["SLASH_COMMAND_TOOL_CHAR_BUDGET"] = "8000"
+        tools = "Skill,Read,Glob,Grep"
+        argv = [
+            "claude",
+            "-p",
+            prompt,
+            "--output-format",
+            "stream-json",
+            "--verbose",
+            "--model",
+            self.cfg.model,
+            "--max-turns",
+            str(self.max_turns),
+            "--tools",
+            tools,
+            "--allowedTools",
+            tools,
+            "--disallowedTools",
+            "Bash,Write,Edit,NotebookEdit,WebFetch,WebSearch,Task",
+            "--strict-mcp-config",
+        ]
+        return Invocation(argv, env, secret, [base])
+
+    def on_event(self, event: dict, obs: Observation, skills: set[str]) -> None:
+        etype = event.get("type")
+        if etype == "assistant":
+            for block in (event.get("message") or {}).get("content") or []:
+                if isinstance(block, dict) and block.get("type") == "tool_use":
+                    obs.tool_calls += 1
+                    if block.get("name") == "Skill":
+                        args = block.get("input") or {}
+                        name = _norm_skill(args.get("skill") or args.get("command") or args.get("name"))
+                        if name:
+                            obs.add(name)
+        elif etype == "result":
+            usage = event.get("usage") or {}
+            obs.usage_seen = True
+            obs.input_tokens += (
+                int(usage.get("input_tokens") or 0)
+                + int(usage.get("cache_read_input_tokens") or 0)
+                + int(usage.get("cache_creation_input_tokens") or 0)
+            )
+            obs.output_tokens += int(usage.get("output_tokens") or 0)
+            if event.get("total_cost_usd") is not None:
+                obs.reported_cost = float(event["total_cost_usd"])
+            if event.get("is_error") and event.get("subtype") not in ("error_max_turns",):
+                obs.error = str(event.get("result") or event.get("subtype") or "error")[:300]
+        elif etype == "system" and event.get("subtype") == "api_retry" and event.get("attempt", 0) >= 3:
+            obs.error = f"model API retrying (status {event.get('error_status')}, {event.get('error')})"
+
+
+class Codex(Agent):
+    name = "codex"
+
+    def build(self, prompt: str) -> Invocation:
+        p = self.provider
+        base = container_url(p.openai_base)
+        provider_toml = f'{{name="doesitfire",base_url="{base}",wire_api="responses"'
+        secret: dict[str, str] = {}
+        key = resolve_key(self.name, p)
+        if key:
+            secret[key[0]] = key[1]
+            provider_toml += f',env_key="{key[0]}"'
+        provider_toml += "}"
+        argv = [
+            "codex",
+            "exec",
+            "--json",
+            "--skip-git-repo-check",
+            "--ephemeral",
+            # Codex's own sandbox (bubblewrap) cannot start in an unprivileged container. The
+            # container is the sandbox: read-only root, no capabilities, no network but the model.
+            "--dangerously-bypass-approvals-and-sandbox",
+            "-c",
+            "analytics.enabled=false",
+            "-c",
+            "model_provider=doesitfire",
+            "-c",
+            f"model_providers.doesitfire={provider_toml}",
+            "-m",
+            self.cfg.model,
+        ]
+        argv.append(prompt)
+        return Invocation(argv, {"NO_COLOR": "1"}, secret, [base])
+
+    @staticmethod
+    def skills_in_command(command: str, skills: set[str]) -> list[str]:
+        found: list[str] = []
+        for match in re.finditer(r"([A-Za-z0-9_.-]+)/SKILL\.md", command):
+            name = match.group(1).lower()
+            if name not in found:
+                found.append(name)
+        return found
+
+    def on_event(self, event: dict, obs: Observation, skills: set[str]) -> None:
+        etype = event.get("type")
+        item = event.get("item") or {}
+        if etype == "item.started" and item.get("type") in ("command_execution", "mcp_tool_call"):
+            obs.tool_calls += 1
+            text = item.get("command") or json.dumps(item.get("arguments") or "")
+            for name in self.skills_in_command(str(text), skills):
+                obs.add(name)
+        elif etype == "turn.completed":
+            usage = event.get("usage") or {}
+            obs.usage_seen = True
+            obs.input_tokens += int(usage.get("input_tokens") or 0)
+            obs.output_tokens += int(usage.get("output_tokens") or 0)
+        elif etype in ("turn.failed", "error") and not str(event.get("message", "")).startswith("Reconnecting"):
+            msg = (event.get("error") or {}).get("message") if etype == "turn.failed" else event.get("message")
+            obs.error = str(msg or etype)[:300]
+
+
+class OpenCode(Agent):
+    name = "opencode"
+
+    def build(self, prompt: str) -> Invocation:
+        p = self.provider
+        secret: dict[str, str] = {}
+        key = resolve_key(self.name, p)
+        if p.name == "anthropic":
+            npm, base = "@ai-sdk/anthropic", container_url((p.anthropic_base or "") + "/v1")
+        else:
+            npm, base = "@ai-sdk/openai-compatible", container_url(p.openai_base)
+        options = {"baseURL": base, "apiKey": "ollama"}
+        if key:
+            secret[key[0]] = key[1]
+            options["apiKey"] = "{env:" + key[0] + "}"
+        model_cfg: dict = {"name": self.cfg.model}
+        config = {
+            "$schema": "https://opencode.ai/config.json",
+            "autoupdate": False,
+            "share": "disabled",
+            "provider": {
+                "doesitfire": {
+                    "npm": npm,
+                    "name": "doesitfire",
+                    "options": options,
+                    "models": {self.cfg.model: model_cfg},
+                }
+            },
+            "permission": {"edit": "deny", "bash": "deny", "webfetch": "deny"},
+            "agent": {"build": {"steps": self.max_turns}, "title": {"disable": True}},
+        }
+        env = {
+            "OPENCODE_CONFIG_CONTENT": json.dumps(config),
+            "OPENCODE_DISABLE_MODELS_FETCH": "1",
+            "OPENCODE_DISABLE_AUTOUPDATE": "1",
+            "OPENCODE_DISABLE_LSP_DOWNLOAD": "1",
+        }
+        argv = ["opencode", "run", "--format", "json", "-m", f"doesitfire/{self.cfg.model}", prompt]
+        return Invocation(argv, env, secret, [base])
+
+    def on_event(self, event: dict, obs: Observation, skills: set[str]) -> None:
+        etype = event.get("type")
+        part = event.get("part") or {}
+        if etype == "tool_use":
+            obs.tool_calls += 1
+            if part.get("tool") == "skill":
+                name = _norm_skill(((part.get("state") or {}).get("input") or {}).get("name"))
+                if name:
+                    obs.add(name)
+        elif etype == "step_finish":
+            tokens = part.get("tokens") or {}
+            cache = tokens.get("cache") or {}
+            obs.usage_seen = True
+            obs.input_tokens += int(tokens.get("input") or 0) + int(cache.get("read") or 0)
+            obs.output_tokens += int(tokens.get("output") or 0) + int(tokens.get("reasoning") or 0)
+            if part.get("cost") is not None:
+                obs.reported_cost = (obs.reported_cost or 0.0) + float(part["cost"])
+        elif etype == "error":
+            err = event.get("error") or {}
+            obs.error = str((err.get("data") or {}).get("message") or err.get("name") or "error")[:300]
+
+
+AGENT_CLASSES: dict[str, type[Agent]] = {"claude-code": ClaudeCode, "codex": Codex, "opencode": OpenCode}
+
+
+def make_agent(cfg: AgentConfig, provider: Provider, max_turns: int) -> Agent:
+    return AGENT_CLASSES[cfg.name](cfg, provider, max_turns)

doesitfire/cli.py

@@ -0,0 +1,263 @@
+"""doesitfire command line: init, run, report, demo."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import sys
+import tempfile
+import webbrowser
+from importlib import resources
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+from rich.markup import escape
+
+from . import __version__
+from .config import CONFIG_NAME, find_skill_dirs, load_config, render_template
+from .errors import BudgetExceeded, DoesItFireError
+from .metrics import check_thresholds, summarize
+
+app = typer.Typer(
+    add_completion=False,
+    no_args_is_help=True,
+    rich_markup_mode="rich",
+    help="Does your SKILL.md actually fire? Test skill activation in Claude Code, Codex and OpenCode.",
+)
+console = Console()
+err = Console(stderr=True)
+
+
+def _version(value: bool) -> None:
+    if value:
+        console.print(f"doesitfire {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def _root(
+    version: Annotated[
+        bool, typer.Option("--version", callback=_version, is_eager=True, help="Show the version.")
+    ] = False,
+) -> None:
+    pass
+
+
+def write_outputs(results: dict, out: Path) -> dict[str, Path]:
+    from .html import write_html
+    from .report import write_badges, write_junit
+
+    out.mkdir(parents=True, exist_ok=True)
+    paths = {
+        "json": out / "results.json",
+        "html": out / "report.html",
+        "junit": out / "junit.xml",
+        "badge": out / "badge.svg",
+        "badge_json": out / "badge.json",
+    }
+    paths["json"].write_text(json.dumps(results, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    write_html(results, paths["html"])
+    write_junit(results, paths["junit"])
+    write_badges(results, out)
+    return paths
+
+
+def _open(path: Path) -> None:
+    if os.environ.get("CI") or not sys.stdout.isatty():
+        return
+    with contextlib.suppress(Exception):  # opening a browser is best effort
+        webbrowser.open(path.resolve().as_uri())
+
+
+@app.command()
+def init(
+    directory: Annotated[Path, typer.Argument(help="Project root to scan for skills.")] = Path("."),
+    force: Annotated[bool, typer.Option("--force", help="Overwrite an existing doesitfire.yaml.")] = False,
+) -> None:
+    """Create a doesitfire.yaml for the skills found under DIRECTORY."""
+    directory = directory.resolve()
+    target = directory / CONFIG_NAME
+    if target.exists() and not force:
+        raise DoesItFireError(f"{target} already exists. Use --force to overwrite it.")
+    skills = find_skill_dirs(directory)
+    target.write_text(render_template(skills, directory), encoding="utf-8")
+    console.print(f"Wrote [bold]{target}[/bold] with {len(skills) or 'a placeholder for'} skill(s).")
+    console.print("Next: replace the TODO prompts, then run [bold]doesitfire run[/bold].")
+
+
+@app.command()
+def run(
+    config: Annotated[Path, typer.Option("--config", "-c", help="Path to doesitfire.yaml.")] = Path(CONFIG_NAME),
+    out: Annotated[Path, typer.Option("--out", "-o", help="Output directory.")] = Path("doesitfire-results"),
+    agents: Annotated[str | None, typer.Option(help="Comma-separated subset of agents to run.")] = None,
+    skills: Annotated[str | None, typer.Option(help="Comma-separated subset of skills to run.")] = None,
+    repeats: Annotated[int | None, typer.Option(help="Runs per prompt and agent (overrides the config).")] = None,
+    max_cost: Annotated[float, typer.Option("--max-cost", help="Stop before spending more than this many USD.")] = 1.0,
+    min_recall: Annotated[
+        float | None, typer.Option(help="Exit 1 if any agent x skill recall is below this (0-1).")
+    ] = None,
+    max_false_fire: Annotated[
+        float | None, typer.Option(help="Exit 1 if any false-fire rate is above this (0-1).")
+    ] = None,
+    rebuild: Annotated[bool, typer.Option(help="Rebuild the agent Docker image.")] = False,
+    resume: Annotated[bool, typer.Option(help="Continue an interrupted run from OUT/results.partial.json.")] = False,
+    open_report: Annotated[bool, typer.Option("--open/--no-open", help="Open the HTML report when done.")] = True,
+) -> None:
+    """Run every agent on every prompt and report how often each skill fires."""
+    from .runner import run_matrix
+
+    cfg = load_config(config)
+    if agents:
+        wanted = {a.strip() for a in agents.split(",") if a.strip()}
+        unknown = wanted - {a.name for a in cfg.agents}
+        if unknown:
+            raise DoesItFireError(f"Not configured in {config}: {', '.join(sorted(unknown))}.")
+        cfg.agents = [a for a in cfg.agents if a.name in wanted]
+    if skills:
+        wanted = {s.strip() for s in skills.split(",") if s.strip()}
+        unknown = wanted - {s.name for s in cfg.skills}
+        if unknown:
+            raise DoesItFireError(f"Not in {config}: {', '.join(sorted(unknown))}.")
+        cfg.skills = [s for s in cfg.skills if s.name in wanted]
+    if repeats is not None:
+        if repeats < 1:
+            raise DoesItFireError("--repeats must be at least 1.")
+        cfg.repeats = repeats
+    if max_cost < 0:
+        raise DoesItFireError("--max-cost cannot be negative.")
+
+    n = cfg.repeats * len(cfg.agents) * sum(len(s.should_fire) + len(s.should_not_fire) for s in cfg.skills)
+    console.print(
+        f"doesitfire {__version__}: {len(cfg.skills)} skill(s) x {len(cfg.agents)} agent(s), "
+        f"{cfg.repeats} repeat(s) = {n} sandboxed runs. Budget {max_cost:.2f} USD."
+    )
+
+    def progress(p: dict) -> None:
+        t = p["trial"]
+        mark = (
+            "[red]error[/red]" if t["error"] else ("[magenta]fired[/magenta]" if t["fired"] else "[dim]no fire[/dim]")
+        )
+        good = (t["fired"] == (t["expect"] == "fire")) and not t["error"]
+        tick = "[green]ok[/green]" if good else "[yellow]!![/yellow]"
+        console.print(
+            f"[dim]{p['done']:>4}/{p['total']}[/dim] {tick} {t['agent']:<11} {t['skill']:<20} {mark:<22} "
+            f"[dim]{t['duration_s']:>5.0f}s  {escape(' '.join(t['prompt'].split())[:60])}[/dim]"
+            + (f"\n          [red]{escape(t['error'])}[/red]" if t["error"] else "")
+        )
+
+    out.mkdir(parents=True, exist_ok=True)
+    partial_file = out / "results.partial.json"
+
+    def checkpoint(results: dict) -> None:
+        partial_file.write_text(json.dumps(results, indent=2, ensure_ascii=False), encoding="utf-8")
+
+    previous = None
+    if resume:
+        if not partial_file.is_file():
+            raise DoesItFireError(f"Nothing to resume: {partial_file} does not exist.")
+        previous = _load_results(partial_file)
+    partial = None
+    try:
+        results = run_matrix(
+            cfg,
+            max_cost=max_cost,
+            log=console.print,
+            progress=progress,
+            checkpoint=checkpoint,
+            logs_dir=out / "logs",
+            resume=previous,
+            rebuild=rebuild,
+        )
+    except BudgetExceeded as exc:
+        partial = exc
+        results = exc.results or {}
+    if not results:
+        raise DoesItFireError("No results.")
+    from .report import print_matrix
+
+    paths = write_outputs(results, out)
+    partial_file.unlink(missing_ok=True)
+    summary = print_matrix(results, console)
+    console.print(f"Report: {paths['html']}  JSON: {paths['json']}  JUnit: {paths['junit']}  Badge: {paths['badge']}")
+    if open_report:
+        _open(paths["html"])
+    if partial is not None:
+        err.print(f"[bold red]Budget limit reached, results are partial:[/bold red] {partial}")
+        raise typer.Exit(partial.exit_code)
+    failures = check_thresholds(summary, min_recall, max_false_fire)
+    if failures:
+        err.print("[bold red]Thresholds not met:[/bold red]\n  " + "\n  ".join(failures))
+        raise typer.Exit(1)
+
+
+@app.command()
+def report(
+    results_file: Annotated[Path, typer.Argument(help="A results.json written by 'doesitfire run'.")],
+    out: Annotated[
+        Path | None, typer.Option("--out", "-o", help="Output directory (default: next to the JSON).")
+    ] = None,
+    open_report: Annotated[bool, typer.Option("--open/--no-open")] = False,
+) -> None:
+    """Re-render the matrix, HTML, JUnit and badge from a results.json."""
+    from .report import print_matrix
+
+    results = _load_results(results_file)
+    paths = write_outputs(results, out or results_file.parent)
+    print_matrix(results, console)
+    console.print(f"Report: {paths['html']}")
+    if open_report:
+        _open(paths["html"])
+
+
+def _load_results(path: Path) -> dict:
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        raise DoesItFireError(f"{path} not found.") from None
+    except json.JSONDecodeError as exc:
+        raise DoesItFireError(f"{path} is not valid JSON: {exc}") from None
+    if not isinstance(data, dict) or data.get("tool") != "doesitfire":
+        raise DoesItFireError(f"{path} is not a doesitfire results file.")
+    summarize(data)  # validates the structure early
+    return data
+
+
+@app.command()
+def demo(
+    out: Annotated[
+        Path | None, typer.Option("--out", "-o", help="Where to write the report (default: a temp dir).")
+    ] = None,
+    open_report: Annotated[bool, typer.Option("--open/--no-open", help="Open the HTML report.")] = True,
+) -> None:
+    """Show a RECORDED real run of the three example skills. No keys, no Docker, no model."""
+    from .report import print_matrix
+
+    data = resources.files("doesitfire") / "data" / "demo-results.json"
+    results = json.loads(data.read_text(encoding="utf-8"))
+    target = out or Path(tempfile.mkdtemp(prefix="doesitfire-demo-"))
+    paths = write_outputs(results, target)
+    print_matrix(results, console)
+    console.print(f"Report: {paths['html']}")
+    console.print("Run it yourself: [bold]doesitfire run -c examples/doesitfire.yaml[/bold] (needs Docker + Ollama).")
+    if open_report:
+        _open(paths["html"])
+
+
+def main() -> None:
+    try:
+        app()
+    except DoesItFireError as exc:
+        err.print(f"[bold red]Error:[/bold red] {escape(str(exc))}")
+        sys.exit(exc.exit_code)
+    except KeyboardInterrupt:
+        err.print("[yellow]Interrupted.[/yellow]")
+        sys.exit(130)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()
+
+__all__ = ["app", "main"]