skillproof 0.1.0__py3-none-any.whl

skillproof/runners/openai_compat.py

@@ -0,0 +1,238 @@
+"""OpenAI-compatible endpoint runner: a small tool-using agent loop over Chat Completions.
+
+Works with OpenAI and anything that speaks the same API (vLLM, Ollama,
+OpenRouter, LM Studio, ...).
+
+APPROXIMATION: this is not a real skill-loading agent. With the skill, SKILL.md
+is pasted into the system prompt and the skill folder is copied to
+`.skills/<name>/` in the sandbox. Real agents only see the skill's description
+and decide whether to load it, so trigger rates are not measured here and
+results are labeled approximate in every report.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from skillproof.errors import RunnerNotReady
+from skillproof.runners.base import AgentRun, Runner, RunRequest, ToolCall, Usage
+from skillproof.sandbox import run_shell
+from skillproof.skill import Skill, copy_skill
+
+MAX_TOOL_OUTPUT = 20_000  # characters returned to the model per tool call
+COMMAND_TIMEOUT = 120  # seconds per run_command call
+
+BASE_PROMPT = """\
+You are a capable assistant working in a project directory. Use the tools to
+inspect files, write files and run shell commands when that helps. When you are
+done, reply with your final answer as plain text.
+"""
+
+SKILL_PROMPT = """
+A skill is installed. Its files are in `.skills/{name}/`. Follow its
+instructions when they are relevant to the request.
+
+<skill name="{name}">
+{content}
+</skill>
+"""
+
+TOOLS: list[dict[str, Any]] = [
+    {
+        "type": "function",
+        "function": {
+            "name": "run_command",
+            "description": "Run a shell command in the project directory.",
+            "parameters": {
+                "type": "object",
+                "properties": {"command": {"type": "string"}},
+                "required": ["command"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "read_file",
+            "description": "Read a text file (path relative to the project directory).",
+            "parameters": {
+                "type": "object",
+                "properties": {"path": {"type": "string"}},
+                "required": ["path"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "write_file",
+            "description": "Create or overwrite a text file (path relative to the project).",
+            "parameters": {
+                "type": "object",
+                "properties": {"path": {"type": "string"}, "content": {"type": "string"}},
+                "required": ["path", "content"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "list_files",
+            "description": "List files under a directory (relative to the project).",
+            "parameters": {
+                "type": "object",
+                "properties": {"path": {"type": "string"}},
+            },
+        },
+    },
+]
+
+
+def system_prompt(skill: Skill | None) -> str:
+    if skill is None:
+        return BASE_PROMPT
+    return BASE_PROMPT + SKILL_PROMPT.format(name=skill.name, content=skill.skill_md.strip())
+
+
+def _inside(workdir: Path, rel: str) -> Path:
+    path = (workdir / rel).resolve()
+    if not path.is_relative_to(workdir.resolve()):
+        raise ValueError(f"{rel!r} is outside the project directory")
+    return path
+
+
+def _truncate(text: str) -> str:
+    if len(text) <= MAX_TOOL_OUTPUT:
+        return text
+    return text[:MAX_TOOL_OUTPUT] + f"\n... [truncated {len(text) - MAX_TOOL_OUTPUT} chars]"
+
+
+async def run_tool(name: str, args: dict[str, Any], workdir: Path) -> str:
+    """Execute one tool call inside the sandbox and return its text result."""
+    try:
+        if name == "run_command":
+            code, out = await run_shell(str(args["command"]), workdir, COMMAND_TIMEOUT)
+            if code is None:
+                return f"command timed out after {COMMAND_TIMEOUT}s"
+            return _truncate(f"exit code {code}\n{out}")
+        if name == "read_file":
+            return _truncate(_inside(workdir, args["path"]).read_text(errors="replace"))
+        if name == "write_file":
+            target = _inside(workdir, args["path"])
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_text(str(args["content"]))
+            return f"wrote {args['path']}"
+        if name == "list_files":
+            root = _inside(workdir, args.get("path") or ".")
+            files = sorted(str(p.relative_to(workdir)) for p in root.rglob("*") if p.is_file())
+            return _truncate("\n".join(files) or "(no files)")
+        return f"unknown tool {name!r}"
+    except Exception as exc:  # tool errors go back to the model, like a real agent
+        return f"error: {exc}"
+
+
+class OpenAICompatibleRunner(Runner):
+    name = "openai-compatible"
+    vendor = "openai"
+    approximate = True
+    requires_model = True
+
+    class Options(BaseModel):
+        model_config = ConfigDict(extra="forbid")
+
+        base_url: str | None = None  # None = api.openai.com
+        api_key_env: str = "OPENAI_API_KEY"
+        tools: bool = True  # set false for endpoints without function calling
+        max_turns: int = Field(default=20, ge=1)
+
+    options: Options
+
+    def check_ready(self) -> None:
+        if self.options.base_url is None and not os.environ.get(self.options.api_key_env):
+            raise RunnerNotReady(
+                f"{self.options.api_key_env} is not set.",
+                hint=f"export {self.options.api_key_env}=..., or set runner_options.base_url "
+                "to a local endpoint (e.g. http://localhost:11434/v1 for Ollama).",
+            )
+
+    def client(self) -> Any:
+        from openai import AsyncOpenAI
+
+        # Local endpoints often need no key, but the client insists on one.
+        api_key = os.environ.get(self.options.api_key_env) or "not-needed"
+        return AsyncOpenAI(base_url=self.options.base_url, api_key=api_key)
+
+    async def run(self, request: RunRequest) -> AgentRun:
+        if request.skill is not None:
+            copy_skill(
+                request.skill,
+                request.workdir / ".skills" / request.skill.name,
+                exclude=request.skill_exclude,
+            )
+        client = self.client()
+        messages: list[dict[str, Any]] = [
+            {"role": "system", "content": system_prompt(request.skill)},
+            {"role": "user", "content": request.prompt},
+        ]
+        usage = Usage()
+        calls: list[ToolCall] = []
+        warnings: list[str] = []
+        final = ""
+        turns = 0
+        try:
+            for turns in range(1, self.options.max_turns + 1):  # noqa: B007 - used after loop
+                kwargs: dict[str, Any] = {"model": request.model, "messages": messages}
+                if self.options.tools:
+                    kwargs["tools"] = TOOLS
+                response = await client.chat.completions.create(**kwargs)
+                add_usage(usage, response.usage)
+                message = response.choices[0].message
+                messages.append(message.model_dump(exclude_none=True))
+                if not message.tool_calls:
+                    final = message.content or ""
+                    break
+                for tool_call in message.tool_calls:
+                    args = parse_arguments(tool_call.function.arguments)
+                    output = await run_tool(tool_call.function.name, args, request.workdir)
+                    calls.append(ToolCall(name=tool_call.function.name, input=args, output=output))
+                    messages.append(
+                        {"role": "tool", "tool_call_id": tool_call.id, "content": output}
+                    )
+            else:
+                warnings.append(f"stopped early: max_turns ({self.options.max_turns})")
+        finally:
+            await client.close()
+
+        return AgentRun(
+            final_text=final,
+            transcript=messages,
+            tool_calls=calls,
+            usage=usage,
+            num_turns=turns,
+            model=request.model,
+            triggered=None,  # can't be measured: the skill is always in the prompt
+            warnings=warnings,
+        )
+
+
+def parse_arguments(raw: str | None) -> dict[str, Any]:
+    try:
+        args = json.loads(raw or "{}")
+    except json.JSONDecodeError:
+        return {"_invalid_json": raw}
+    return args if isinstance(args, dict) else {"_value": args}
+
+
+def add_usage(total: Usage, usage: Any) -> None:
+    if usage is None:
+        return
+    details = getattr(usage, "prompt_tokens_details", None)
+    cached = (getattr(details, "cached_tokens", None) or 0) if details else 0
+    total.input_tokens += (usage.prompt_tokens or 0) - cached
+    total.cache_read_tokens += cached
+    total.output_tokens += usage.completion_tokens or 0

skillproof/sandbox.py

@@ -0,0 +1,85 @@
+"""Fresh temp directories for each run, so runs never touch the user's project.
+
+Sandboxes live in the system temp dir (not inside the user's repo) so agents
+that walk up the directory tree looking for config (CLAUDE.md, AGENTS.md, .git)
+don't pick up the user's real project.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import re
+import shutil
+import signal
+import subprocess
+import tempfile
+from pathlib import Path
+
+# Env vars whose values must never reach logs, saved transcripts, or `command`
+# assertions. Matched against the variable *name*.
+SECRET_NAME = re.compile(r"(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTH)", re.IGNORECASE)
+MIN_SECRET_LENGTH = 8  # shorter values are too likely to collide with normal text
+
+
+def create_sandbox(fixtures: Path | None = None, prefix: str = "skillproof-") -> Path:
+    workdir = Path(tempfile.mkdtemp(prefix=prefix)).resolve()
+    if fixtures is not None:
+        shutil.copytree(fixtures, workdir, dirs_exist_ok=True)
+    return workdir
+
+
+def remove_sandbox(workdir: Path) -> None:
+    shutil.rmtree(workdir, ignore_errors=True)
+
+
+def scrubbed_env() -> dict[str, str]:
+    """A copy of os.environ without anything that looks like a credential."""
+    return {k: v for k, v in os.environ.items() if not SECRET_NAME.search(k)}
+
+
+def secret_values() -> list[str]:
+    values = [
+        v for k, v in os.environ.items() if SECRET_NAME.search(k) and len(v) >= MIN_SECRET_LENGTH
+    ]
+    return sorted(set(values), key=len, reverse=True)
+
+
+def redact(text: str) -> str:
+    """Replace any credential values from the environment with ***."""
+    for value in secret_values():
+        text = text.replace(value, "***")
+    return text
+
+
+def _run_shell(command: str, cwd: Path, timeout: float) -> tuple[int | None, str]:
+    proc = subprocess.Popen(
+        command,
+        shell=True,
+        cwd=cwd,
+        env=scrubbed_env(),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        start_new_session=True,  # so a timeout can kill the whole process group
+    )
+    try:
+        out, _ = proc.communicate(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        if hasattr(os, "killpg"):
+            os.killpg(proc.pid, signal.SIGKILL)
+        else:
+            proc.kill()
+        proc.communicate()
+        return None, ""
+    return proc.returncode, out.decode(errors="replace")
+
+
+async def run_shell(command: str, cwd: Path, timeout: float) -> tuple[int | None, str]:
+    """Run a shell command in the sandbox without credentials in its environment.
+
+    Returns (exit code, combined output); exit code None means it timed out.
+    Runs in a worker thread with plain `subprocess` rather than asyncio
+    subprocesses: on Python 3.10/3.11, asyncio's child watcher can collide with
+    the Agent SDK's own subprocesses and misreport exit codes as 255.
+    """
+    return await asyncio.to_thread(_run_shell, command, cwd, timeout)

skillproof/scaffold.py

@@ -0,0 +1,153 @@
+"""`skillproof init`: write a starter evals.yaml and fixtures folder."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from skillproof.errors import SkillproofError
+from skillproof.skill import SKILL_FILE, SkillError, load_skill
+
+EVALS_TEMPLATE = """\
+# skillproof evals for the "{name}" skill.
+#
+# Every case runs `trials` times WITH the skill installed and `trials` times
+# WITHOUT it (the baseline). skillproof then reports whether the skill was
+# triggered, pass rate with vs. without (the "lift"), and token/latency deltas.
+#
+# Run it:   skillproof run {display_path}
+
+skill: ./                # folder containing SKILL.md, relative to this file
+runner: {runner}{runner_pad}# claude-code | codex | openai-compatible
+{model_line}
+trials: 5                # runs per case, per variant. More trials = tighter numbers.
+timeout: 300             # seconds per run
+
+# Only needed for llm_judge assertions. `auto` picks a judge from a different
+# vendor than the agent under test, so a model never grades itself.
+# judge:
+#   provider: auto       # auto | anthropic | openai
+#   model: ...
+
+cases:
+  # A task the skill SHOULD help with. Write it the way a real user would:
+  # don't name the skill, that's what the trigger rate measures.
+  - name: "example task"
+    prompt: |
+      TODO: replace with a realistic request that needs this skill:
+      {description}
+    fixtures: ./fixtures/example/   # copied into a fresh temp dir for every run
+    assert:
+      # Deterministic checks first: cheap, fast and reproducible.
+      - type: contains            # final answer contains this text (case-insensitive)
+        value: "TODO"
+      # - type: not_contains
+      #   value: "I can't"
+      # - type: regex
+      #   pattern: "line \\\\d+"
+      # - type: file_exists       # path is relative to the sandbox
+      #   path: "output.md"
+      # - type: file_contains
+      #   path: "output.md"
+      #   value: "## Summary"
+      # - type: command           # runs in the sandbox, passes on exit code 0
+      #   run: "python -m pytest -q"
+      # - type: max_tokens        # input + output + cache tokens for the run
+      #   value: 50000
+      # - type: llm_judge         # optional, graded by a different-vendor model
+      #   rubric: "States the root cause in the first sentence."
+
+  # A request the skill should NOT activate for. Catches over-eager
+  # descriptions that load the skill (and burn tokens) when it isn't needed.
+  - name: "unrelated request"
+    prompt: "What is the capital of France? Answer in one word."
+    should_trigger: false
+    assert:
+      - type: contains
+        value: "Paris"
+"""
+
+MODEL_LINES = {
+    "openai-compatible": "model: gpt-6.1-sol       # required: any model your endpoint serves\n"
+    "runner_options:\n"
+    "  # base_url: http://localhost:11434/v1   # e.g. Ollama; default is api.openai.com",
+    "default": "# model: ...             # optional; defaults to the agent's own default model",
+}
+
+FIXTURE_README = """\
+Files in this folder are copied into a fresh temporary directory before every
+run, so the agent can read and edit them without touching your real project.
+
+Replace this file with whatever the "example task" case needs: a repo snippet,
+a log file, a CSV, a failing test...
+"""
+
+SKILL_TEMPLATE = """\
+---
+name: {name}
+description: TODO - what this skill does and when the agent should use it.
+---
+
+# {name}
+
+TODO: instructions for the agent.
+"""
+
+
+@dataclass
+class InitResult:
+    created: list[Path] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+
+def init_skill(path: Path, runner: str = "claude-code", force: bool = False) -> InitResult:
+    result = InitResult()
+    path.mkdir(parents=True, exist_ok=True)
+    evals_path = path / "evals.yaml"
+    if evals_path.exists() and not force:
+        raise SkillproofError(
+            f"{evals_path} already exists.",
+            hint="Use --force to overwrite it, or edit it by hand.",
+        )
+
+    name = path.resolve().name
+    description = "TODO: describe the task."
+    if (path / SKILL_FILE).is_file():
+        try:
+            skill = load_skill(path)
+            name, description = skill.name, skill.description
+        except SkillError as exc:
+            result.warnings.append(f"{exc.message} {exc.hint or ''}".strip())
+    else:
+        (path / SKILL_FILE).write_text(SKILL_TEMPLATE.format(name=name), encoding="utf-8")
+        result.created.append(path / SKILL_FILE)
+        result.warnings.append(f"No {SKILL_FILE} found, so a placeholder was created. Fill it in.")
+
+    evals_path.write_text(
+        EVALS_TEMPLATE.format(
+            name=name,
+            display_path=_display(path),
+            runner=runner,
+            runner_pad=" " * max(1, 17 - len(runner)),
+            model_line=MODEL_LINES.get(runner, MODEL_LINES["default"]),
+            description=" ".join(description.split()),
+        ),
+        encoding="utf-8",
+    )
+    result.created.append(evals_path)
+
+    fixtures = path / "fixtures" / "example"
+    fixtures.mkdir(parents=True, exist_ok=True)
+    readme = fixtures / "README.txt"
+    if not readme.exists():
+        readme.write_text(FIXTURE_README, encoding="utf-8")
+        result.created.append(readme)
+    return result
+
+
+def _display(path: Path) -> str:
+    try:
+        rel = path.resolve().relative_to(Path.cwd())
+        return f"./{rel}" if str(rel) != "." else "."
+    except ValueError:
+        return str(path)

skillproof/skill.py

@@ -0,0 +1,97 @@
+"""Loading a skill folder (a directory with a SKILL.md) and copying it into sandboxes."""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import yaml
+from pydantic import BaseModel
+
+from skillproof.errors import SkillError
+
+SKILL_FILE = "SKILL.md"
+
+# Never copied into the sandbox along with the skill. The agent must not be able
+# to read the eval cases (and their assertions) while it is being evaluated.
+ALWAYS_EXCLUDED = {".git", ".skillproof", "evals.yaml", "evals.yml", "__pycache__", ".venv"}
+
+
+class Skill(BaseModel):
+    path: Path
+    name: str
+    description: str
+    body: str  # SKILL.md content without the frontmatter
+
+    @property
+    def skill_md(self) -> str:
+        return (self.path / SKILL_FILE).read_text(encoding="utf-8")
+
+
+def parse_frontmatter(text: str) -> tuple[dict, str]:
+    """Split a markdown file into (frontmatter dict, body). Missing frontmatter -> {}."""
+    if not text.startswith("---"):
+        return {}, text
+    lines = text.splitlines(keepends=True)
+    for i, line in enumerate(lines[1:], start=1):
+        if line.strip() == "---":
+            raw = "".join(lines[1:i])
+            body = "".join(lines[i + 1 :])
+            try:
+                data = yaml.safe_load(raw) or {}
+            except yaml.YAMLError as exc:
+                raise SkillError(
+                    f"SKILL.md frontmatter is not valid YAML: {exc}",
+                    hint="Fix the YAML between the two '---' lines at the top of SKILL.md.",
+                ) from exc
+            if not isinstance(data, dict):
+                return {}, body
+            return data, body
+    return {}, text
+
+
+def load_skill(path: Path) -> Skill:
+    path = path.resolve()
+    skill_md = path / SKILL_FILE
+    if not path.is_dir():
+        raise SkillError(
+            f"Skill folder not found: {path}",
+            hint="Set `skill:` in evals.yaml to the folder that contains SKILL.md.",
+        )
+    if not skill_md.is_file():
+        raise SkillError(
+            f"No {SKILL_FILE} in {path}",
+            hint="A skill is a folder with a SKILL.md file. Point `skill:` at that folder.",
+        )
+    meta, body = parse_frontmatter(skill_md.read_text(encoding="utf-8"))
+    name = str(meta.get("name") or "").strip()
+    description = str(meta.get("description") or "").strip()
+    if not name or not description:
+        raise SkillError(
+            f"{skill_md} must have `name` and `description` in its frontmatter.",
+            hint=(
+                "Start SKILL.md with:\n---\nname: my-skill\n"
+                "description: What it does and when to use it.\n---"
+            ),
+        )
+    return Skill(path=path, name=name, description=description, body=body)
+
+
+def copy_skill(skill: Skill, dest: Path, exclude: list[Path] | None = None) -> Path:
+    """Copy the skill folder to `dest`, leaving out eval files and fixtures.
+
+    `exclude` holds absolute paths (e.g. fixture folders that live inside the
+    skill folder) that must not be visible to the agent.
+    """
+    excluded = {p.resolve() for p in (exclude or [])}
+
+    def ignore(directory: str, names: list[str]) -> set[str]:
+        skipped = set()
+        for name in names:
+            full = (Path(directory) / name).resolve()
+            if name in ALWAYS_EXCLUDED or full in excluded:
+                skipped.add(name)
+        return skipped
+
+    shutil.copytree(skill.path, dest, ignore=ignore, dirs_exist_ok=True)
+    return dest