turnloop 0.1.0__py3-none-any.whl

turnloop/tools/runner.py

@@ -0,0 +1,284 @@
+"""Tool dispatch — the only place `Tool.run()` is called.
+
+Everything that must happen for every tool call happens here exactly once:
+schema validation, hooks, permission checks, timeout, metrics, and error
+containment. A test asserts that no other module calls `.run(` on a tool, because
+the moment there are two dispatch paths the permission system has a hole in it.
+
+The invariant that matters most: **dispatch never raises.** Every failure becomes
+an error `tool_result` the model can read and react to. A tool that crashes must
+cost one turn, not the whole session — and the error text it produces is also the
+raw signal the tool-calling-reliability experiment measures.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+
+import anyio
+from pydantic import ValidationError
+
+from turnloop.core.events import ToolFinished, ToolStarted
+from turnloop.core.messages import ToolResultBlock, ToolUseBlock
+from turnloop.permissions.engine import (
+    PermissionDecision,
+    PermissionEngine,
+    PermissionRequest,
+    Scope,
+    Verdict,
+)
+from turnloop.tools.base import Tool, ToolContext, ToolOutput, ToolRegistry
+
+# Classified error kinds, recorded per call. These names are the schema the
+# reliability report groups by, so they are stable.
+ERROR_UNKNOWN_TOOL = "unknown_tool"
+ERROR_BAD_JSON = "malformed_args"
+ERROR_SCHEMA = "schema_violation"
+ERROR_DENIED = "permission_denied"
+ERROR_TIMEOUT = "timeout"
+ERROR_CRASH = "tool_crash"
+ERROR_TOOL = "tool_error"
+
+
+@dataclass(slots=True)
+class ToolCallRecord:
+    tool: str
+    ok: bool
+    error_kind: str | None
+    duration_s: float
+    metrics: dict = field(default_factory=dict)
+
+
+class ToolRunner:
+    def __init__(self, registry: ToolRegistry, permissions: PermissionEngine,
+                 hooks=None, ui=None):
+        self.registry = registry
+        self.permissions = permissions
+        self.hooks = hooks
+        self.ui = ui
+        self.records: list[ToolCallRecord] = []
+
+    async def dispatch(self, block: ToolUseBlock, ctx: ToolContext) -> ToolResultBlock:
+        started = time.monotonic()
+        ctx = ctx.child(tool_use_id=block.id)
+
+        tool = self.registry.get(block.name)
+        if tool is None:
+            return await self._fail(
+                block, ctx, ERROR_UNKNOWN_TOOL,
+                f"No tool named {block.name!r}. Available tools: "
+                f"{', '.join(self.registry.names())}.",
+                started,
+            )
+
+        # A parse failure upstream leaves this marker rather than raising, so the
+        # model gets told its JSON was broken instead of the turn dying.
+        if "_parse_error" in block.args:
+            return await self._fail(
+                block, ctx, ERROR_BAD_JSON,
+                f"Your tool arguments were not valid JSON ({block.args['_parse_error']}). "
+                f"Call {block.name} again with well-formed JSON.",
+                started, tool=tool,
+            )
+
+        try:
+            args = tool.Args.model_validate(block.args)
+        except ValidationError as exc:
+            return await self._fail(
+                block, ctx, ERROR_SCHEMA,
+                f"Invalid arguments for {block.name}:\n{_format_validation(exc)}",
+                started, tool=tool,
+            )
+
+        await self._emit(ToolStarted(block.id, tool.name, tool.summary(args), ctx.subagent_id))
+
+        # --- hooks may block before anything happens ------------------------
+        if self.hooks is not None:
+            outcome = await self.hooks.run_pre_tool(tool.name, args, ctx)
+            if outcome is not None and outcome.blocked:
+                return await self._fail(
+                    block, ctx, ERROR_DENIED,
+                    f"Blocked by a PreToolUse hook: {outcome.reason}",
+                    started, tool=tool,
+                )
+
+        # --- permissions ----------------------------------------------------
+        decision = self.permissions.check(tool, args)
+        if decision.verdict is Verdict.DENY:
+            self._log_permission(ctx, tool.name, decision.verdict.value, decision.reason)
+            return await self._fail(
+                block, ctx, ERROR_DENIED,
+                f"Permission denied: {decision.reason}", started, tool=tool,
+            )
+
+        if decision.verdict is Verdict.ASK:
+            answer = await self._ask(tool, args, ctx, decision.reason)
+            self._log_permission(
+                ctx, tool.name, "approved" if answer.approved else "rejected", answer.reason
+            )
+            if not answer.approved:
+                return await self._fail(
+                    block, ctx, ERROR_DENIED,
+                    "The user declined this call."
+                    + (f" They said: {answer.reason}" if answer.reason else "")
+                    + " Do not retry it; ask what they would prefer.",
+                    started, tool=tool,
+                )
+            if answer.scope is not Scope.ONCE and answer.rule:
+                self.permissions.grant(answer.rule, answer.scope)
+        else:
+            self._log_permission(ctx, tool.name, "allowed", decision.reason)
+
+        # --- run ------------------------------------------------------------
+        timeout = tool.timeout_s
+        try:
+            if timeout:
+                with anyio.fail_after(timeout):
+                    output = await tool.run(args, ctx)
+            else:
+                output = await tool.run(args, ctx)
+        except TimeoutError:
+            return await self._fail(
+                block, ctx, ERROR_TIMEOUT,
+                f"{tool.name} exceeded its {timeout:.0f}s timeout and was cancelled.",
+                started, tool=tool,
+            )
+        except anyio.get_cancelled_exc_class():
+            # A user interrupt must propagate — it is not a tool failure.
+            raise
+        except Exception as exc:  # noqa: BLE001 - containment is the point
+            return await self._fail(
+                block, ctx, ERROR_CRASH,
+                f"{tool.name} raised {type(exc).__name__}: {exc}",
+                started, tool=tool,
+            )
+
+        duration = time.monotonic() - started
+        if self.hooks is not None:
+            await self.hooks.run_post_tool(tool.name, args, output, ctx)
+
+        self._record(tool.name, not output.is_error,
+                     ERROR_TOOL if output.is_error else None, duration, output.metrics)
+        self._log_metrics(ctx, tool.name, output, duration)
+        await self._emit(
+            ToolFinished(block.id, tool.name, output.is_error, output.display, duration,
+                         ctx.subagent_id)
+        )
+
+        return ToolResultBlock(
+            tool_use_id=block.id,
+            content=output.content,
+            is_error=output.is_error,
+            display=output.display,
+        )
+
+    # --- helpers -----------------------------------------------------------
+
+    async def _ask(self, tool: Tool, args, ctx: ToolContext, reason: str) -> PermissionDecision:
+        request = PermissionRequest(
+            tool_name=tool.name,
+            target=tool.permission_target(args),
+            summary=tool.summary(args),
+            args_preview=_preview(args),
+            is_read_only=tool.is_read_only_for(args),
+            suggested_rule=self.permissions.suggested_rule(tool, args),
+            reason=reason,
+            subagent_id=ctx.subagent_id,
+        )
+        return await ctx.ask(request)
+
+    async def _fail(self, block: ToolUseBlock, ctx: ToolContext, kind: str, message: str,
+                    started: float, tool: Tool | None = None) -> ToolResultBlock:
+        duration = time.monotonic() - started
+        name = tool.name if tool else block.name
+        self._record(name, False, kind, duration, {})
+        if ctx.session is not None and getattr(ctx.session, "store", None) is not None:
+            ctx.session.store.write_tool_metrics(  # type: ignore[attr-defined]
+                name, {"ok": False, "error_kind": kind, "duration_s": round(duration, 4)}
+            )
+        await self._emit(ToolFinished(block.id, name, True, None, duration, ctx.subagent_id))
+        return ToolResultBlock(
+            tool_use_id=block.id, content=message, is_error=True, display=message
+        )
+
+    def _record(self, name: str, ok: bool, kind: str | None, duration: float,
+                metrics: dict) -> None:
+        self.records.append(
+            ToolCallRecord(tool=name, ok=ok, error_kind=kind, duration_s=duration,
+                           metrics=dict(metrics))
+        )
+
+    def _log_metrics(self, ctx: ToolContext, name: str, output: ToolOutput,
+                     duration: float) -> None:
+        store = getattr(ctx.session, "store", None)
+        if store is None:
+            return
+        store.write_tool_metrics(
+            name,
+            {
+                "ok": not output.is_error,
+                "error_kind": ERROR_TOOL if output.is_error else None,
+                "duration_s": round(duration, 4),
+                **output.metrics,
+            },
+        )
+
+    def _log_permission(self, ctx: ToolContext, tool: str, verdict: str, reason: str) -> None:
+        store = getattr(ctx.session, "store", None)
+        if store is None:
+            return
+        store.write_record("permission", {"tool": tool, "verdict": verdict, "reason": reason})
+
+    async def _emit(self, event) -> None:
+        if self.ui is not None:
+            await self.ui.send(event)
+
+    # --- reporting ---------------------------------------------------------
+
+    def error_counts(self) -> dict[str, int]:
+        counts: dict[str, int] = {}
+        for record in self.records:
+            if record.error_kind:
+                counts[record.error_kind] = counts.get(record.error_kind, 0) + 1
+        return counts
+
+    def recovery_rate(self) -> float | None:
+        """Fraction of failed calls followed by a success on the same tool.
+
+        The single most informative reliability number: models differ far more in
+        whether they recover from a bad call than in how often they make one.
+        """
+        failures = [i for i, r in enumerate(self.records) if r.error_kind]
+        if not failures:
+            return None
+        recovered = 0
+        for i in failures:
+            tool = self.records[i].tool
+            if any(r.tool == tool and r.ok for r in self.records[i + 1 : i + 6]):
+                recovered += 1
+        return recovered / len(failures)
+
+
+def _format_validation(exc: ValidationError, limit: int = 6) -> str:
+    """Render pydantic errors so a model can act on them.
+
+    Verbatim pydantic text is genuinely good at this — it names the field, the
+    expected type and what arrived — so it is passed through rather than
+    paraphrased, and it is also exactly what the reliability experiment counts.
+    """
+    lines = []
+    for err in exc.errors()[:limit]:
+        loc = ".".join(str(p) for p in err["loc"]) or "(root)"
+        lines.append(f"- {loc}: {err['msg']}")
+    if len(exc.errors()) > limit:
+        lines.append(f"- ... and {len(exc.errors()) - limit} more")
+    return "\n".join(lines)
+
+
+def _preview(args, limit: int = 600) -> str:
+    try:
+        text = args.model_dump_json(indent=2)
+    except Exception:  # noqa: BLE001
+        text = str(args)
+    return text if len(text) <= limit else text[:limit] + "\n…"

turnloop/tools/shell.py

@@ -0,0 +1,222 @@
+"""Shell resolution and process execution.
+
+The reason this file exists rather than a one-line `subprocess.run(["bash", ...])`:
+
+On a default Windows install, `bash` on PATH is `C:\\Windows\\System32\\bash.exe`
+— the WSL launcher. With no distribution installed it exits 255 and prints its
+error in UTF-16LE, so a naive shell tool fails on its first call with a garbled
+message and no obvious cause. Real Git Bash lives under the Git installation.
+Any `bash.exe` beneath %SystemRoot% is therefore rejected outright.
+
+Second trap, also Windows-specific: `bash -lc "<command>"` makes the actual
+command a *grandchild* of the process we spawn. Calling `terminate()` on the
+parent kills the shell and orphans the real work, which then keeps running and
+holding file locks after a timeout. Killing the whole tree via `taskkill /T` is
+the only reliable answer.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from turnloop.errors import ShellNotFound
+
+ShellKind = Literal["gitbash", "bash", "pwsh", "powershell", "sh"]
+
+IS_WINDOWS = sys.platform == "win32"
+
+
+@dataclass(frozen=True, slots=True)
+class ShellSpec:
+    kind: ShellKind
+    exe: Path
+    posix: bool
+
+    def argv(self, command: str) -> list[str]:
+        if self.posix:
+            # -l loads the login profile so PATH matches an interactive shell.
+            return [str(self.exe), "-lc", command]
+        return [
+            str(self.exe),
+            "-NoProfile",
+            "-NonInteractive",
+            "-ExecutionPolicy",
+            "Bypass",
+            "-Command",
+            command,
+        ]
+
+    @property
+    def label(self) -> str:
+        return f"{self.kind} ({self.exe})"
+
+
+def _is_wsl_stub(path: Path) -> bool:
+    """True for the WSL launcher masquerading as bash."""
+    system_root = Path(os.environ.get("SystemRoot", r"C:\Windows")).resolve()
+    try:
+        resolved = path.resolve()
+    except OSError:
+        return False
+    return resolved == system_root or system_root in resolved.parents
+
+
+def _git_bash_candidates() -> list[Path]:
+    candidates: list[Path] = []
+    git = shutil.which("git")
+    if git:
+        # <install>/cmd/git.exe -> <install>/bin/bash.exe
+        install = Path(git).resolve().parent.parent
+        candidates += [install / "bin" / "bash.exe", install / "usr" / "bin" / "bash.exe"]
+    for drive in ("C:", "D:", "E:", "F:"):
+        candidates += [
+            Path(f"{drive}/Program Files/Git/bin/bash.exe"),
+            Path(f"{drive}/Program Files (x86)/Git/bin/bash.exe"),
+            Path(f"{drive}/Git/bin/bash.exe"),
+        ]
+    return candidates
+
+
+def resolve_shell(explicit: str | None = None) -> ShellSpec:
+    """Find a usable shell, preferring POSIX so model-written commands work.
+
+    Models overwhelmingly emit POSIX shell. Handing them PowerShell means `ls
+    -la`, `cat`, `&&` chains and `2>/dev/null` all fail, so Git Bash is preferred
+    on Windows even though PowerShell is the native shell.
+    """
+    if explicit:
+        exe = Path(explicit)
+        if not exe.exists():
+            found = shutil.which(explicit)
+            if not found:
+                raise ShellNotFound(f"configured shell not found: {explicit}")
+            exe = Path(found)
+        if IS_WINDOWS and exe.name.lower() == "bash.exe" and _is_wsl_stub(exe):
+            raise ShellNotFound(
+                f"{exe} is the WSL launcher, not a real bash. Point bash.shell at "
+                r"Git Bash, e.g. C:\Program Files\Git\bin\bash.exe"
+            )
+        posix = exe.name.lower() in ("bash.exe", "bash", "sh", "sh.exe", "zsh", "dash")
+        kind: ShellKind = "bash" if posix else ("pwsh" if "pwsh" in exe.name.lower() else "powershell")
+        return ShellSpec(kind=kind, exe=exe, posix=posix)
+
+    if not IS_WINDOWS:
+        for name in ("bash", "sh"):
+            if found := shutil.which(name):
+                return ShellSpec(kind="bash" if name == "bash" else "sh",
+                                 exe=Path(found), posix=True)
+        raise ShellNotFound("no POSIX shell found on PATH")
+
+    for candidate in _git_bash_candidates():
+        if candidate.is_file() and not _is_wsl_stub(candidate):
+            return ShellSpec(kind="gitbash", exe=candidate, posix=True)
+
+    # A bash on PATH is acceptable only if it is not the System32 stub.
+    if found := shutil.which("bash"):
+        path = Path(found)
+        if not _is_wsl_stub(path):
+            return ShellSpec(kind="bash", exe=path, posix=True)
+
+    for name in ("pwsh", "powershell"):
+        if found := shutil.which(name):
+            ps_kind: ShellKind = "pwsh" if name == "pwsh" else "powershell"
+            return ShellSpec(kind=ps_kind, exe=Path(found), posix=False)
+
+    raise ShellNotFound(
+        "no usable shell found. Install Git for Windows (provides Git Bash) or "
+        "ensure powershell.exe is on PATH."
+    )
+
+
+def describe_shell_environment() -> list[tuple[str, str]]:
+    """Diagnostics for `turnloop doctor`."""
+    rows: list[tuple[str, str]] = []
+    try:
+        spec = resolve_shell()
+        rows.append(("shell", spec.label))
+        rows.append(("shell dialect", "POSIX" if spec.posix else "PowerShell"))
+    except ShellNotFound as exc:
+        rows.append(("shell", f"NOT FOUND — {exc}"))
+
+    if IS_WINDOWS:
+        on_path = shutil.which("bash")
+        if on_path and _is_wsl_stub(Path(on_path)):
+            rows.append(
+                ("bash on PATH", f"{on_path} — WSL launcher, ignored (it is not a real bash)")
+            )
+        elif on_path:
+            rows.append(("bash on PATH", on_path))
+        else:
+            rows.append(("bash on PATH", "not present"))
+    return rows
+
+
+# --------------------------------------------------------------------------
+# process control
+# --------------------------------------------------------------------------
+
+
+def spawn_kwargs() -> dict:
+    """Platform flags needed to kill a whole process tree later."""
+    if IS_WINDOWS:
+        return {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP}
+    return {"start_new_session": True}
+
+
+def kill_tree(pid: int) -> None:
+    """Kill a process and everything it spawned.
+
+    Required because `bash -lc` puts the real command one level down; terminating
+    only the shell leaves the command running and holding locks.
+    """
+    if IS_WINDOWS:
+        subprocess.run(
+            ["taskkill", "/T", "/F", "/PID", str(pid)],
+            capture_output=True,
+            check=False,
+        )
+        return
+    import signal
+
+    try:
+        # POSIX-only attributes; this branch never runs on Windows.
+        os.killpg(os.getpgid(pid), signal.SIGKILL)  # type: ignore[attr-defined]
+    except (ProcessLookupError, PermissionError):
+        pass
+
+
+def child_env(extra: dict[str, str] | None = None) -> dict[str, str]:
+    """Environment for spawned commands.
+
+    Forcing UTF-8 on the child avoids decoding surprises from Python subprocesses
+    and from PowerShell, which otherwise emits UTF-16LE in some configurations.
+    """
+    env = dict(os.environ)
+    env.update(
+        {
+            "PYTHONIOENCODING": "utf-8",
+            "PYTHONUNBUFFERED": "1",
+            "TERM": env.get("TERM", "dumb"),
+            "NO_COLOR": "1",  # ANSI escapes in tool output are wasted context
+            "GIT_PAGER": "cat",
+            "PAGER": "cat",
+            "TURNLOOP": "1",  # let scripts and hooks detect the harness
+        }
+    )
+    env.update(extra or {})
+    return env
+
+
+def decode_output(raw: bytes) -> str:
+    """Decode child output, tolerating PowerShell's UTF-16."""
+    if raw.startswith(b"\xff\xfe"):
+        return raw.decode("utf-16-le", errors="replace").lstrip("\ufeff")
+    if raw.startswith(b"\xef\xbb\xbf"):
+        return raw.decode("utf-8-sig", errors="replace")
+    return raw.decode("utf-8", errors="replace")

turnloop/tools/skill.py

@@ -0,0 +1,89 @@
+"""Skill — load a set of instructions on demand.
+
+Progressive disclosure, and on a 65k window it is the difference between having
+skills and not. The system prompt carries one line per skill; the body enters
+context only when the model decides the skill applies.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from turnloop.commands.loader import Skill as SkillDef
+from turnloop.tools.base import Tool, ToolContext, ToolOutput
+
+
+class SkillArgs(BaseModel):
+    skill: str = Field(description="Exact skill name from the available-skills list.")
+    args: str = Field(default="", description="Optional arguments to pass through.")
+
+
+class SkillTool(Tool):
+    name = "Skill"
+    Args = SkillArgs
+    read_only = True
+    parallel_safe = False
+
+    def __init__(self, skills: dict[str, SkillDef] | None = None):
+        self.skills = skills or {}
+
+    descriptions = {
+        "terse": """
+Load a skill's full instructions by exact name. Call it when a listed skill covers
+the task, then follow what it returns.
+""",
+        "normal": """
+Load the full instructions for a skill.
+
+- `skill` must be an exact name from the available-skills list in your system
+  prompt. Do not guess names.
+- The returned instructions apply to the current task; follow them in place of
+  your default approach.
+- Call this before starting work the skill covers, not after.
+""",
+        "verbose": """
+Load the full instructions for a skill.
+
+Arguments:
+- `skill`: an exact name from the available-skills list in your system prompt.
+  Guessing a name simply fails.
+- `args`: optional arguments, passed through to the skill.
+
+Why this is a tool rather than part of your prompt: skill bodies are long. Keeping
+them all loaded would consume a large fraction of the context window on
+instructions that are mostly irrelevant to any given task. You are shown each
+skill's name and description, and you fetch the body when it is the one you need.
+
+Once loaded, treat the instructions as authoritative for this task — they encode
+how this project wants the work done, which takes precedence over your defaults.
+""",
+    }
+
+    async def run(self, args: SkillArgs, ctx: ToolContext) -> ToolOutput:
+        skill = self.skills.get(args.skill)
+        if skill is None:
+            available = ", ".join(sorted(self.skills)) or "none configured"
+            return ToolOutput.error(
+                f"No skill named {args.skill!r}. Available: {available}."
+            )
+        body = skill.body.strip()
+        if args.args:
+            body += f"\n\nArguments supplied: {args.args}"
+        return ToolOutput(
+            content=f"--- skill: {skill.name} ({skill.path}) ---\n{body}",
+            display=f"Loaded skill {skill.name}",
+            metrics={"skill": skill.name, "body_chars": len(body)},
+        )
+
+    def summary(self, args: SkillArgs) -> str:  # type: ignore[override]
+        return f"Skill {args.skill}"
+
+    def advertise(self) -> str:
+        """The system-prompt segment listing what is available."""
+        if not self.skills:
+            return ""
+        lines = [skill.advertise() for skill in sorted(self.skills.values(), key=lambda s: s.name)]
+        return (
+            "Available skills (load one with the Skill tool when it applies):\n"
+            + "\n".join(lines)
+        )