turnloop 0.1.0__py3-none-any.whl

turnloop/diagnostics.py

@@ -0,0 +1,233 @@
+"""`turnloop doctor`.
+
+Not a nicety on Windows. Two of this harness's failure modes are invisible from a
+traceback — the WSL `bash` stub, and a self-hosted endpoint that is merely cold
+rather than broken — and both are things a user will otherwise spend an hour on.
+"""
+
+from __future__ import annotations
+
+import shutil
+import sys
+from pathlib import Path
+
+from turnloop.config import Settings
+from turnloop.core.tokens import rough_tokens, tool_spec_tokens
+
+OK = "ok"
+WARN = "warn"
+FAIL = "fail"
+
+_MARKS = {OK: "+", WARN: "!", FAIL: "x"}
+
+
+def run_doctor(settings: Settings, cwd: Path) -> int:
+    rows: list[tuple[str, str, str]] = []
+
+    rows.append((OK, "python", f"{sys.version.split()[0]} on {sys.platform}"))
+    rows.append((OK, "project root", str(settings.project_root)))
+    rows.append((OK, "config layers", ", ".join(settings.sources)))
+
+    rows += _shell_rows()
+    rows += _tooling_rows()
+    rows += _provider_rows(settings)
+    rows += _context_rows(settings, cwd)
+    rows += _permission_rows(settings)
+    rows += _extras_rows(settings)
+
+    width = max(len(label) for _, label, _ in rows) + 2
+    for status, label, detail in rows:
+        print(f" {_MARKS[status]} {label.ljust(width)} {detail}")
+
+    failures = sum(1 for status, _, _ in rows if status == FAIL)
+    warnings = sum(1 for status, _, _ in rows if status == WARN)
+    print()
+    if failures:
+        print(f"{failures} problem(s), {warnings} warning(s).")
+        return 1
+    print(f"No problems. {warnings} warning(s).")
+    return 0
+
+
+def _shell_rows() -> list[tuple[str, str, str]]:
+    from turnloop.errors import ShellNotFound
+    from turnloop.tools.shell import IS_WINDOWS, _is_wsl_stub, resolve_shell
+
+    rows: list[tuple[str, str, str]] = []
+    try:
+        spec = resolve_shell()
+        rows.append((OK, "shell", f"{spec.kind} at {spec.exe}"))
+        if not spec.posix:
+            rows.append(
+                (
+                    WARN,
+                    "shell dialect",
+                    "PowerShell. Models write POSIX shell by default, so expect "
+                    "some Bash calls to fail. Installing Git for Windows fixes this.",
+                )
+            )
+        else:
+            rows.append((OK, "shell dialect", "POSIX"))
+    except ShellNotFound as exc:
+        rows.append((FAIL, "shell", str(exc)))
+
+    if IS_WINDOWS:
+        on_path = shutil.which("bash")
+        if on_path and _is_wsl_stub(Path(on_path)):
+            rows.append(
+                (
+                    WARN,
+                    "bash on PATH",
+                    f"{on_path} is the WSL launcher, not a real bash. It is ignored "
+                    "deliberately — with no distribution installed it exits 255 with "
+                    "UTF-16LE error text.",
+                )
+            )
+    return rows
+
+
+def _tooling_rows() -> list[tuple[str, str, str]]:
+    rows = []
+    for name, purpose in (
+        ("git", "ignore-aware Glob, repository context"),
+        ("rg", "fast Grep (falls back to pure Python)"),
+    ):
+        found = shutil.which(name)
+        rows.append(
+            (OK if found else WARN, name, found or f"not found — {purpose} degrades")
+        )
+    return rows
+
+
+def _provider_rows(settings: Settings) -> list[tuple[str, str, str]]:
+    import os
+
+    rows: list[tuple[str, str, str]] = []
+    active = settings.provider
+    for name in sorted(settings.providers):
+        cfg = settings.providers[name]
+        marker = " (active)" if name == active else ""
+        detail = f"{cfg.kind} {cfg.model}"
+
+        if cfg.api_key_env:
+            if os.environ.get(cfg.api_key_env):
+                detail += f", {cfg.api_key_env} set"
+                status = OK
+            else:
+                detail += f", {cfg.api_key_env} NOT set"
+                status = FAIL if name == active else WARN
+        else:
+            status = OK
+            if cfg.kind == "openai_compat" and not cfg.api_key:
+                detail += ", unauthenticated"
+
+        if cfg.caps.cost_per_hour:
+            detail += f", ${cfg.caps.cost_per_hour:.2f}/hr wall clock"
+        if cfg.health_url:
+            detail += ", health-gated"
+        if cfg.timeout_s is None and cfg.health_url:
+            detail += ", unbounded read timeout"
+
+        rows.append((status, f"provider {name}{marker}", detail))
+    return rows
+
+
+def _context_rows(settings: Settings, cwd: Path) -> list[tuple[str, str, str]]:
+    from turnloop.agent.system_prompt import build_system
+    from turnloop.providers.base import Capabilities
+    from turnloop.tools.builtin import build_registry
+
+    cfg = settings.provider_config()
+    caps: Capabilities = cfg.caps
+    registry = build_registry(settings, cwd)
+    verbosity = settings.verbosity_for()
+    system = build_system(settings, cwd, registry, caps, settings.permission_mode)
+
+    system_tokens = rough_tokens("\n\n".join(system))
+    tools_tokens = tool_spec_tokens(registry.specs(verbosity))
+    from turnloop.context.budget import output_reserve
+
+    reserve = output_reserve(settings.compaction.reserve_output_tokens, caps.max_output)
+    available = caps.max_context - reserve - system_tokens - tools_tokens
+
+    rows = [
+        (OK, "tools", f"{len(registry)} registered ({', '.join(registry.names())})"),
+        (
+            OK,
+            "context budget",
+            f"{caps.max_context:,} window − {reserve:,} output − {system_tokens:,} system "
+            f"− {tools_tokens:,} tools = {available:,} for history",
+        ),
+        (OK, "tool verbosity", verbosity),
+    ]
+    if available < 20_000:
+        rows.append(
+            (
+                WARN,
+                "context headroom",
+                f"only {available:,} tokens for conversation. Consider "
+                "tool_verbosity=terse for this provider.",
+            )
+        )
+    return rows
+
+
+def _permission_rows(settings: Settings) -> list[tuple[str, str, str]]:
+    from turnloop.errors import ConfigError
+    from turnloop.permissions.rules import parse_rules
+
+    rows: list[tuple[str, str, str]] = []
+    rows.append((OK, "permission mode", settings.permission_mode))
+    for label, raws in (
+        ("allow", settings.permissions.allow),
+        ("deny", settings.permissions.deny),
+        ("ask", settings.permissions.ask),
+    ):
+        try:
+            parsed = parse_rules(raws)
+            rows.append((OK, f"rules ({label})", f"{len(parsed)} valid"))
+        except ConfigError as exc:
+            rows.append((FAIL, f"rules ({label})", str(exc)))
+
+    if settings.permission_mode == "bypass":
+        rows.append(
+            (
+                WARN,
+                "bypass mode",
+                "every call is allowed except explicit deny rules. Deny rules still apply.",
+            )
+        )
+    return rows
+
+
+def _extras_rows(settings: Settings) -> list[tuple[str, str, str]]:
+    from turnloop.commands.loader import load_commands, load_skills
+    from turnloop.context.memory import discover_memory
+
+    rows: list[tuple[str, str, str]] = []
+    commands = load_commands(settings.project_root)
+    skills = load_skills(settings.project_root)
+    memory = discover_memory(Path.cwd(), settings.project_root)
+
+    rows.append((OK, "slash commands", f"{len(commands)} user-defined" if commands else "none"))
+    rows.append((OK, "skills", ", ".join(sorted(skills)) if skills else "none"))
+    if memory:
+        total = sum(m.tokens for m in memory)
+        status = WARN if total > 4_000 else OK
+        rows.append(
+            (
+                status,
+                "memory files",
+                f"{len(memory)} file(s), ~{total:,} tokens"
+                + (" (over the 4,000 budget; the excess is dropped)" if total > 4_000 else ""),
+            )
+        )
+    else:
+        rows.append((OK, "memory files", "none (create TURNLOOP.md to add project instructions)"))
+
+    enabled = [n for n, c in settings.mcp_servers.items() if c.enabled]
+    rows.append((OK, "mcp servers", ", ".join(enabled) if enabled else "none configured"))
+
+    hook_events = [event for event, matchers in settings.hooks.items() if matchers]
+    rows.append((OK, "hooks", ", ".join(hook_events) if hook_events else "none configured"))
+    return rows

turnloop/errors.py

@@ -0,0 +1,64 @@
+"""Exception taxonomy.
+
+The rule that shapes this file: anything raised below the tool layer must be
+classifiable into retry / cold-boot / fatal without inspecting strings.
+"""
+
+from __future__ import annotations
+
+
+class TurnloopError(Exception):
+    """Base for every error this package raises deliberately."""
+
+
+class ConfigError(TurnloopError):
+    """Malformed settings file, unknown provider name, bad permission rule."""
+
+
+class ProviderError(TurnloopError):
+    """Base for provider/transport failures."""
+
+    def __init__(self, message: str, *, status: int | None = None, body: str | None = None,
+                 retry_after: float | None = None):
+        super().__init__(message)
+        self.status = status
+        self.body = body
+        # Seconds the server asked us to wait. A per-minute token quota needs the
+        # server's number, not our exponential guess: backoff of 2/4/8s against a
+        # 60-second window just spends the retry budget without waiting it out.
+        self.retry_after = retry_after
+
+
+class FatalProviderError(ProviderError):
+    """4xx that will never succeed on retry: bad schema, bad auth, wrong model."""
+
+
+class RetryableProviderError(ProviderError):
+    """429/5xx or a mid-stream disconnect. Exponential backoff applies."""
+
+
+class ColdBootTimeout(ProviderError):
+    """A self-hosted endpoint never became healthy inside its boot budget."""
+
+
+class StreamTruncated(RetryableProviderError):
+    """The stream ended without a terminal event. Partial output is discarded."""
+
+
+class ContextOverflow(TurnloopError):
+    """History still exceeds the window after every compaction tier ran."""
+
+
+class ToolError(TurnloopError):
+    """A tool failed in a way the model should see and can act on.
+
+    Never propagates past ToolRunner.dispatch — it becomes an error tool_result.
+    """
+
+
+class PermissionDenied(ToolError):
+    """A deny rule, or plan mode, blocked this call."""
+
+
+class ShellNotFound(TurnloopError):
+    """No usable shell on this machine (see tools/shell.py for why that happens)."""

turnloop/experiments/configs/bakeoff.yaml

@@ -0,0 +1,50 @@
+# Provider bake-off: the same tasks across every backend.
+#
+# Cost warning. The `glm` arm runs on a self-hosted 4xH200 endpoint at $18.16/hour
+# with a ~29 minute cold boot. The runner groups execution by provider precisely so
+# this arm boots once and then runs everything, instead of paying a cold boot per
+# task. Even so: `modal app stop glm-5-2-serve` afterwards.
+#
+# Ordering note: `glm` is listed last, but the runner sorts mock first and otherwise
+# alphabetically, so provider grouping — not arm order — determines execution.
+
+experiment: provider-bakeoff
+suite: default
+repeats: 3
+
+tasks:
+  - fix_failing_test
+  - implement_from_spec
+  - multi_file_rename
+  - add_cli_flag
+  - grep_then_edit
+  - structural_function
+  - find_the_bug
+  - refuse_outside_root
+
+arms:
+  - name: sonnet
+    provider: anthropic
+    model: claude-sonnet-4-5
+
+  - name: haiku
+    provider: anthropic
+    model: claude-haiku-4-5
+
+  - name: gpt-4.1
+    provider: openai
+    model: gpt-4.1
+
+  - name: gemini-flash
+    provider: gemini
+    model: gemini-2.5-flash
+
+  - name: gpt-oss-120b
+    provider: groq
+    model: openai/gpt-oss-120b
+
+  # Self-hosted GLM-5.2, 744B MoE at W4A16 on 4xH200. 65k context, so it also
+  # exercises the compaction path that the hosted arms mostly never reach.
+  - name: glm-5.2-selfhosted
+    provider: glm
+    tool_verbosity: terse

turnloop/experiments/configs/ceiling-check.yaml

@@ -0,0 +1,31 @@
+# Does a weaker model break the ceiling?
+#
+# GLM-5.2 scored 100% on all four of these tasks across 84 runs, which means no
+# context-engineering ablation run against it can discriminate — every arm ties at
+# the top. An ablation needs a model that sometimes fails.
+#
+# This is the cheapest possible test of whether gpt-4o-mini sits in the useful band.
+# One repeat, four tasks, one arm. What we want is a pass rate somewhere around
+# 40-70%: high enough that the agent is genuinely working, low enough that removing
+# the system prompt or the tool descriptions has room to show up.
+#
+# 0% would be as useless as 100% — a floor is just a ceiling upside down.
+#
+# Note `forces_compaction` will NOT trigger compaction here. The bulky fixture is
+# sized against a 65,536-token window (tier 3 at ~43k); gpt-4o-mini's 128k window
+# puts the trigger near 90k, which the fixture's top-3-files-in-full total of ~50k
+# does not reach. That task measures only correctness on this provider.
+
+experiment: ceiling-check
+suite: default
+repeats: 1
+
+tasks:
+  - fix_failing_test
+  - implement_from_spec
+  - grep_then_edit
+  - forces_compaction
+
+arms:
+  - name: gpt-4o-mini
+    provider: blaxel

turnloop/experiments/configs/context-glm.yaml

@@ -0,0 +1,59 @@
+# context.yaml, retargeted at the self-hosted GLM-5.2 on Modal.
+#
+# Same seven arms and four tasks as context.yaml; only the provider differs. This
+# exists as a separate file rather than an edit because the two endpoints are not
+# interchangeable: NIM gives a 200k window for free but rate-limits hard, while this
+# one gives 65,536 tokens on a GPU billed at $18.16/hour of wall clock.
+#
+# 65k is the window the compaction ladder was designed against, so `forces_compaction`
+# is a real test here rather than a formality -- on a 200k window it barely triggers.
+#
+# COST: 84 runs at roughly 30s each is about 45 minutes of GPU, so on the order of
+# $13, on top of whatever the cold boot already cost. Only run this in an already-warm
+# window, and run `modal app stop glm-5-2-serve` the moment it finishes -- the
+# container otherwise idles for another 10 minutes before scaling down.
+
+experiment: context-engineering-glm
+suite: default
+repeats: 3
+
+tasks:
+  - fix_failing_test
+  - implement_from_spec
+  - grep_then_edit
+  - forces_compaction
+
+arms:
+  - name: baseline
+    provider: glm
+    tool_verbosity: normal
+    compaction: summarize
+    system_variant: default
+
+  - name: terse-tools
+    provider: glm
+    tool_verbosity: terse
+    compaction: summarize
+    system_variant: default
+
+  - name: verbose-tools
+    provider: glm
+    tool_verbosity: verbose
+    compaction: summarize
+    system_variant: default
+
+  - name: micro-only-compaction
+    provider: glm
+    compaction: micro_only
+
+  - name: no-compaction
+    provider: glm
+    compaction: none
+
+  - name: minimal-prompt
+    provider: glm
+    system_variant: minimal
+
+  - name: careful-prompt
+    provider: glm
+    system_variant: careful

turnloop/experiments/configs/context.yaml

@@ -0,0 +1,65 @@
+# Context-engineering ablations, all on one model so the model is not the variable.
+#
+# The model is GLM-5.2 via NVIDIA NIM: a 200k window, which is what makes these
+# ablations runnable at all. `forces_compaction` and the multi-file tasks cannot
+# be expressed inside the 7k the Groq free tier allows.
+#
+# Three questions, and the reason each is worth asking:
+#
+# 1. Tool-description verbosity. Verbose descriptions cost ~1,100 tokens more than
+#    terse ones with the current 10-tool set. On a 65k window that is 2% of
+#    everything. If pass rate is unchanged, terse is free money; if it drops, the
+#    descriptions are load-bearing and that is worth knowing too.
+#
+# 2. Compaction strategy. `summarize` spends a model call; `micro_only` spends
+#    nothing but forgets more; `none` is the control that shows what compaction is
+#    actually buying.
+#
+# 3. System prompt variant. `minimal` strips the behavioral guidance down to two
+#    sentences. This measures how much of a coding agent's competence comes from
+#    its harness rather than its weights — the single most interesting number here.
+
+experiment: context-engineering
+suite: default
+repeats: 3
+
+tasks:
+  - fix_failing_test
+  - implement_from_spec
+  - grep_then_edit
+  - forces_compaction
+
+arms:
+  - name: baseline
+    provider: nvidia
+    tool_verbosity: normal
+    compaction: summarize
+    system_variant: default
+
+  - name: terse-tools
+    provider: nvidia
+    tool_verbosity: terse
+    compaction: summarize
+    system_variant: default
+
+  - name: verbose-tools
+    provider: nvidia
+    tool_verbosity: verbose
+    compaction: summarize
+    system_variant: default
+
+  - name: micro-only-compaction
+    provider: nvidia
+    compaction: micro_only
+
+  - name: no-compaction
+    provider: nvidia
+    compaction: none
+
+  - name: minimal-prompt
+    provider: nvidia
+    system_variant: minimal
+
+  - name: careful-prompt
+    provider: nvidia
+    system_variant: careful

turnloop/experiments/configs/glm-selfhosted.yaml

@@ -0,0 +1,36 @@
+# Same model weights, two serving stacks.
+#
+# GLM-5.2 self-hosted on vLLM (Modal, `--tool-call-parser glm47`) against the same
+# GLM-5.2 hosted on NVIDIA NIM. The model is held constant, so any difference in
+# tool-calling reliability is attributable to the serving stack rather than to the
+# weights -- which is the only way to measure a parser.
+#
+# The two stacks demonstrably differ on the wire: NIM emits each tool call whole in
+# a single delta (id + name + complete arguments JSON), while vLLM's glm47 parser
+# streams the arguments as fragments to be accumulated by index. Whether that costs
+# reliability is the question this run answers.
+#
+# COST: the self-hosted arm bills $18.16/hour of wall clock from cold boot, and the
+# container idles down after 10 minutes. Keep this config small and run it only in
+# a window where the endpoint is already warm. Nine runs is roughly six minutes of
+# GPU. Run `modal app stop glm-5-2-serve` when finished.
+#
+# Arms are executed grouped by provider, so the GPU arm runs its tasks back to back
+# inside one warm window instead of cold-booting per task.
+
+experiment: glm-selfhosted-vs-nim
+suite: default
+repeats: 3
+
+tasks:
+  - fix_failing_test
+  - structural_function
+  - find_the_bug
+
+arms:
+  - name: glm-5.2-nim
+    provider: nvidia
+
+  - name: glm-5.2-selfhosted
+    provider: glm
+    tool_verbosity: terse

turnloop/experiments/configs/groq-free.yaml

@@ -0,0 +1,28 @@
+# Groq free tier, three small tasks, single repeat.
+#
+# Sized for the constraint rather than the model: the on-demand tier allows 8,000
+# tokens per minute *including* requested completion tokens, which leaves ~2,400 for
+# conversation history. That rules out the multi-file tasks — they cannot fit — and
+# it means a multi-turn task spends real wall-clock waiting for the quota window.
+#
+# The mock arm is the free control: same tasks, no network, so a difference between
+# the two rows is the model rather than the harness.
+
+experiment: groq-free
+suite: default
+repeats: 1
+
+tasks:
+  - fix_failing_test
+  - structural_function
+  - find_the_bug
+
+arms:
+  - name: mock-control
+    provider: mock
+    mock_mode: echo
+
+  - name: gpt-oss-120b
+    provider: groq
+    model: openai/gpt-oss-120b
+    tool_verbosity: terse

turnloop/experiments/configs/hard-calibration.yaml

@@ -0,0 +1,26 @@
+# Difficulty calibration for the five ceiling-breaker tasks.
+#
+# Each of these has a plausible wrong solution baked in (see default.yaml's
+# "ceiling-breakers" section), but "has a decoy" is a design intent, not a
+# measurement. This config exists to check the intent held: run each task a
+# few times against a real model and see where the pass rate lands.
+#
+# Target band is 40-70% per task. Below that and the task is just hard, not
+# discriminating; above it and the decoy isn't tempting enough to matter.
+# One arm, three repeats, cheap enough to iterate on the fixtures themselves
+# before trusting them in a bigger sweep.
+
+experiment: hard-calibration
+suite: default
+repeats: 3
+
+tasks:
+  - regression_trap
+  - cross_file_contract
+  - subtle_spec_edge
+  - circular_import
+  - misleading_traceback
+
+arms:
+  - name: gpt-4o-mini
+    provider: blaxel

turnloop/experiments/configs/hard-glm.yaml

@@ -0,0 +1,41 @@
+# The five ceiling-breaker tasks against the real target model.
+#
+# Calibration happens on Blaxel (free, ~5s/request); this config exists to confirm
+# the difficulty transfers to GLM-5.2, which is the model the project actually
+# targets. A task tuned to trip gpt-4o-mini is not automatically hard for a
+# stronger model, and a suite calibrated against the wrong model measures nothing.
+#
+# Two arms so the comparison is in one table: the self-hosted GPU and the same five
+# tasks on Blaxel. Provider grouping means the GPU arm runs its 15 back to back
+# inside a single warm window rather than cold-booting per task.
+#
+# COST: 15 GPU runs at roughly a minute each is ~15 minutes, about $4.50, plus the
+# cold boot (~29 min / ~$9) if the container is not already up. Only run this in an
+# already-warm window, and run `modal app stop glm-5-2-serve -y` the moment it
+# finishes — the container idles a further 10 minutes before scaling down on its own.
+
+experiment: hard-glm
+suite: default
+repeats: 3
+
+# Three of the five, not all five. Calibration on gpt-4o-mini put
+# `misleading_traceback` and `subtle_spec_edge` at 3/3 — a task the *weaker* model
+# solves every time cannot discriminate on the stronger one, so paying GPU minutes
+# to watch GLM-5.2 also score 100% buys nothing.
+#
+# The three kept are the ones whose answer is genuinely unknown: `regression_trap`
+# landed at 1/3 (already in band), and the two that floored at 0/3 are the real
+# question — difficulty is model-relative, so a task a weak model never solves may
+# sit squarely in the useful band for a stronger one.
+tasks:
+  - regression_trap
+  - cross_file_contract
+  - circular_import
+
+arms:
+  - name: gpt-4o-mini
+    provider: blaxel
+
+  - name: glm-5.2-selfhosted
+    provider: glm
+    tool_verbosity: terse

turnloop/experiments/configs/loop.yaml

@@ -0,0 +1,34 @@
+# Agent-loop variants on identical tasks.
+#
+# All three arms run the same harness and the same model; only the instruction
+# scaffolding differs. That is deliberate — if the loops were separate code paths,
+# a difference between arms could be a difference in implementation quality rather
+# than in strategy.
+#
+# What to look for: fan-out should win on wall-clock for the multi-file tasks and
+# lose on token cost (every subagent re-pays for the system prompt and tool
+# schemas). Plan-then-execute should show more turns and better pass rates on the
+# tasks with several steps.
+
+experiment: loop-variants
+suite: default
+repeats: 3
+
+tasks:
+  - multi_file_rename
+  - grep_then_edit
+  - find_the_bug
+  - fix_failing_test
+
+arms:
+  - name: single
+    provider: nvidia
+    loop_variant: single
+
+  - name: fanout
+    provider: nvidia
+    loop_variant: fanout
+
+  - name: plan-then-execute
+    provider: nvidia
+    loop_variant: plan_then_execute