local-coding-agent 1.1.0__py3-none-any.whl

local_coding_agent/benchmark/_runner.py

@@ -0,0 +1,259 @@
+"""Benchmark execution: run one model against a disposable fixture and judge it."""
+
+from __future__ import annotations
+
+import json
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from time import perf_counter_ns
+from typing import Any, Mapping, Protocol, Sequence
+
+from ..controller import Controller
+
+from ._cases import BenchmarkCase, default_cases
+from ._judge import _judge_patch, _validate_patch_for_case
+
+
+class ChatModel(Protocol):
+    def chat(self, messages: list[dict[str, Any]], *, tools: list[dict[str, Any]]) -> dict[str, Any]: ...
+
+
+class InstrumentedModel:
+    """Collect metrics returned by Ollama without changing the model response."""
+
+    def __init__(self, model: ChatModel) -> None:
+        self.model = model
+        self.model_calls = 0
+        self.total_duration_ns = 0
+        self.load_duration_ns = 0
+        self.prompt_tokens = 0
+        self.prompt_eval_duration_ns = 0
+        self.eval_count = 0
+        self.eval_duration_ns = 0
+        self.proposed_patches: list[str] = []
+        self.proposed_edits: list[list[dict[str, Any]]] = []
+
+    def chat(self, messages: list[dict[str, Any]], *, tools: list[dict[str, Any]]) -> dict[str, Any]:
+        response = self.model.chat(messages, tools=tools)
+        self.model_calls += 1
+        self._record_proposed_patches(response)
+        self.total_duration_ns += _metric_int(response, "total_duration")
+        self.load_duration_ns += _metric_int(response, "load_duration")
+        self.prompt_tokens += _metric_int(response, "prompt_eval_count")
+        self.prompt_eval_duration_ns += _metric_int(response, "prompt_eval_duration")
+        self.eval_count += _metric_int(response, "eval_count")
+        self.eval_duration_ns += _metric_int(response, "eval_duration")
+        return response
+
+    def _record_proposed_patches(self, response: Mapping[str, Any]) -> None:
+        message = response.get("message")
+        if not isinstance(message, Mapping):
+            return
+        calls = message.get("tool_calls") or []
+        if not calls:
+            compatible_call = _decode_content_tool_call(message.get("content"))
+            if compatible_call is not None:
+                calls = [compatible_call]
+        for call in calls:
+            if not isinstance(call, Mapping):
+                continue
+            function = call.get("function")
+            if not isinstance(function, Mapping) or function.get("name") != "propose_patch":
+                continue
+            arguments = function.get("arguments")
+            if isinstance(arguments, str):
+                try:
+                    arguments = json.loads(arguments)
+                except json.JSONDecodeError:
+                    continue
+            if isinstance(arguments, Mapping):
+                if isinstance(arguments.get("patch"), str):
+                    self.proposed_patches.append(arguments["patch"])
+                if isinstance(arguments.get("edits"), list):
+                    self.proposed_edits.append(arguments["edits"])
+
+
+@dataclass(frozen=True)
+class BenchmarkCaseResult:
+    case_id: str
+    status: str
+    correct: bool
+    loop_reliable: bool
+    validation_valid: bool
+    patch_applied: bool
+    patch_source: str
+    patch_error: str
+    wall_time_ms: float
+    model_calls: int
+    tool_calls: int
+    total_duration_ns: int
+    load_duration_ns: int
+    prompt_tokens: int
+    prompt_eval_duration_ns: int
+    eval_count: int
+    eval_duration_ns: int
+    result: Mapping[str, Any]
+
+    def as_dict(self) -> dict[str, Any]:
+        prompt_eval_tps = (
+            round(self.prompt_tokens * 1e9 / self.prompt_eval_duration_ns, 2)
+            if self.prompt_eval_duration_ns > 0
+            else 0.0
+        )
+        eval_tps = (
+            round(self.eval_count * 1e9 / self.eval_duration_ns, 2)
+            if self.eval_duration_ns > 0
+            else 0.0
+        )
+        return {
+            "case_id": self.case_id,
+            "status": self.status,
+            "correct": self.correct,
+            "loop_reliable": self.loop_reliable,
+            "validation_valid": self.validation_valid,
+            "patch_applied": self.patch_applied,
+            "patch_source": self.patch_source,
+            "patch_error": self.patch_error,
+            "wall_time_ms": self.wall_time_ms,
+            "model_calls": self.model_calls,
+            "tool_calls": self.tool_calls,
+            "total_duration_ns": self.total_duration_ns,
+            "load_duration_ns": self.load_duration_ns,
+            "prompt_tokens": self.prompt_tokens,
+            "prompt_eval_duration_ns": self.prompt_eval_duration_ns,
+            "prompt_eval_tps": prompt_eval_tps,
+            "eval_count": self.eval_count,
+            "eval_duration_ns": self.eval_duration_ns,
+            "eval_tps": eval_tps,
+            "result": dict(self.result),
+        }
+
+
+def _decode_content_tool_call(content: Any) -> dict[str, Any] | None:
+    if not isinstance(content, str) or not content.strip():
+        return None
+    try:
+        payload = json.loads(content)
+    except json.JSONDecodeError:
+        return None
+    if not isinstance(payload, Mapping):
+        return None
+    name = payload.get("name")
+    arguments = payload.get("arguments")
+    if not isinstance(name, str) or not name or not isinstance(arguments, (Mapping, str)):
+        return None
+    return {"function": {"name": name, "arguments": arguments}}
+
+
+def run_case(
+    model: ChatModel,
+    case: BenchmarkCase,
+    *,
+    max_turns: int = 4,
+) -> BenchmarkCaseResult:
+    """Run one model against a disposable fixture and judge its proposal externally."""
+
+    instrumented = model if isinstance(model, InstrumentedModel) else InstrumentedModel(model)
+    with tempfile.TemporaryDirectory(prefix="local-benchmark-") as temp_dir:
+        workspace = Path(temp_dir)
+        for raw_path, content in case.fixture.items():
+            target = workspace / raw_path
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_text(content, encoding="utf-8", newline="")
+
+        started = perf_counter_ns()
+        result = Controller(instrumented, workspace, max_turns=max_turns).run(case.task)
+        wall_time_ms = (perf_counter_ns() - started) / 1_000_000
+        fallback_patch = instrumented.proposed_patches[-1] if instrumented.proposed_patches else ""
+        fallback_edits = instrumented.proposed_edits[-1] if instrumented.proposed_edits else None
+        patch_applied, correct, patch_error, patch_source = _judge_patch(
+            result,
+            case,
+            workspace,
+            fallback_patch=fallback_patch,
+            fallback_edits=fallback_edits,
+        )
+        tool_calls = sum(
+            1
+            for event in result.get("audit", [])
+            if isinstance(event, Mapping) and event.get("event") == "tool_call"
+        )
+        validation = result.get("validation")
+        validation_valid = isinstance(validation, Mapping) and validation.get("valid") is True
+        if patch_source == "tool_proposal":
+            validation_valid = _validate_patch_for_case(fallback_patch, case).valid
+        status = result.get("status") if isinstance(result.get("status"), str) else "failed"
+        loop_reliable = status == "accepted" and not _has_loop_error(result)
+        return BenchmarkCaseResult(
+            case_id=case.id,
+            status=status,
+            correct=correct,
+            loop_reliable=loop_reliable,
+            validation_valid=validation_valid,
+            patch_applied=patch_applied,
+            patch_source=patch_source,
+            patch_error=patch_error,
+            wall_time_ms=wall_time_ms,
+            model_calls=instrumented.model_calls,
+            tool_calls=tool_calls,
+            total_duration_ns=instrumented.total_duration_ns,
+            load_duration_ns=instrumented.load_duration_ns,
+            prompt_tokens=instrumented.prompt_tokens,
+            prompt_eval_duration_ns=instrumented.prompt_eval_duration_ns,
+            eval_count=instrumented.eval_count,
+            eval_duration_ns=instrumented.eval_duration_ns,
+            result=result,
+        )
+
+
+def run_benchmark(
+    model_name: str,
+    model: ChatModel,
+    *,
+    cases: Sequence[BenchmarkCase] | None = None,
+    repeats: int = 1,
+    max_turns: int = 4,
+) -> dict[str, Any]:
+    if repeats <= 0:
+        raise ValueError("repeats must be positive")
+    from ._summary import summarize_results
+    selected = tuple(cases or default_cases())
+    results = [
+        run_case(model, case, max_turns=max_turns)
+        for _ in range(repeats)
+        for case in selected
+    ]
+    return {
+        "model": model_name,
+        "repeats": repeats,
+        "cases": [result.as_dict() for result in results],
+        "summary": summarize_results(model_name, results),
+    }
+
+
+def write_artifact(path: str | Path, artifact: Mapping[str, Any]) -> None:
+    target = Path(path)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_bytes(
+        (json.dumps(artifact, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
+    )
+
+
+def _has_loop_error(result: Mapping[str, Any]) -> bool:
+    error = result.get("error")
+    if not isinstance(error, Mapping):
+        return False
+    return error.get("kind") in {
+        "duplicate_tool_call",
+        "max_turns",
+        "model_error",
+        "policy",
+        "invalid_json",
+        "invalid_response",
+    }
+
+
+def _metric_int(response: Mapping[str, Any], key: str) -> int:
+    value = response.get(key)
+    return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0

local_coding_agent/benchmark/_stats.py

@@ -0,0 +1,22 @@
+"""Statistical helpers used by benchmark summaries."""
+
+from __future__ import annotations
+
+import math
+
+
+def _percent(numerator: int, denominator: int) -> float:
+    return round(numerator * 100.0 / denominator, 2)
+
+
+def _wilson_score_interval(successes: int, total: int, confidence_z: float = 1.96) -> tuple[float, float]:
+    if total <= 0:
+        return 0.0, 0.0
+    p = successes / total
+    z = confidence_z
+    denominator = 1.0 + (z * z) / total
+    center = (p + (z * z) / (2.0 * total)) / denominator
+    margin = (z * math.sqrt((p * (1.0 - p) + (z * z) / (4.0 * total)) / total)) / denominator
+    low = max(0.0, center - margin)
+    high = min(1.0, center + margin)
+    return round(low * 100.0, 2), round(high * 100.0, 2)

local_coding_agent/benchmark/_summary.py

@@ -0,0 +1,143 @@
+"""Benchmark result summaries and failure taxonomy."""
+
+from __future__ import annotations
+
+from statistics import mean, median
+from typing import Any, Mapping, Sequence
+
+from ._runner import BenchmarkCaseResult
+from ._stats import _percent, _wilson_score_interval
+
+
+def summarize_results(model_name: str, results: Sequence[BenchmarkCaseResult]) -> dict[str, Any]:
+    total = len(results)
+    if total == 0:
+        raise ValueError("cannot summarize an empty benchmark")
+    total_eval_tokens = sum(result.eval_count for result in results)
+    total_eval_duration_ns = sum(result.eval_duration_ns for result in results)
+    eval_tps = (
+        round(total_eval_tokens * 1_000_000_000.0 / total_eval_duration_ns, 2)
+        if total_eval_duration_ns > 0
+        else 0.0
+    )
+    total_prompt_tokens = sum(result.prompt_tokens for result in results)
+    total_prompt_eval_duration_ns = sum(result.prompt_eval_duration_ns for result in results)
+    prompt_eval_tps = (
+        round(total_prompt_tokens * 1_000_000_000.0 / total_prompt_eval_duration_ns, 2)
+        if total_prompt_eval_duration_ns > 0
+        else 0.0
+    )
+    correct_count = sum(result.correct for result in results)
+    loop_reliable_count = sum(result.loop_reliable for result in results)
+    patch_applied_count = sum(result.patch_applied for result in results)
+    return {
+        "model": model_name,
+        "cases": total,
+        "correctness_percent": _percent(correct_count, total),
+        "correctness_ci_95": _wilson_score_interval(correct_count, total),
+        "tool_loop_reliability_percent": _percent(loop_reliable_count, total),
+        "tool_loop_reliability_ci_95": _wilson_score_interval(loop_reliable_count, total),
+        "validation_percent": _percent(sum(result.validation_valid for result in results), total),
+        "patch_apply_percent": _percent(patch_applied_count, total),
+        "patch_apply_ci_95": _wilson_score_interval(patch_applied_count, total),
+        "error_categories": _categorize_errors(results),
+        "failure_taxonomy": categorize_failure_taxonomy(results),
+        "average_wall_time_ms": round(mean(result.wall_time_ms for result in results), 3),
+        "median_wall_time_ms": round(median(result.wall_time_ms for result in results), 3),
+        "model_calls": sum(result.model_calls for result in results),
+        "tool_calls": sum(result.tool_calls for result in results),
+        "total_duration_ms": round(sum(result.total_duration_ns for result in results) / 1_000_000, 3),
+        "load_duration_ms": round(sum(result.load_duration_ns for result in results) / 1_000_000, 3),
+        "prompt_tokens": total_prompt_tokens,
+        "eval_tokens": total_eval_tokens,
+        "prompt_eval_duration_ms": round(total_prompt_eval_duration_ns / 1_000_000, 3),
+        "eval_duration_ms": round(total_eval_duration_ns / 1_000_000, 3),
+        "eval_tokens_per_second": eval_tps,
+        "prompt_tokens_per_second": prompt_eval_tps,
+    }
+
+
+def _categorize_errors(results: Sequence[BenchmarkCaseResult]) -> dict[str, int]:
+    categories: dict[str, int] = {}
+    for r in results:
+        if r.correct and r.loop_reliable:
+            continue
+        err_msg = r.patch_error or ""
+        error_kind = (
+            r.result.get("error", {}).get("kind")
+            if isinstance(r.result.get("error"), Mapping)
+            else None
+        )
+        if "not line-aligned" in err_msg:
+            cat = "search_not_line_aligned"
+        elif "search block not found" in err_msg or "not found" in err_msg:
+            cat = "search_not_found"
+        elif "ambiguous" in err_msg:
+            cat = "search_ambiguous"
+        elif "git apply exited" in err_msg or "does not apply" in err_msg:
+            cat = "git_apply_failed"
+        elif "oracle mismatch" in err_msg:
+            cat = "oracle_mismatch"
+        elif error_kind:
+            cat = str(error_kind)
+        elif not r.patch_applied:
+            cat = "patch_not_applied"
+        else:
+            cat = "other"
+        categories[cat] = categories.get(cat, 0) + 1
+    return dict(sorted(categories.items()))
+
+
+CONTRACT_FRICTION_CATEGORIES: frozenset[str] = frozenset({
+    "search_not_line_aligned",
+    "search_not_found",
+    "search_ambiguous",
+    "git_apply_failed",
+    "patch_not_applied",
+    "invalid_json",
+    "invalid_response",
+    "duplicate_tool_call",
+    "preflight_rejected",
+    "context_limit",
+    "policy",
+})
+
+CAPABILITY_FAILURE_CATEGORIES: frozenset[str] = frozenset({
+    "oracle_mismatch",
+    "test_check_failed",
+    "check_failed",
+    "model_error",
+    "timeout",
+    "cancelled",
+})
+
+
+def categorize_failure_taxonomy(results: Sequence[BenchmarkCaseResult]) -> dict[str, Any]:
+    categories = _categorize_errors(results)
+    contract_friction: dict[str, int] = {}
+    capability_failures: dict[str, int] = {}
+    other: dict[str, int] = {}
+
+    for cat, count in categories.items():
+        if cat in CONTRACT_FRICTION_CATEGORIES:
+            contract_friction[cat] = count
+        elif cat in CAPABILITY_FAILURE_CATEGORIES:
+            capability_failures[cat] = count
+        else:
+            other[cat] = count
+
+    friction_count = sum(contract_friction.values())
+    capability_count = sum(capability_failures.values())
+    other_count = sum(other.values())
+    total_failures = friction_count + capability_count + other_count
+
+    return {
+        "total_failures": total_failures,
+        "contract_friction_count": friction_count,
+        "capability_failure_count": capability_count,
+        "friction_ratio": round(friction_count / total_failures, 2) if total_failures else 0.0,
+        "capability_ratio": round(capability_count / total_failures, 2) if total_failures else 0.0,
+        "contract_friction": dict(sorted(contract_friction.items())),
+        "capability_failures": dict(sorted(capability_failures.items())),
+        "other": dict(sorted(other.items())),
+    }

local_coding_agent/benchmark_oracle_worker.py

@@ -0,0 +1,169 @@
+"""Restricted child process for benchmark correctness oracles."""
+
+from __future__ import annotations
+
+import builtins
+import contextlib
+import io
+import json
+import sys
+from pathlib import Path as HostPath
+from typing import Any
+
+
+class _RestrictedPath:
+    def __init__(self, root: str, relative: str = "") -> None:
+        self._root_text = str(HostPath(root).resolve())
+        self._relative = relative.replace("\\", "/").strip("/")
+
+    def __truediv__(self, child: str) -> "_RestrictedPath":
+        if not isinstance(child, str):
+            raise TypeError("restricted path component must be a string")
+        relative = "/".join(part for part in (self._relative, child) if part)
+        return _RestrictedPath(self._root_text, relative)
+
+    @property
+    def name(self) -> str:
+        return self._relative.rsplit("/", 1)[-1]
+
+    def _host_path(self) -> HostPath:
+        root = HostPath(self._root_text)
+        candidate = (root / self._relative).resolve()
+        try:
+            candidate.relative_to(root)
+        except ValueError as error:
+            raise PermissionError("oracle path escapes the disposable workspace") from error
+        return candidate
+
+    def read_text(self, encoding: str = "utf-8") -> str:
+        return self._host_path().read_text(encoding=encoding)
+
+    def __str__(self) -> str:
+        return str(self._host_path())
+
+
+def _restricted_open(file: Any, mode: str = "r", *args: Any, **kwargs: Any):
+    if not isinstance(file, _RestrictedPath):
+        raise PermissionError("oracle file access is limited to the disposable workspace")
+    if any(flag in mode for flag in ("w", "a", "+", "x")):
+        raise PermissionError("oracle workspace is read-only")
+    return open(file._host_path(), mode, *args, **kwargs)
+
+
+def _restricted_import(name: str, *args: Any, **kwargs: Any) -> Any:
+    if name != "json":
+        raise ImportError(f"oracle import is not allowlisted: {name}")
+    return builtins.__import__(name, *args, **kwargs)
+
+
+class _BoundedText(io.StringIO):
+    def __init__(self, limit: int = 4096) -> None:
+        super().__init__()
+        self._limit = limit
+
+    def write(self, value: str) -> int:
+        remaining = max(0, self._limit - len(self.getvalue()))
+        return super().write(value[:remaining])
+
+
+def _fixture_builtins() -> dict[str, Any]:
+    allowed = {
+        key: getattr(builtins, key)
+        for key in (
+            "bool",
+            "dict",
+            "enumerate",
+            "Exception",
+            "float",
+            "ImportError",
+            "int",
+            "isinstance",
+            "len",
+            "list",
+            "range",
+            "set",
+            "str",
+            "sum",
+            "tuple",
+            "type",
+            "ValueError",
+            "abs",
+            "all",
+            "any",
+            "max",
+            "min",
+            "reversed",
+            "sorted",
+            "zip",
+            "__build_class__",
+        )
+    }
+    allowed["open"] = _restricted_open
+    allowed["__import__"] = _restricted_import
+    return allowed
+
+
+def _load_function(path: _RestrictedPath, name: str) -> Any:
+    source = path.read_text(encoding="utf-8")
+    namespace: dict[str, Any] = {
+        "__builtins__": _fixture_builtins(),
+        "__name__": "benchmark_fixture",
+        "__file__": str(path),
+    }
+    exec(compile(source, str(path), "exec"), namespace)
+    function = namespace.get(name)
+    if not callable(function):
+        raise ValueError(f"external oracle could not load {name} from {path.name}")
+    return function
+
+
+def main() -> int:
+    payload = json.loads(sys.stdin.buffer.read().decode("utf-8"))
+    root = payload["workspace"]
+    oracle_name = payload["oracle_name"]
+    oracle_source = payload["oracle_source"]
+    if not isinstance(root, str) or not isinstance(oracle_name, str) or not isinstance(oracle_source, str):
+        raise ValueError("oracle worker payload is invalid")
+    workspace = _RestrictedPath(root)
+    namespace: dict[str, Any] = {
+        "__builtins__": {
+            key: getattr(builtins, key)
+            for key in (
+                "bool",
+                "dict",
+                "Exception",
+                "isinstance",
+                "str",
+                "tuple",
+                "ValueError",
+                "__build_class__",
+            )
+        },
+        "Path": _RestrictedPath,
+        "_load_function": _load_function,
+    }
+    namespace["__builtins__"]["__import__"] = _restricted_import
+    exec(compile(oracle_source, "<benchmark-oracle>", "exec"), namespace)
+    oracle = namespace.get(oracle_name)
+    if not callable(oracle):
+        raise ValueError(f"oracle function not found: {oracle_name}")
+    correct, detail = oracle(workspace)
+    if not isinstance(correct, bool) or not isinstance(detail, str):
+        raise ValueError("oracle must return (bool, str)")
+    print(json.dumps({"ok": True, "correct": correct, "detail": detail}, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    output = _BoundedText()
+    error_output = _BoundedText()
+    try:
+        with contextlib.redirect_stdout(output), contextlib.redirect_stderr(error_output):
+            exit_code = main()
+    except BaseException as error:  # worker boundary: serialize oracle failure
+        exit_code = 0
+        output = _BoundedText()
+        output.write(json.dumps({"ok": False, "error": str(error)}, ensure_ascii=False))
+    sys.stdout.write(output.getvalue())
+    sys.stderr.write(error_output.getvalue())
+    raise SystemExit(exit_code)