mindcode 0.2.0__py3-none-any.whl

mindcode/bench/predictions.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+from mindagent.core.artifact import validate_artifact_path, write_artifact_text
+
+from .schema import BenchmarkCase, BenchmarkResult
+
+
+PATCH_PATTERN = re.compile(
+    r"(?:diff[\w_\.\ /\-]+\n)?---\s+a/(?:.*?)\n"
+    r"\+\+\+\s+b/(?:.*?)(?=diff\ |---\ a/|\Z)",
+    re.DOTALL,
+)
+
+
+def extract_patch(text: str | None) -> str:
+    if not text:
+        return ""
+    matches = PATCH_PATTERN.findall(text)
+    if matches:
+        return "\n".join(match.rstrip() for match in matches).strip() + "\n"
+    stripped = text.strip()
+    if stripped.startswith("--- a/") and "\n+++ b/" in stripped:
+        return stripped + "\n"
+    return ""
+
+
+def append_swe_prediction(
+    run_dir: Path,
+    case: BenchmarkCase,
+    result: BenchmarkResult,
+    model_patch: str,
+    model_name: str,
+) -> None:
+    prediction = {
+        "instance_id": case.case_id,
+        "model_name_or_path": model_name,
+        "model_patch": model_patch,
+    }
+    path = run_dir / "predictions.jsonl"
+    path = validate_artifact_path(run_dir, path)
+    write_artifact_text(
+        path,
+        json.dumps(prediction, ensure_ascii=False) + "\n",
+        root=run_dir,
+        append=True,
+    )
+
+    manifest_path = run_dir / "swe_predictions.json"
+    manifest_path = validate_artifact_path(run_dir, manifest_path)
+    manifest = {
+        "predictions_path": str(path),
+        "instances": [case.case_id],
+    }
+    if manifest_path.exists():
+        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        manifest["instances"].append(case.case_id)
+    write_artifact_text(
+        manifest_path,
+        json.dumps(manifest, ensure_ascii=False, indent=2),
+        root=run_dir,
+    )

mindcode/bench/report.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any
+
+from .schema import BenchmarkResult
+
+
+def build_summary(results: list[BenchmarkResult]) -> dict[str, Any]:
+    total = len(results)
+    passed = sum(1 for result in results if result.passed)
+    by_category: dict[str, dict[str, Any]] = defaultdict(
+        lambda: {"total": 0, "passed": 0, "pass_rate": 0.0}
+    )
+    for result in results:
+        category = str(result.metrics.get("category", "uncategorized"))
+        item = by_category[category]
+        item["total"] += 1
+        item["passed"] += int(result.passed)
+
+    for item in by_category.values():
+        item["pass_rate"] = item["passed"] / item["total"]
+
+    latency_values = [
+        float(result.metrics.get("latency_s", 0.0)) for result in results
+    ]
+    return {
+        "total": total,
+        "passed": passed,
+        "failed": total - passed,
+        "pass_rate": passed / total if total else 0.0,
+        "avg_latency_s": (
+            sum(latency_values) / len(latency_values)
+            if latency_values
+            else 0.0
+        ),
+        "by_category": dict(sorted(by_category.items())),
+        "failed_cases": [
+            {
+                "case_id": result.case_id,
+                "score": result.score,
+                "failure_phase": result.metrics.get("failure_phase"),
+                "failure_reason": result.metrics.get("failure_reason"),
+            }
+            for result in results
+            if not result.passed
+        ],
+    }

mindcode/bench/runner.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from pathlib import Path
+
+from mindagent.core import AgentResult, safe_run_id_filename
+from mindagent.core.artifact import (
+    ensure_artifact_directory,
+    validate_artifact_path,
+    write_artifact_text,
+)
+
+from ..config import load_config
+from ..policy import ApprovalMode
+from ..runtime import build_runtime
+from .adapters import built_in_adapters
+from .fake_tools import build_fake_tools
+from .jsonutil import to_jsonable
+from .predictions import append_swe_prediction, extract_patch
+from .report import build_summary
+from .schema import BenchmarkCase, BenchmarkResult, BenchmarkRunConfig, CaseTrace
+from .trace import BenchmarkTraceRecorder
+from .workspace import prepare_case_workspace, workspace_diff
+
+
+class MindcodeBenchmarkRunner:
+    def __init__(self, benchmark_root: Path):
+        self.benchmark_root = benchmark_root
+        self.adapters = built_in_adapters()
+
+    def list_suites(self) -> list[tuple[str, int]]:
+        suites: list[tuple[str, int]] = []
+        for name, adapter in sorted(self.adapters.items()):
+            try:
+                suites.append((name, len(adapter.load_cases(self.benchmark_root))))
+            except FileNotFoundError:
+                suites.append((name, 0))
+        return suites
+
+    async def run(self, config: BenchmarkRunConfig) -> list[BenchmarkResult]:
+        if config.concurrency != 1:
+            raise ValueError("第一版 benchmark runner 仅支持 concurrency=1")
+        adapter = self.adapters.get(config.suite)
+        if adapter is None:
+            available = ", ".join(sorted(self.adapters))
+            raise ValueError(f"未知 suite: {config.suite}; 可用: {available}")
+
+        cases = _select_cases(
+            adapter.load_cases(self.benchmark_root),
+            config.case_ids,
+            config.limit,
+        )
+        scorer = adapter.build_scorer()
+        output_root = ensure_artifact_directory(config.output_root)
+        run_dir = _run_output_dir(config.output_root, config.run_id)
+        traces_dir = run_dir / "traces"
+        workspaces_root = run_dir / "workspaces"
+        ensure_artifact_directory(run_dir, root=output_root)
+        ensure_artifact_directory(traces_dir, root=output_root)
+        ensure_artifact_directory(workspaces_root, root=output_root)
+        config_path = validate_artifact_path(
+            output_root,
+            run_dir / "config.json",
+        )
+        write_artifact_text(
+            config_path,
+            json.dumps(to_jsonable(config), ensure_ascii=False, indent=2),
+            root=output_root,
+        )
+
+        results: list[BenchmarkResult] = []
+        results_path = run_dir / "results.jsonl"
+        for case in cases:
+            result = await self._run_case(
+                case=case,
+                config=config,
+                scorer=scorer,
+                workspaces_root=workspaces_root,
+                traces_dir=traces_dir,
+            )
+            results.append(result)
+            write_artifact_text(
+                results_path,
+                json.dumps(to_jsonable(result), ensure_ascii=False) + "\n",
+                root=output_root,
+                append=True,
+            )
+            summary_path = validate_artifact_path(
+                output_root,
+                run_dir / "summary.json",
+            )
+            write_artifact_text(
+                summary_path,
+                json.dumps(build_summary(results), ensure_ascii=False, indent=2),
+                root=output_root,
+            )
+        return results
+
+    async def _run_case(
+        self,
+        *,
+        case: BenchmarkCase,
+        config: BenchmarkRunConfig,
+        scorer,
+        workspaces_root: Path,
+        traces_dir: Path,
+    ) -> BenchmarkResult:
+        started_at = time.time()
+        workspace = prepare_case_workspace(workspaces_root, case)
+        recorder = BenchmarkTraceRecorder()
+        run_result: AgentResult | None = None
+        error: str | None = None
+        runtime = None
+
+        try:
+            mindcode_config = load_config()
+            provider_config = mindcode_config.get_provider(config.provider)
+            runtime = build_runtime(
+                workspace,
+                provider_config=provider_config,
+                approve_mode=ApprovalMode(config.approve_mode),
+                model=config.model,
+                stream=False,
+                event_handler=recorder,
+                max_steps=config.max_steps,
+                step_timeout_s=config.step_timeout_s,
+                total_timeout_s=config.total_timeout_s,
+                extra_tools=build_fake_tools(case)
+                if case.suite == "mini_bfcl"
+                else None,
+            )
+            run_result = await runtime.run(
+                case.prompt,
+                metadata={
+                    "benchmark_suite": case.suite,
+                    "benchmark_case_id": case.case_id,
+                },
+            )
+        except Exception as exc:  # pragma: no cover - exercised by real runs
+            error = f"{type(exc).__name__}: {exc}"
+        finally:
+            if runtime is not None:
+                try:
+                    await runtime.close(close_dependencies=True)
+                except Exception as exc:
+                    if error is None:
+                        error = f"{type(exc).__name__}: {exc}"
+
+        trace = CaseTrace(
+            case=case,
+            config=config,
+            agent_events=recorder.agent_events,
+            observations=to_jsonable(run_result.observations)
+            if run_result
+            else [],
+            tool_calls=recorder.tool_calls,
+            workspace_diff=workspace_diff(case, workspace),
+        )
+        score = scorer.score(case, run_result, workspace, trace)
+        if score.failure_phase or score.failure_reason:
+            trace.failure_analysis = {
+                "phase": score.failure_phase,
+                "reason": score.failure_reason,
+            }
+        trace_path = _case_trace_path(traces_dir, case.case_id)
+        completed_at = time.time()
+        metrics = {
+            **score.metrics,
+            "category": case.category,
+            "latency_s": completed_at - started_at,
+            "steps": run_result.steps if run_result else 0,
+            "tool_calls": len(recorder.tool_calls),
+            "agent_events": len(recorder.agent_events),
+            "run_outcome": run_result.outcome.value if run_result else None,
+            "failure_phase": score.failure_phase,
+            "failure_reason": score.failure_reason,
+        }
+        result = BenchmarkResult(
+            case_id=case.case_id,
+            suite=case.suite,
+            passed=score.passed,
+            score=score.score,
+            final_answer=run_result.final_answer if run_result else None,
+            error=error or (run_result.error if run_result else None),
+            started_at=started_at,
+            completed_at=completed_at,
+            metrics=metrics,
+            trace_path=str(trace_path),
+            workspace_path=str(workspace),
+        )
+        if case.suite == "swe_bench":
+            patch = trace.workspace_diff or extract_patch(result.final_answer)
+            append_swe_prediction(
+                traces_dir.parent,
+                case,
+                result,
+                patch,
+                config.model or config.provider,
+            )
+        trace_payload = {
+            "case": to_jsonable(case),
+            "config": to_jsonable(config),
+            "result": to_jsonable(result),
+            "agent_events": trace.agent_events,
+            "observations": trace.observations,
+            "tool_calls": trace.tool_calls,
+            "workspace_diff": trace.workspace_diff,
+            "failure_analysis": trace.failure_analysis,
+        }
+        write_artifact_text(
+            trace_path,
+            json.dumps(trace_payload, ensure_ascii=False, indent=2),
+            root=traces_dir.parent,
+        )
+        return result
+
+
+def run_benchmark(config: BenchmarkRunConfig, benchmark_root: Path) -> list[BenchmarkResult]:
+    return asyncio.run(MindcodeBenchmarkRunner(benchmark_root).run(config))
+
+
+def _case_trace_path(traces_dir: Path, case_id: str) -> Path:
+    """Keep per-case trace artifacts contained by the configured directory."""
+    path = traces_dir / f"{safe_run_id_filename(case_id)}.json"
+    return validate_artifact_path(traces_dir, path)
+
+
+def _run_output_dir(output_root: Path, run_id: str) -> Path:
+    """Keep the benchmark run directory inside the configured output root."""
+    path = output_root / safe_run_id_filename(run_id)
+    return validate_artifact_path(output_root, path)
+
+
+def _select_cases(
+    cases: list[BenchmarkCase],
+    case_ids: tuple[str, ...],
+    limit: int | None,
+) -> list[BenchmarkCase]:
+    selected = cases
+    if case_ids:
+        requested = set(case_ids)
+        selected = [case for case in selected if case.case_id in requested]
+        missing = sorted(requested - {case.case_id for case in selected})
+        if missing:
+            raise ValueError("未知 case: " + ", ".join(missing))
+    if limit is not None:
+        selected = selected[:limit]
+    return selected

mindcode/bench/schema.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class BenchmarkFile:
+    path: str
+    content: str
+
+
+@dataclass(frozen=True)
+class BenchmarkCase:
+    case_id: str
+    suite: str
+    category: str
+    prompt: str
+    expected: Any | None = None
+    files: tuple[BenchmarkFile, ...] = ()
+    tools: tuple[str, ...] = ()
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class ScoreResult:
+    passed: bool
+    score: float
+    metrics: dict[str, Any] = field(default_factory=dict)
+    failure_phase: str | None = None
+    failure_reason: str | None = None
+
+
+@dataclass(frozen=True)
+class BenchmarkResult:
+    case_id: str
+    suite: str
+    passed: bool
+    score: float
+    final_answer: str | None
+    error: str | None
+    started_at: float
+    completed_at: float
+    metrics: dict[str, Any]
+    trace_path: str
+    workspace_path: str
+
+
+@dataclass(frozen=True)
+class BenchmarkRunConfig:
+    suite: str
+    run_id: str
+    provider: str
+    model: str | None
+    approve_mode: str
+    workspace_root: Path
+    output_root: Path
+    limit: int | None = None
+    case_ids: tuple[str, ...] = ()
+    concurrency: int = 1
+    max_steps: int = 50
+    step_timeout_s: float | None = 300.0
+    total_timeout_s: float | None = 1800.0
+    enable_subagents: bool = False
+    enable_background_tasks: bool = False
+    seed: int | None = None
+
+
+@dataclass
+class CaseTrace:
+    case: BenchmarkCase
+    config: BenchmarkRunConfig
+    agent_events: list[dict[str, Any]] = field(default_factory=list)
+    observations: list[dict[str, Any]] = field(default_factory=list)
+    tool_calls: list[dict[str, Any]] = field(default_factory=list)
+    workspace_diff: str = ""
+    failure_analysis: dict[str, Any] = field(default_factory=dict)

mindcode/bench/scorers/base.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Protocol
+
+from mindagent.core import AgentResult
+
+from ..schema import BenchmarkCase, CaseTrace, ScoreResult
+
+
+class Scorer(Protocol):
+    def score(
+        self,
+        case: BenchmarkCase,
+        run_result: AgentResult | None,
+        workspace: Path,
+        trace: CaseTrace,
+    ) -> ScoreResult: ...
+
+
+def normalize_text(value: Any) -> str:
+    return str(value or "").strip()

mindcode/bench/scorers/composite.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from mindagent.core import AgentResult
+
+from ..schema import BenchmarkCase, CaseTrace, ScoreResult
+from .base import Scorer
+
+
+class CompositeScorer:
+    def __init__(self, scorers: list[Scorer]):
+        self.scorers = scorers
+
+    def score(
+        self,
+        case: BenchmarkCase,
+        run_result: AgentResult | None,
+        workspace: Path,
+        trace: CaseTrace,
+    ) -> ScoreResult:
+        results = [
+            scorer.score(case, run_result, workspace, trace)
+            for scorer in self.scorers
+        ]
+        passed = all(result.passed for result in results)
+        score = (
+            sum(result.score for result in results) / len(results)
+            if results
+            else 0.0
+        )
+        first_failure = next(
+            (result for result in results if not result.passed),
+            None,
+        )
+        return ScoreResult(
+            passed=passed,
+            score=score,
+            metrics={
+                "components": [
+                    {
+                        "passed": result.passed,
+                        "score": result.score,
+                        "failure_phase": result.failure_phase,
+                        "failure_reason": result.failure_reason,
+                        "metrics": result.metrics,
+                    }
+                    for result in results
+                ]
+            },
+            failure_phase=first_failure.failure_phase
+            if first_failure
+            else None,
+            failure_reason=first_failure.failure_reason
+            if first_failure
+            else None,
+        )

mindcode/bench/scorers/exact.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from mindagent.core import AgentResult
+
+from ..schema import BenchmarkCase, CaseTrace, ScoreResult
+from .base import normalize_text
+
+
+class ExactAnswerScorer:
+    def score(
+        self,
+        case: BenchmarkCase,
+        run_result: AgentResult | None,
+        workspace: Path,
+        trace: CaseTrace,
+    ) -> ScoreResult:
+        expected = normalize_text(case.expected).casefold()
+        actual = normalize_text(
+            run_result.final_answer if run_result else None
+        ).casefold()
+        passed = bool(expected) and expected == actual
+        return ScoreResult(
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            metrics={"expected": case.expected, "actual": actual},
+            failure_phase=None if passed else "final_format",
+            failure_reason=None if passed else "answer_mismatch",
+        )

mindcode/bench/scorers/json_call.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from mindagent.core import AgentResult
+
+from ..schema import BenchmarkCase, CaseTrace, ScoreResult
+
+
+class JsonToolCallScorer:
+    def score(
+        self,
+        case: BenchmarkCase,
+        run_result: AgentResult | None,
+        workspace: Path,
+        trace: CaseTrace,
+    ) -> ScoreResult:
+        expected = case.expected if isinstance(case.expected, dict) else {}
+        expected_calls = expected.get("tool_calls", [])
+        actual_calls = _compact_calls(trace.tool_calls)
+        passed = actual_calls == expected_calls
+        return ScoreResult(
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            metrics={
+                "expected_tool_calls": expected_calls,
+                "actual_tool_calls": actual_calls,
+            },
+            failure_phase=None if passed else "tool_selection",
+            failure_reason=None if passed else "tool_calls_mismatch",
+        )
+
+
+def _compact_calls(calls: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    compacted: list[dict[str, Any]] = []
+    for call in calls:
+        compacted.append(
+            {
+                "name": call.get("name"),
+                "arguments": call.get("arguments", {}),
+            }
+        )
+    return compacted

mindcode/bench/scorers/swe.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from mindagent.core import AgentResult
+
+from ..predictions import extract_patch
+from ..schema import BenchmarkCase, CaseTrace, ScoreResult
+
+
+class PatchGeneratedScorer:
+    def score(
+        self,
+        case: BenchmarkCase,
+        run_result: AgentResult | None,
+        workspace: Path,
+        trace: CaseTrace,
+    ) -> ScoreResult:
+        patch = trace.workspace_diff or extract_patch(
+            run_result.final_answer if run_result else None
+        )
+        passed = bool(patch.strip())
+        return ScoreResult(
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            metrics={"patch_bytes": len(patch.encode("utf-8"))},
+            failure_phase=None if passed else "workspace_state",
+            failure_reason=None if passed else "empty_patch",
+        )

mindcode/bench/scorers/terminal.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from mindagent.core import AgentResult
+from mindagent.core.artifact import validate_artifact_path
+
+from ..schema import BenchmarkCase, CaseTrace, ScoreResult
+
+
+class FileStateScorer:
+    def score(
+        self,
+        case: BenchmarkCase,
+        run_result: AgentResult | None,
+        workspace: Path,
+        trace: CaseTrace,
+    ) -> ScoreResult:
+        expected = case.expected if isinstance(case.expected, dict) else {}
+        expected_files = expected.get("files", {})
+        failures: list[str] = []
+        for rel_path, expected_content in expected_files.items():
+            target = workspace / rel_path
+            try:
+                validate_artifact_path(workspace, target)
+            except ValueError:
+                failures.append(f"unsafe:{rel_path}")
+                continue
+            if not target.exists():
+                failures.append(f"missing:{rel_path}")
+                continue
+            actual = target.read_text(encoding="utf-8")
+            if actual != expected_content:
+                failures.append(f"content_mismatch:{rel_path}")
+
+        passed = not failures
+        return ScoreResult(
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            metrics={"file_failures": failures},
+            failure_phase=None if passed else "workspace_state",
+            failure_reason=None if passed else ",".join(failures),
+        )