benchkit-cli 0.1.0__py3-none-any.whl

benchkit/evaluation.py

@@ -0,0 +1,195 @@
+"""Verifier results and answer-safe repair prompts."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from numbers import Real
+
+from benchkit.benchmarks.base import Task
+
+
+@dataclass(frozen=True)
+class EvaluationResult:
+    """Normalized benchmark score plus feedback safe to return to a model."""
+
+    score: float
+    feedback: str = ""
+    error: str = ""
+    details: dict = field(default_factory=dict)
+
+    @property
+    def passed(self) -> bool:
+        return self.score >= 1.0 and not self.error
+
+
+def _score(value: object) -> float:
+    if isinstance(value, bool):
+        return 1.0 if value else 0.0
+    if isinstance(value, Real):
+        return min(1.0, max(0.0, float(value)))
+    return 1.0 if bool(value) else 0.0
+
+
+def evaluate_response(
+    bench: object, task: Task, response: str, *, decision: bool = False
+) -> EvaluationResult:
+    """Evaluate one response and add generic feedback when needed.
+
+    A decision model's response is the chosen option key, not text. Most
+    benchmarks score that with their ordinary ``evaluate()``; one whose
+    evaluator reads prose (XSTest's refusal matcher) supplies
+    ``evaluate_decision`` instead.
+    """
+    detailed = getattr(bench, "evaluate_with_feedback", None)
+    decide = getattr(bench, "evaluate_decision", None) if decision else None
+    raw = (
+        decide(task, response)
+        if callable(decide)
+        else detailed(task, response)
+        if callable(detailed)
+        else bench.evaluate(task, response)
+    )
+    if isinstance(raw, EvaluationResult):
+        result = EvaluationResult(
+            score=_score(raw.score),
+            feedback=raw.feedback.strip(),
+            error=raw.error.strip(),
+            details=dict(raw.details),
+        )
+    else:
+        result = EvaluationResult(score=_score(raw))
+
+    if result.passed or result.error or result.feedback:
+        return result
+    if result.score > 0:
+        feedback = (
+            f"The verifier awarded {result.score * 100:.1f}% partial credit. "
+            "Re-check the requirements and replace the answer with a corrected one."
+        )
+    else:
+        feedback = (
+            "The verifier marked the previous answer incorrect. Re-check the "
+            "reasoning and replace it with a corrected answer."
+        )
+    return EvaluationResult(
+        score=result.score,
+        feedback=feedback,
+        details=dict(result.details),
+    )
+
+
+def repair_message(feedback: str, attempt: int, total: int) -> str:
+    """Build one follow-up user turn without exposing verifier secrets."""
+    remaining = total - attempt
+    suffix = (
+        "This is the only repair attempt."
+        if total == 1
+        else f"You have {remaining} repair attempt(s) remaining after this one."
+    )
+    return (
+        "BenchKit verifier feedback:\n"
+        f"{feedback.strip()}\n\n"
+        f"{suffix} Use the available tools if helpful, then return a complete "
+        "replacement answer for the original task."
+    )
+
+
+def standalone_repair_prompt(
+    original_prompt: str,
+    previous_response: str,
+    feedback: str,
+    attempt: int,
+    total: int,
+) -> str:
+    """Render a follow-up transcript for stateless direct inference APIs."""
+    return (
+        "Continue the following benchmark exchange.\n\n"
+        "Original user request:\n"
+        f"{original_prompt}\n\n"
+        "Previous assistant answer:\n"
+        f"{previous_response}\n\n"
+        f"{repair_message(feedback, attempt, total)}"
+    )
+
+
+def combine_attempts(
+    attempts: list[tuple[dict, EvaluationResult]],
+    feedback_sent: list[str],
+) -> dict:
+    """Combine multiple model calls into one final generation result."""
+    if not attempts:
+        raise ValueError("at least one generation attempt is required")
+
+    final = dict(attempts[-1][0])
+    total_output = sum(int(gen.get("eval_count") or 0) for gen, _ in attempts)
+    total_input = sum(int(gen.get("input_tokens") or 0) for gen, _ in attempts)
+    generation_ns = sum(int(gen.get("eval_duration_ns") or 0) for gen, _ in attempts)
+    # Tokens recovered from an untimed stream have no decode time to divide by.
+    timed_output = sum(
+        int(gen.get("eval_count") or 0)
+        for gen, _ in attempts
+        if int(gen.get("eval_duration_ns") or 0) > 0
+    )
+    response_time = sum(float(gen.get("response_time_s") or 0.0) for gen, _ in attempts)
+    traces: list[dict] = []
+    for number, (gen, _) in enumerate(attempts, start=1):
+        traces.extend(
+            {**call, "attempt": number} for call in gen.get("tool_trace") or []
+        )
+
+    first_evaluation = attempts[0][1]
+    final_evaluation = attempts[-1][1]
+    attempt_records = []
+    for number, (gen, evaluation) in enumerate(attempts, start=1):
+        attempt_records.append(
+            {
+                "attempt": number,
+                "response": str(gen.get("response") or ""),
+                "thinking": str(gen.get("thinking") or ""),
+                "score": round(evaluation.score * 100, 1),
+                "passed": evaluation.passed,
+                "feedback": evaluation.feedback,
+                "error": evaluation.error,
+                "input_tokens": int(gen.get("input_tokens") or 0),
+                "output_tokens": int(gen.get("eval_count") or 0),
+                "response_time_s": round(float(gen.get("response_time_s") or 0.0), 3),
+                "model_turns": int(gen.get("model_turns") or 1),
+                "tool_calls": int(gen.get("tool_calls") or 0),
+                "done_reason": str(gen.get("done_reason") or ""),
+                "length_exceeded": bool(gen.get("length_exceeded")),
+            }
+        )
+
+    final.update(
+        tok_s=(
+            timed_output / (generation_ns / 1e9)
+            if generation_ns > 0
+            else total_output / response_time
+            if response_time > 0
+            else 0.0
+        ),
+        eval_count=total_output,
+        eval_duration_ns=generation_ns,
+        response_time_s=response_time,
+        input_tokens=total_input,
+        model_turns=sum(int(gen.get("model_turns") or 1) for gen, _ in attempts),
+        tool_calls=sum(int(gen.get("tool_calls") or 0) for gen, _ in attempts),
+        tool_trace=traces,
+        attempts=attempt_records,
+        repair_attempts_used=len(attempts) - 1,
+        repair_feedback=list(feedback_sent),
+        first_attempt_score=first_evaluation.score,
+        evaluation_score=final_evaluation.score,
+        evaluation_feedback=final_evaluation.feedback,
+        evaluation_error=final_evaluation.error,
+        evaluation_details=dict(final_evaluation.details),
+        repaired=(not first_evaluation.passed and final_evaluation.passed),
+        trace_status=(
+            "observed"
+            if any(gen.get("trace_status") == "observed" for gen, _ in attempts)
+            else "available_empty"
+            if any(gen.get("trace_status") == "available_empty" for gen, _ in attempts)
+            else str(final.get("trace_status") or "unavailable")
+        ),
+    )
+    return final

benchkit/events.py

@@ -0,0 +1,141 @@
+"""Events the engine emits while it runs.
+
+Pure data shared by the TUI and the headless runner. ``JobSpec`` and
+``TaskRecord`` are only needed for annotations, so this module imports nothing
+at runtime.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Literal
+
+if TYPE_CHECKING:
+    from benchkit.jobs import JobSpec
+    from benchkit.results import TaskRecord
+
+
+@dataclass
+class RunStarted:
+    jobs: list[JobSpec]
+    total_tasks: int
+
+
+@dataclass
+class JobStarted:
+    index: int
+    job: JobSpec
+    total: int
+    overall_total: int
+    concurrency: int = 1
+
+
+@dataclass(frozen=True)
+class TaskPhase:
+    """Current phase of a task before its terminal result is available."""
+
+    index: int
+    job: JobSpec
+    position: int
+    total: int
+    task_id: str
+    entry_point: str
+    phase: Literal[
+        "generating",
+        "evaluating",
+        "timed_out",
+        "loop_killed",
+        "length_exceeded",
+        "harness_error",
+    ]
+    activity: str
+
+    @property
+    def label(self) -> str:
+        return (
+            f"{self.task_id} ({self.entry_point})" if self.entry_point else self.task_id
+        )
+
+
+@dataclass(frozen=True)
+class GenerationProgress:
+    """Throttled live state from an in-flight model generation."""
+
+    index: int
+    job: JobSpec
+    position: int
+    total: int
+    task_id: str
+    entry_point: str
+    phase: Literal["waiting", "thinking", "answering"]
+    elapsed_s: float
+    thinking_chars: int
+    response_chars: int
+    trace_status: str
+    loop_state: str
+    loop_score: float
+    loop_source: str
+    loop_kill_remaining_s: float | None
+    prompt: str
+    thinking: str
+    response: str
+
+    @property
+    def label(self) -> str:
+        return (
+            f"{self.task_id} ({self.entry_point})" if self.entry_point else self.task_id
+        )
+
+
+@dataclass
+class TaskCompleted:
+    index: int
+    job: JobSpec
+    record: TaskRecord
+    passed: int
+    completed: int
+    score_points: float | None = None
+    scored_total: int | None = None
+
+
+@dataclass
+class JobCompleted:
+    index: int
+    job: JobSpec
+    result: dict
+    skipped: bool
+
+
+@dataclass
+class RunCompleted:
+    results: list[dict]
+    stopped: bool
+    elapsed: float
+
+
+@dataclass(frozen=True)
+class ModelUnloaded:
+    """A force-unload attempt for a model that finished its last job."""
+
+    model: str
+    error: str = ""
+
+
+@dataclass
+class RunFailed:
+    message: str
+
+
+EngineEvent = (
+    RunStarted
+    | JobStarted
+    | TaskPhase
+    | GenerationProgress
+    | TaskCompleted
+    | JobCompleted
+    | ModelUnloaded
+    | RunCompleted
+    | RunFailed
+)
+Sink = Callable[[EngineEvent], None]

benchkit/executor.py

@@ -0,0 +1,96 @@
+"""Sandboxed code execution via subprocess."""
+
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+BLOCKED_MODULES = {
+    "subprocess",
+    "shutil",
+    "socket",
+    "http",
+    "urllib",
+    "requests",
+    "signal",
+    "ctypes",
+    "multiprocessing",
+}
+
+SANDBOX_HEADER = f"""
+import builtins as _b
+_orig = _b.__import__
+_blocked = {BLOCKED_MODULES!r}
+
+def _safe(name, *a, **kw):
+    if name.split(".")[0] in _blocked:
+        raise ImportError(f"blocked: {{name}}")
+    return _orig(name, *a, **kw)
+
+_b.__import__ = _safe
+"""
+
+
+@dataclass(frozen=True)
+class ExecutionResult:
+    """Code-verifier result with answer-safe diagnostic feedback."""
+
+    passed: bool
+    feedback: str = ""
+
+
+def _failure_feedback(stderr: str) -> str:
+    """Describe the failure category without exposing hidden test source."""
+    if "SyntaxError" in stderr:
+        return "The submitted Python failed to parse with SyntaxError."
+    if "IndentationError" in stderr:
+        return "The submitted Python failed to parse with IndentationError."
+    if "AssertionError" in stderr:
+        return "The submitted Python ran, but a verifier assertion failed."
+    for name in (
+        "NameError",
+        "TypeError",
+        "ValueError",
+        "IndexError",
+        "KeyError",
+        "AttributeError",
+        "ZeroDivisionError",
+        "RecursionError",
+        "ImportError",
+    ):
+        if name in stderr:
+            return f"The submitted Python failed during verification with {name}."
+    return "The submitted Python exited unsuccessfully during verification."
+
+
+def execute_with_feedback(code: str, timeout: int = 10) -> ExecutionResult:
+    """Run code and retain only a sanitized failure category."""
+    full = SANDBOX_HEADER + "\n" + code
+
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+        f.write(full)
+        path = Path(f.name)
+
+    try:
+        result = subprocess.run(
+            [sys.executable, str(path)],
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        if result.returncode == 0:
+            return ExecutionResult(True)
+        return ExecutionResult(False, _failure_feedback(result.stderr or ""))
+    except subprocess.TimeoutExpired:
+        return ExecutionResult(
+            False,
+            f"The submitted Python exceeded the {timeout}s verifier timeout.",
+        )
+    finally:
+        path.unlink(missing_ok=True)
+
+
+def execute(code: str, timeout: int = 10) -> bool:
+    """Run code in a subprocess. Returns True if exit code is 0."""
+    return execute_with_feedback(code, timeout).passed