self-evolve-framework 1.4.0 → 1.6.0

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/spreadsheetbench/rollout.py

@@ -0,0 +1,979 @@
+"""SpreadsheetBench rollout — codegen & ReAct batch execution.
+
+Provides:
+  - process_one_codegen(): single/multi-round code generation (no tool-call)
+  - run_spreadsheet_batch_codegen(): batch wrapper for codegen
+  - process_one(): ReAct agent with tool-call (legacy)
+  - run_spreadsheet_batch(): batch wrapper for ReAct (legacy)
+  - load_items(): load benchmark .json/.jsonl files
+"""
+from __future__ import annotations
+
+import glob as _glob
+import json
+import os
+import shutil
+import tempfile
+import time
+import traceback
+from concurrent.futures import (
+    FIRST_COMPLETED,
+    ThreadPoolExecutor,
+    wait,
+    TimeoutError as FuturesTimeoutError,
+)
+
+import openpyxl
+
+from skillopt.envs.spreadsheetbench.react_agent import run_react
+from skillopt.envs.spreadsheetbench.evaluator import (
+    evaluate, _generate_cell_names, _compare_cell_value,
+)
+from skillopt.envs.spreadsheetbench.executor import run_generated_code
+
+
+# ── Data loading ─────────────────────────────────────────────────────────────
+
+
+def load_items(path: str) -> list[dict]:
+    """Load a benchmark file. Supports both .jsonl and .json (list of dicts)."""
+    if path.endswith(".json"):
+        with open(path) as f:
+            data = json.load(f)
+        if isinstance(data, dict):
+            data = data.get("data") or list(data.values())
+        return list(data)
+    items = []
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                items.append(json.loads(line))
+    return items
+
+
+# ── Test case discovery ──────────────────────────────────────────────────────
+
+
+def _find_test_cases(task_dir: str) -> list[tuple[str, str, str]]:
+    """Return [(case_no, input_path, answer_path), ...] sorted by case_no.
+
+    Supports naming conventions used by SpreadsheetBench releases:
+      * ``{no}_{id}_input.xlsx``  + ``{no}_{id}_answer.xlsx``  (original)
+      * ``{no}_{id}_init.xlsx``   + ``{no}_{id}_golden.xlsx``  (verified_400)
+      * ``initial.xlsx``          + ``golden.xlsx``             (verified_400, no prefix)
+    """
+    cases: list[tuple[str, str, str]] = []
+    inputs = sorted(_glob.glob(os.path.join(task_dir, "*_input.xlsx")))
+    for ip in inputs:
+        no = os.path.basename(ip).split("_", 1)[0]
+        ap = ip.replace("_input.xlsx", "_answer.xlsx")
+        if os.path.exists(ap):
+            cases.append((no, ip, ap))
+    inits = sorted(_glob.glob(os.path.join(task_dir, "*_init.xlsx")))
+    for ip in inits:
+        no = os.path.basename(ip).split("_", 1)[0]
+        ap = ip.replace("_init.xlsx", "_golden.xlsx")
+        if os.path.exists(ap):
+            cases.append((no, ip, ap))
+
+    # Fallback: bare initial.xlsx + golden.xlsx (no numbered prefix)
+    if not cases:
+        bare_init = os.path.join(task_dir, "initial.xlsx")
+        bare_gold = os.path.join(task_dir, "golden.xlsx")
+        if os.path.exists(bare_init) and os.path.exists(bare_gold):
+            cases.append(("1", bare_init, bare_gold))
+
+    return cases
+
+
+# ── Auto-verify helper ──────────────────────────────────────────────────────
+
+# The official SpreadsheetBench evaluator never serialises cells to text — it
+# compares in memory and returns only a pass/fail bool. The per-cell report
+# below is a repo-local training aid (fed back to the model on retry and saved
+# into the trajectory for reflection). On most tasks the answer range is a
+# handful of cells, so the full report is tiny. But a few tasks have answer
+# ranges spanning tens of thousands of cells (e.g. 80-42 =
+# 'Consolidate_ALL'!A2:L8000 ≈ 96k cells); dumping every cell explodes the
+# report to several MB, floods the model's context and bloats conversation
+# files. We therefore apply the same head+tail character truncation the rest of
+# the codebase uses for oversized trajectory text (cf. reflect.py / slow_update.py
+# `text[:half] + "...[truncated]...\n" + text[-half:]`): keep the first and last
+# `_MAX_REPORT_CHARS // 2` chars so both the leading and trailing wrong cells
+# stay visible. Small reports are unchanged.
+_MAX_REPORT_CHARS = 12000      # head+tail char budget (~6000 head + 6000 tail)
+
+
+def _auto_verify_output(
+    pred_path: str,
+    gold_path: str,
+    answer_position: str,
+) -> str:
+    """Reopen the predicted xlsx and compare cells at answer_position with gold.
+
+    Returns a human-readable verification report that can be appended to the
+    trajectory so the error analyst can see exactly what went wrong (e.g.
+    ``cell A1: got=None, expected=420``). Oversized reports are head+tail
+    truncated to `_MAX_REPORT_CHARS` chars, matching the rest of the codebase.
+    """
+    if not os.path.exists(pred_path):
+        return "Verification: output file does not exist."
+    try:
+        wb_pred = openpyxl.load_workbook(pred_path, data_only=True)
+        wb_gold = openpyxl.load_workbook(gold_path, data_only=True)
+    except Exception as e:
+        return f"Verification: could not open workbooks: {e}"
+
+    lines = ["## Output Verification"]
+    try:
+        for scr in (answer_position or "").split(","):
+            scr = scr.strip()
+            if not scr:
+                continue
+            if "!" in scr:
+                sheet_name, cell_range = scr.split("!", 1)
+                sheet_name = sheet_name.strip().strip("'\"")
+            else:
+                sheet_name = wb_gold.sheetnames[0]
+                cell_range = scr
+            cell_range = cell_range.strip().strip("'\"")
+
+            cell_names = _generate_cell_names(cell_range)
+            ws_pred = wb_pred[sheet_name] if sheet_name in wb_pred.sheetnames else None
+            ws_gold = wb_gold[sheet_name] if sheet_name in wb_gold.sheetnames else None
+
+            if ws_pred is None:
+                lines.append(f"  Sheet '{sheet_name}' NOT FOUND in output.")
+                continue
+
+            n_empty_correct = 0   # empty-on-both correct cells collapsed to a count
+            for cn in cell_names:
+                gv = ws_gold[cn].value if ws_gold else "N/A"
+                pv = ws_pred[cn].value
+                # Use the official cell comparator so this report's ✓/✗ agrees
+                # with the real scorer (evaluate). repr() equality would wrongly
+                # flag e.g. 5 vs 5.0 or None vs "" as mismatches and mislead the
+                # model into "fixing" cells that already pass scoring.
+                ok_cell = ws_gold is not None and _compare_cell_value(gv, pv)
+                # Collapse only cells that are correct AND empty on both sides
+                # (got=None, expected=None ✓): pure noise. Every other cell —
+                # including non-empty correct cells — is listed in full; the
+                # final head+tail char cap keeps the report bounded.
+                if ok_cell and gv in (None, "") and pv in (None, ""):
+                    n_empty_correct += 1
+                    continue
+                match = "✓" if ok_cell else "✗"
+                lines.append(f"  {sheet_name}!{cn}: got={pv!r}, expected={gv!r} {match}")
+            if n_empty_correct:
+                lines.append(
+                    f"  (+{n_empty_correct} empty cells correct, omitted)"
+                )
+
+        # Also check if any cells in the output contain formula strings
+        formula_cells = []
+        for sn in wb_pred.sheetnames:
+            ws = wb_pred[sn]
+            for row in ws.iter_rows(max_row=min(ws.max_row, 200), values_only=False):
+                for cell in row:
+                    if isinstance(cell.value, str) and cell.value.startswith("="):
+                        formula_cells.append(f"{sn}!{cell.coordinate}={cell.value}")
+                        if len(formula_cells) >= 10:
+                            break
+                if len(formula_cells) >= 10:
+                    break
+            if len(formula_cells) >= 10:
+                break
+        if formula_cells:
+            lines.append(f"\n  WARNING: {len(formula_cells)} cells contain Excel formulas (openpyxl cannot evaluate them):")
+            for fc in formula_cells[:5]:
+                lines.append(f"    {fc}")
+            if len(formula_cells) > 5:
+                lines.append(f"    ... and {len(formula_cells) - 5} more")
+    finally:
+        wb_pred.close()
+        wb_gold.close()
+
+    report = "\n".join(lines)
+    # Head+tail truncation, matching reflect.py / slow_update.py: keep the first
+    # and last half so both leading and trailing wrong cells remain visible.
+    if len(report) > _MAX_REPORT_CHARS:
+        half = _MAX_REPORT_CHARS // 2
+        report = (
+            report[:half]
+            + f"\n  ...[verification report truncated, {len(report)} chars total]...\n"
+            + report[-half:]
+        )
+    return report
+
+
+# ── Per-task worker ──────────────────────────────────────────────────────────
+
+
+def process_one(
+    item: dict,
+    data_root: str,
+    out_root: str,
+    skill_content: str,
+    max_turns: int,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+    max_completion_tokens: int = 16384,
+) -> dict:
+    """Run the ReAct agent on a single SpreadsheetBench task.
+
+    Returns a result dict compatible with ``compute_score()``.
+    """
+    task_id = str(item["id"])
+    instruction = item["instruction"]
+    instruction_type = item.get("instruction_type", "")
+    answer_position = item.get("answer_position", "")
+    answer_sheet = item.get("answer_sheet", "")
+    if answer_position and answer_sheet and "!" not in answer_position:
+        answer_position_eval = f"{answer_sheet}!{answer_position}"
+    else:
+        answer_position_eval = answer_position
+
+    # Determine task_type from instruction_type
+    itype_lower = (instruction_type or "").lower()
+    if "cell" in itype_lower:
+        task_type = "cell_level"
+    elif "sheet" in itype_lower:
+        task_type = "sheet_level"
+    else:
+        task_type = "other"
+
+    sp = item.get("spreadsheet_path", f"spreadsheet/{task_id}")
+    task_dir = sp if os.path.isabs(sp) else os.path.join(data_root, sp)
+
+    result = {
+        "id": task_id,
+        "ok": False,
+        "instruction_type": instruction_type,
+        "task_type": task_type,
+        "task_description": instruction,
+        "phase": "setup",
+        "fail_reason": "",
+        "agent_ok": False,
+        "exec_ok": False,
+        "n_cases": 0,
+        "n_exec_pass": 0,
+        "n_pass": 0,
+        "soft": 0.0,
+        "hard": 0,
+        "n_turns": 0,
+        "cases": [],
+        "error": "",
+    }
+
+    try:
+        cases = _find_test_cases(task_dir)
+        result["n_cases"] = len(cases)
+        if not cases:
+            result["fail_reason"] = "no-test-cases"
+            return result
+
+        task_out_dir = os.path.join(out_root, "predictions", task_id)
+        os.makedirs(task_out_dir, exist_ok=True)
+
+        no1, ip1, _ = cases[0]
+        pred_path_1 = os.path.join(task_out_dir, f"{no1}_pred.xlsx")
+        target_prompt_parts = [
+            f"# Instruction\n{instruction}",
+            f"# Input file\n{ip1}",
+            f"# Output file\n{pred_path_1}",
+        ]
+        if instruction_type:
+            target_prompt_parts.append(f"# Instruction type\n{instruction_type}")
+        if answer_position_eval:
+            target_prompt_parts.append(f"# Answer position\n{answer_position_eval}")
+        if diagnostic_trace_context.strip():
+            target_prompt_parts.insert(
+                0,
+                "# Previous Codex Trace Snapshot\n"
+                "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n"
+                f"{diagnostic_trace_context.strip()}",
+            )
+        if diagnostic_mode and diagnostic_instruction.strip():
+            target_prompt_parts.append(f"# Training readout\n{diagnostic_instruction.strip()}")
+        target_user_prompt = "\n\n".join(target_prompt_parts)
+        try:
+            from skillopt.envs.spreadsheetbench.react_agent import _build_system
+            target_system_prompt = _build_system(skill_content)
+        except Exception:
+            target_system_prompt = ""
+        if target_system_prompt:
+            with open(os.path.join(task_out_dir, "target_system_prompt.txt"), "w") as f:
+                f.write(target_system_prompt)
+            result["target_system_prompt"] = target_system_prompt
+        with open(os.path.join(task_out_dir, "target_user_prompt.txt"), "w") as f:
+            f.write(target_user_prompt)
+        result["target_user_prompt"] = target_user_prompt
+
+        # ── Stage 1: run ReAct agent on test case 1 ─────────────────────
+        result["phase"] = "agent"
+
+        work_dir = tempfile.mkdtemp(prefix=f"react_{task_id}_")
+        try:
+            # Copy input so agent works in an isolated directory
+            work_input = os.path.join(work_dir, os.path.basename(ip1))
+            shutil.copy2(ip1, work_input)
+
+            agent_result = run_react(
+                instruction=instruction,
+                input_path=work_input,
+                output_path=pred_path_1,
+                work_dir=work_dir,
+                instruction_type=instruction_type,
+                answer_position=answer_position_eval,
+                skill_content=skill_content,
+                max_turns=max_turns,
+                max_output_tokens=max_completion_tokens,
+                diagnostic_mode=diagnostic_mode,
+                diagnostic_instruction=diagnostic_instruction,
+                diagnostic_trace_context=diagnostic_trace_context,
+            )
+            result["n_turns"] = agent_result.get("n_turns", 0)
+            if agent_result.get("target_system_prompt"):
+                with open(os.path.join(task_out_dir, "target_system_prompt.txt"), "w") as f:
+                    f.write(agent_result["target_system_prompt"])
+                result["target_system_prompt"] = agent_result["target_system_prompt"]
+            if agent_result.get("target_user_prompt"):
+                with open(os.path.join(task_out_dir, "target_user_prompt.txt"), "w") as f:
+                    f.write(agent_result["target_user_prompt"])
+                result["target_user_prompt"] = agent_result["target_user_prompt"]
+
+            # Save conversation log
+            with open(os.path.join(task_out_dir, "conversation.json"), "w") as f:
+                json.dump(
+                    agent_result.get("conversation", []),
+                    f, ensure_ascii=False, indent=2,
+                )
+
+            # Copy solution.py if the agent wrote one
+            solution_src = os.path.join(work_dir, "solution.py")
+            solution_dst = os.path.join(task_out_dir, "solution.py")
+            if os.path.exists(solution_src):
+                shutil.copy2(solution_src, solution_dst)
+
+        except Exception as e:
+            result["fail_reason"] = f"agent-error: {type(e).__name__}: {e}"
+            result["error"] = traceback.format_exc()
+            return result
+        finally:
+            shutil.rmtree(work_dir, ignore_errors=True)
+
+        result["agent_ok"] = True
+
+        # ── Stage 2: evaluate all test cases ─────────────────────────────
+        result["phase"] = "eval"
+        solution_path = os.path.join(task_out_dir, "solution.py")
+        all_exec = True
+
+        for i, (no, ip, ap) in enumerate(cases):
+            pred_path = os.path.join(task_out_dir, f"{no}_pred.xlsx")
+
+            if i > 0:
+                # Re-apply solution.py to subsequent test cases
+                if not os.path.exists(solution_path):
+                    all_exec = False
+                    result["cases"].append(
+                        {"no": no, "stage": "exec", "ok": False, "error": "no-solution-py"}
+                    )
+                    if not result["fail_reason"]:
+                        result["fail_reason"] = "no-solution-py-for-other-cases"
+                    continue
+
+                with open(solution_path) as f:
+                    code = f.read()
+
+                # Prepend new INPUT_PATH / OUTPUT_PATH
+                preamble = (
+                    f"INPUT_PATH  = {ip!r}\n"
+                    f"OUTPUT_PATH = {pred_path!r}\n"
+                )
+                full_code = preamble + code
+
+                ok_exec, err = run_generated_code(full_code, ip, pred_path)
+                if not ok_exec:
+                    all_exec = False
+                    result["cases"].append(
+                        {"no": no, "stage": "exec", "ok": False, "error": err[:500]}
+                    )
+                    if not result["fail_reason"]:
+                        tail = err.strip().splitlines()[-1][:200] if err.strip() else "unknown"
+                        result["fail_reason"] = f"exec-error: {tail}"
+                    continue
+
+            # ── Evaluate ─────────────────────────────────────────────────
+            if not os.path.exists(pred_path):
+                all_exec = False
+                result["cases"].append(
+                    {"no": no, "stage": "exec", "ok": False, "error": "output-not-found"}
+                )
+                if not result["fail_reason"]:
+                    result["fail_reason"] = "output-not-found"
+                continue
+
+            result["n_exec_pass"] += 1
+            try:
+                ev = evaluate(pred_path, ap, instruction_type, answer_position_eval)
+            except Exception as e:  # noqa: BLE001
+                ev = {"ok": False, "reason": f"eval-exception: {type(e).__name__}: {e}"}
+
+            if ev["ok"]:
+                result["n_pass"] += 1
+            else:
+                if not result["fail_reason"]:
+                    result["fail_reason"] = f"eval-mismatch: {ev['reason'][:200]}"
+            result["cases"].append(
+                {"no": no, "stage": "eval", "ok": ev["ok"], "reason": ev.get("reason", "")}
+            )
+
+        result["exec_ok"] = all_exec
+        n_cases = result["n_cases"]
+        n_pass = result["n_pass"]
+        result["soft"] = (n_pass / n_cases) if n_cases else 0.0
+        result["hard"] = 1 if (n_cases > 0 and n_pass == n_cases) else 0
+        result["ok"] = bool(result["hard"])
+        if result["ok"]:
+            result["fail_reason"] = ""
+        return result
+
+    except Exception as e:  # noqa: BLE001
+        result["fail_reason"] = f"unexpected: {type(e).__name__}: {e}"
+        result["error"] = traceback.format_exc()
+        return result
+
+
+# ── Batch runner ─────────────────────────────────────────────────────────────
+
+
+def run_spreadsheet_batch(
+    items: list[dict],
+    data_root: str,
+    out_root: str,
+    skill_content: str,
+    max_turns: int = 30,
+    max_completion_tokens: int = 16384,
+    max_api_workers: int = 64,
+    task_timeout: int = 600,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context_by_id: dict[str, str] | None = None,
+) -> list[dict]:
+    """Run the ReAct agent on all items with ThreadPoolExecutor.
+
+    Returns list of result dicts compatible with ``compute_score()``.
+    """
+    os.makedirs(out_root, exist_ok=True)
+
+    # Check for already-done items (resume support)
+    results_path = os.path.join(out_root, "results.jsonl")
+    done_ids: set[str] = set()
+    existing: list[dict] = []
+    if os.path.exists(results_path):
+        with open(results_path) as f:
+            for line in f:
+                try:
+                    r = json.loads(line)
+                    done_ids.add(str(r["id"]))
+                    existing.append(r)
+                except Exception:
+                    pass
+
+    pending = [it for it in items if str(it["id"]) not in done_ids]
+    print(
+        f"  [spreadsheet rollout] total={len(items)} done={len(done_ids)} "
+        f"pending={len(pending)} workers={max_api_workers} task_timeout={task_timeout}s"
+    )
+
+    if not pending:
+        return existing
+
+    t0 = time.time()
+    results = list(existing)
+    started_at: dict[str, float] = {}
+
+    def _timeout_result(item: dict) -> dict:
+        return {
+            "id": str(item["id"]),
+            "ok": False,
+            "phase": "timeout",
+            "fail_reason": f"task-timeout-{task_timeout}s",
+            "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0,
+            "n_turns": 0, "cases": [], "error": "timeout",
+        }
+
+    def _error_result(item: dict, exc: Exception) -> dict:
+        return {
+            "id": str(item["id"]),
+            "ok": False,
+            "phase": "error",
+            "fail_reason": f"unexpected: {type(exc).__name__}: {exc}",
+            "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0,
+            "n_turns": 0, "cases": [], "error": str(exc),
+        }
+
+    def _run_one(it: dict) -> dict:
+        started_at[str(it["id"])] = time.time()
+        return process_one(
+            it,
+            data_root,
+            out_root,
+            skill_content,
+            max_turns,
+            diagnostic_mode,
+            diagnostic_instruction,
+            (diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""),
+            max_completion_tokens,
+        )
+
+    ex = ThreadPoolExecutor(max_workers=max_api_workers)
+    try:
+        futs = {ex.submit(_run_one, it): it for it in pending}
+        pending_futs = set(futs)
+        finished = 0
+        while pending_futs:
+            done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED)
+            now = time.time()
+            timed_out = [
+                fut for fut in pending_futs - done
+                if str(futs[fut]["id"]) in started_at
+                and now - started_at[str(futs[fut]["id"])] >= task_timeout
+            ]
+            for fut in done:
+                pending_futs.remove(fut)
+                item = futs[fut]
+                try:
+                    res = fut.result()
+                except FuturesTimeoutError:
+                    res = _timeout_result(item)
+                except Exception as e:  # noqa: BLE001
+                    res = _error_result(item, e)
+                results.append(res)
+                finished += 1
+                status = "PASS" if res.get("hard") else ("TIMEOUT" if res.get("phase") == "timeout" else "FAIL")
+                dt = time.time() - t0
+                print(
+                    f"    {finished}/{len(pending)} id={res['id']:<10} {status}  "
+                    f"turns={res.get('n_turns', 0):<3} "
+                    f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)}  "
+                    f"dt={dt:.0f}s"
+                )
+            for fut in timed_out:
+                pending_futs.remove(fut)
+                res = _timeout_result(futs[fut])
+                results.append(res)
+                finished += 1
+                status = "TIMEOUT"
+                dt = time.time() - t0
+                print(
+                    f"    {finished}/{len(pending)} id={res['id']:<10} {status}  "
+                    f"turns={res.get('n_turns', 0):<3} "
+                    f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)}  "
+                    f"dt={dt:.0f}s"
+                )
+    finally:
+        ex.shutdown(wait=False, cancel_futures=True)
+
+    return results
+
+
+# ── Codegen per-task worker (no tool-call) ──────────────────────────────────
+
+
+def process_one_codegen(
+    item: dict,
+    data_root: str,
+    out_root: str,
+    skill_content: str,
+    mode: str = "single",
+    max_turns: int = 5,
+    max_completion_tokens: int = 16384,
+    task_timeout: int = 600,
+    use_eval_feedback: bool = False,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+) -> dict:
+    """Run codegen agent (single or multi-round) on one SpreadsheetBench task.
+
+    This matches the official evaluation setting: LLM generates a Python code
+    block, no function-calling / tool-use.
+    """
+    from skillopt.envs.spreadsheetbench.codegen_agent import run_single, run_multi
+
+    task_id = str(item["id"])
+    instruction = item["instruction"]
+    instruction_type = item.get("instruction_type", "")
+    answer_position = item.get("answer_position", "")
+    answer_sheet = item.get("answer_sheet", "")
+    if answer_position and answer_sheet and "!" not in answer_position:
+        answer_position_eval = f"{answer_sheet}!{answer_position}"
+    else:
+        answer_position_eval = answer_position
+
+    itype_lower = (instruction_type or "").lower()
+    if "cell" in itype_lower:
+        task_type = "cell_level"
+    elif "sheet" in itype_lower:
+        task_type = "sheet_level"
+    else:
+        task_type = "other"
+
+    sp = item.get("spreadsheet_path", f"spreadsheet/{task_id}")
+    task_dir = sp if os.path.isabs(sp) else os.path.join(data_root, sp)
+
+    result = {
+        "id": task_id,
+        "ok": False,
+        "instruction_type": instruction_type,
+        "task_type": task_type,
+        "task_description": instruction,
+        "phase": "setup",
+        "fail_reason": "",
+        "llm_ok": False,
+        "code_ok": False,
+        "exec_ok": False,
+        "n_cases": 0,
+        "n_exec_pass": 0,
+        "n_pass": 0,
+        "soft": 0.0,
+        "hard": 0,
+        "n_turns": 0,
+        "cases": [],
+        "error": "",
+    }
+
+    try:
+        cases = _find_test_cases(task_dir)
+        result["n_cases"] = len(cases)
+        if not cases:
+            result["fail_reason"] = "no-test-cases"
+            return result
+
+        task_out_dir = os.path.join(out_root, "predictions", task_id)
+        os.makedirs(task_out_dir, exist_ok=True)
+
+        # ── Save context for Optimizer (Reflect stage) ──────────────────
+        from skillopt.envs.spreadsheetbench.codegen_agent import (
+            _preview_workbook, _build_system, _build_user,
+        )
+        first_input_for_preview = cases[0][1]
+        try:
+            preview_text = _preview_workbook(first_input_for_preview)
+        except Exception:
+            preview_text = "(preview failed)"
+        target_system = _build_system(skill_content)
+        target_user = _build_user(
+            instruction,
+            first_input_for_preview,
+            instruction_type,
+            answer_position_eval,
+            diagnostic_mode=diagnostic_mode,
+            diagnostic_instruction=diagnostic_instruction,
+            diagnostic_trace_context=diagnostic_trace_context,
+        )
+
+        with open(os.path.join(task_out_dir, "spreadsheet_preview.txt"), "w") as f:
+            f.write(preview_text)
+        with open(os.path.join(task_out_dir, "target_system_prompt.txt"), "w") as f:
+            f.write(target_system)
+        with open(os.path.join(task_out_dir, "target_user_prompt.txt"), "w") as f:
+            f.write(target_user)
+
+        result["spreadsheet_preview"] = preview_text
+        result["target_system_prompt"] = target_system
+        result["target_user_prompt"] = target_user
+
+        # ── LLM phase ──────────────────────────────────────────────────
+        result["phase"] = "llm"
+        first_input = cases[0][1]
+        first_gold = cases[0][2]
+        first_pred = os.path.join(task_out_dir, f"{cases[0][0]}_pred.xlsx")
+
+        try:
+            if mode == "multi":
+                agent_result = run_multi(
+                    instruction=instruction,
+                    input_xlsx=first_input,
+                    output_path=first_pred,
+                    instruction_type=instruction_type,
+                    answer_position=answer_position_eval,
+                    skill_content=skill_content,
+                    max_turns=max_turns,
+                    max_output_tokens=max_completion_tokens,
+                    task_timeout=task_timeout,
+                    gold_path=first_gold if use_eval_feedback else "",
+                    diagnostic_mode=diagnostic_mode,
+                    diagnostic_instruction=diagnostic_instruction,
+                    diagnostic_trace_context=diagnostic_trace_context,
+                )
+            else:
+                agent_result = run_single(
+                    instruction=instruction,
+                    input_xlsx=first_input,
+                    output_path=first_pred,
+                    instruction_type=instruction_type,
+                    answer_position=answer_position_eval,
+                    skill_content=skill_content,
+                    max_output_tokens=max_completion_tokens,
+                    task_timeout=task_timeout,
+                    diagnostic_mode=diagnostic_mode,
+                    diagnostic_instruction=diagnostic_instruction,
+                    diagnostic_trace_context=diagnostic_trace_context,
+                )
+        except Exception as e:  # noqa: BLE001
+            result["fail_reason"] = f"llm-call-failed: {type(e).__name__}: {e}"
+            result["error"] = traceback.format_exc()
+            return result
+
+        result["llm_ok"] = True
+        result["n_turns"] = agent_result.get("n_turns", 1)
+        code = agent_result.get("code", "")
+        raw = agent_result.get("raw", "")
+
+        # Save artifacts
+        with open(os.path.join(task_out_dir, "code.py"), "w") as f:
+            f.write(code)
+        with open(os.path.join(task_out_dir, "raw.txt"), "w") as f:
+            f.write(raw)
+        if agent_result.get("conversation"):
+            with open(os.path.join(task_out_dir, "conversation.json"), "w") as f:
+                json.dump(agent_result["conversation"], f, ensure_ascii=False, indent=2)
+
+        if not code.strip():
+            result["phase"] = "extract"
+            result["fail_reason"] = "empty-code-block"
+            return result
+        result["code_ok"] = True
+
+        # ── Exec + eval per test case ──────────────────────────────────
+        result["phase"] = "exec"
+        all_exec = True
+        # Collect enrichment info for the conversation/trajectory
+        enrichment_parts: list[str] = []
+
+        for no, ip, ap in cases:
+            pred_path = os.path.join(task_out_dir, f"{no}_pred.xlsx")
+
+            # For multi mode, the first case may already be produced
+            if not os.path.exists(pred_path):
+                ok_exec, err = run_generated_code(code, ip, pred_path)
+                if not ok_exec:
+                    all_exec = False
+                    result["cases"].append(
+                        {"no": no, "stage": "exec", "ok": False, "error": err[:500]}
+                    )
+                    if not result["fail_reason"]:
+                        tail = err.strip().splitlines()[-1][:200] if err.strip() else "unknown"
+                        result["fail_reason"] = f"exec-error: {tail}"
+                    enrichment_parts.append(
+                        f"## Execution (case {no})\nERROR: {err[:500]}"
+                    )
+                    continue
+
+            if not os.path.exists(pred_path):
+                all_exec = False
+                result["cases"].append(
+                    {"no": no, "stage": "exec", "ok": False, "error": "output-not-found"}
+                )
+                if not result["fail_reason"]:
+                    result["fail_reason"] = "output-not-found"
+                continue
+
+            result["n_exec_pass"] += 1
+            try:
+                ev = evaluate(pred_path, ap, instruction_type, answer_position_eval)
+            except Exception as e:  # noqa: BLE001
+                ev = {"ok": False, "reason": f"eval-exception: {type(e).__name__}: {e}"}
+
+            if ev["ok"]:
+                result["n_pass"] += 1
+            else:
+                if not result["fail_reason"]:
+                    result["fail_reason"] = f"eval-mismatch: {ev['reason'][:200]}"
+            result["cases"].append(
+                {"no": no, "stage": "eval", "ok": ev["ok"], "reason": ev.get("reason", "")}
+            )
+
+            # Auto-verify: reopen output and compare cells at answer_position
+            if answer_position_eval:
+                verify_report = _auto_verify_output(pred_path, ap, answer_position_eval)
+                enrichment_parts.append(
+                    f"## Eval Result (case {no}): {'PASS' if ev['ok'] else 'FAIL'}\n"
+                    f"{ev.get('reason', '')}\n\n{verify_report}"
+                )
+
+        result["exec_ok"] = all_exec
+
+        # ── Enrich conversation with eval details ──────────────────────
+        if enrichment_parts:
+            enrichment_msg = "\n\n---\n\n".join(enrichment_parts)
+            conversation = agent_result.get("conversation", [])
+            conversation.append({
+                "role": "system",
+                "content": f"[POST-EXECUTION VERIFICATION]\n\n{enrichment_msg}",
+            })
+            # Re-save the enriched conversation
+            with open(os.path.join(task_out_dir, "conversation.json"), "w") as f:
+                json.dump(conversation, f, ensure_ascii=False, indent=2)
+        n_cases = result["n_cases"]
+        n_pass = result["n_pass"]
+        result["soft"] = (n_pass / n_cases) if n_cases else 0.0
+        result["hard"] = 1 if (n_cases > 0 and n_pass == n_cases) else 0
+        result["ok"] = bool(result["hard"])
+        if result["ok"]:
+            result["fail_reason"] = ""
+        return result
+
+    except Exception as e:  # noqa: BLE001
+        result["fail_reason"] = f"unexpected: {type(e).__name__}: {e}"
+        result["error"] = traceback.format_exc()
+        return result
+
+
+# ── Codegen batch runner ────────────────────────────────────────────────────
+
+
+def run_spreadsheet_batch_codegen(
+    items: list[dict],
+    data_root: str,
+    out_root: str,
+    skill_content: str,
+    mode: str = "single",
+    max_turns: int = 5,
+    max_completion_tokens: int = 16384,
+    max_api_workers: int = 32,
+    task_timeout: int = 0,
+    use_eval_feedback: bool = False,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context_by_id: dict[str, str] | None = None,
+) -> list[dict]:
+    """Run codegen agent on all items (no tool-call).
+
+    Args:
+        mode: "single" or "multi".
+        task_timeout: Hard per-task timeout in seconds at the future level.
+            0 or negative disables the per-task timeout.
+    """
+    no_task_timeout = task_timeout <= 0
+    task_timeout_label = "none" if no_task_timeout else f"{task_timeout}s"
+
+    os.makedirs(out_root, exist_ok=True)
+
+    results_path = os.path.join(out_root, "results.jsonl")
+    done_ids: set[str] = set()
+    existing: list[dict] = []
+    if os.path.exists(results_path):
+        with open(results_path) as f:
+            for line in f:
+                try:
+                    r = json.loads(line)
+                    done_ids.add(str(r["id"]))
+                    existing.append(r)
+                except Exception:
+                    pass
+
+    pending = [it for it in items if str(it["id"]) not in done_ids]
+    print(
+        f"  [spreadsheet codegen-{mode}] total={len(items)} done={len(done_ids)} "
+        f"pending={len(pending)} workers={max_api_workers} task_timeout={task_timeout_label}"
+    )
+
+    if not pending:
+        return existing
+
+    t0 = time.time()
+    results = list(existing)
+
+    started_at: dict[str, float] = {}
+
+    def _run_one(it: dict) -> dict:
+        started_at[str(it["id"])] = time.time()
+        return process_one_codegen(
+            it,
+            data_root,
+            out_root,
+            skill_content,
+            mode,
+            max_turns,
+            max_completion_tokens,
+            task_timeout,
+            use_eval_feedback,
+            diagnostic_mode,
+            diagnostic_instruction,
+            (diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""),
+        )
+
+    def _timeout_result(item: dict) -> dict:
+        return {
+            "id": str(item["id"]),
+            "ok": False,
+            "instruction_type": item.get("instruction_type", ""),
+            "task_type": "other",
+            "phase": "timeout",
+            "fail_reason": f"task-timeout-{task_timeout}s",
+            "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0,
+            "n_turns": 0, "cases": [], "error": "timeout",
+        }
+
+    def _error_result(item: dict, e: Exception) -> dict:
+        return {
+            "id": str(item["id"]),
+            "ok": False,
+            "instruction_type": item.get("instruction_type", ""),
+            "task_type": "other",
+            "phase": "error",
+            "fail_reason": f"unexpected: {type(e).__name__}: {e}",
+            "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0,
+            "n_turns": 0, "cases": [], "error": str(e),
+        }
+
+    def _record(res: dict, i: int) -> None:
+        results.append(res)
+        status = "PASS" if res.get("hard") else ("TIMEOUT" if res.get("phase") == "timeout" else "FAIL")
+        dt = time.time() - t0
+        print(
+            f"    {i}/{len(pending)} id={res['id']:<10} {status}  "
+            f"turns={res.get('n_turns', 0):<3} "
+            f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)}  "
+            f"dt={dt:.0f}s"
+        )
+
+    ex = ThreadPoolExecutor(max_workers=max_api_workers)
+    try:
+        futs = {ex.submit(_run_one, it): it for it in pending}
+        pending_futs = set(futs)
+        finished = 0
+        while pending_futs:
+            done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED)
+            now = time.time()
+            timed_out = [] if no_task_timeout else [
+                fut for fut in pending_futs - done
+                if str(futs[fut]["id"]) in started_at
+                and now - started_at[str(futs[fut]["id"])] >= task_timeout
+            ]
+            for fut in done:
+                pending_futs.remove(fut)
+                item = futs[fut]
+                try:
+                    res = fut.result()
+                except FuturesTimeoutError:
+                    res = _timeout_result(item)
+                except Exception as e:  # noqa: BLE001
+                    res = _error_result(item, e)
+                finished += 1
+                _record(res, finished)
+            for fut in timed_out:
+                pending_futs.remove(fut)
+                fut.cancel()
+                finished += 1
+                _record(_timeout_result(futs[fut]), finished)
+    finally:
+        ex.shutdown(wait=False, cancel_futures=True)
+
+    return results