self-evolve-framework 1.5.0 → 1.6.0

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/spreadsheetbench/dataloader.py

@@ -0,0 +1,37 @@
+"""SpreadsheetBench task dataloader."""
+from __future__ import annotations
+
+from skillopt.datasets.base import SplitDataLoader
+
+
+class SpreadsheetBenchDataLoader(SplitDataLoader):
+    """SpreadsheetBench dataloader.
+
+    Each split directory contains a .json file (JSON array of task items).
+    Spreadsheet files referenced by items live under a separate ``data_root``.
+    """
+
+    def __init__(
+        self,
+        split_dir: str = "",
+        data_path: str = "",
+        split_mode: str = "ratio",
+        split_ratio: str = "2:1:7",
+        split_seed: int = 42,
+        split_output_dir: str = "",
+        data_root: str = "",
+        seed: int = 42,
+        limit: int = 0,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            split_dir=split_dir,
+            data_path=data_path,
+            split_mode=split_mode,
+            split_ratio=split_ratio,
+            split_seed=split_seed,
+            split_output_dir=split_output_dir,
+            seed=seed,
+            limit=limit,
+        )
+        self.data_root = data_root

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/spreadsheetbench/evaluator.py

@@ -0,0 +1,158 @@
+"""Cell-value evaluator faithful to the official SpreadsheetBench
+`evaluation/evaluation.py` (https://github.com/RUCKBReasoning/SpreadsheetBench).
+
+Key rules (copied from the official `transform_value` / `compare_cell_value`):
+  * numeric values (int/float and numeric strings) are compared after
+    ``round(float(v), 2)`` — a fixed 2-decimal quantization (NOT a tolerance);
+  * ``datetime.time`` is stringified and the trailing microseconds stripped;
+  * ``datetime.datetime`` is converted to an Excel serial day and rounded
+    to an integer day;
+  * an empty string ``""`` and ``None`` are considered equal, but otherwise
+    ``type(v1) != type(v2)`` fails the comparison.
+
+Format/style comparison is deliberately NOT performed — the official
+reference evaluator also skips it (the relevant lines are commented out
+in `cell_level_compare`). soft vs hard is defined at the run_bench level
+across a task's multiple test cases, not here.
+"""
+from __future__ import annotations
+
+import datetime
+import os
+import re
+
+import openpyxl
+
+
+# ---------- value transform / compare (official port) ----------
+
+def _datetime_to_float(dt: datetime.datetime) -> float:
+    excel_start_date = datetime.datetime(1899, 12, 30)
+    delta = dt - excel_start_date
+    return delta.days + delta.seconds / 86400.0
+
+
+def _transform_value(v):
+    if isinstance(v, bool):
+        # openpyxl can return Python bool; official code doesn't special-case
+        # bools, but round(float(True), 2) == 1.0 which breaks 1 vs True. Keep
+        # parity with the official transform by promoting bool -> float.
+        return round(float(v), 2)
+    if isinstance(v, (int, float)):
+        return round(float(v), 2)
+    if isinstance(v, datetime.time):
+        return str(v)[:-3]
+    if isinstance(v, datetime.datetime):
+        return round(_datetime_to_float(v), 0)
+    if isinstance(v, str):
+        try:
+            return round(float(v), 2)
+        except ValueError:
+            return v
+    return v
+
+
+def _compare_cell_value(v1, v2) -> bool:
+    v1 = _transform_value(v1)
+    v2 = _transform_value(v2)
+    if (v1 == "" and v2 is None) or (v1 is None and v2 == ""):
+        return True
+    if (v1 == "" and v2 == "") or (v1 is None and v2 is None):
+        return True
+    if type(v1) is not type(v2):
+        return False
+    return v1 == v2
+
+
+# ---------- range parsing (official port) ----------
+
+def _col_num2name(n: int) -> str:
+    name = ""
+    while n > 0:
+        n, r = divmod(n - 1, 26)
+        name = chr(65 + r) + name
+    return name
+
+
+def _col_name2num(name: str) -> int:
+    num = 0
+    for c in name:
+        num = num * 26 + (ord(c) - ord("A") + 1)
+    return num
+
+
+def _parse_range(range_str: str):
+    start_cell, end_cell = range_str.split(":")
+    sc = "".join(ch for ch in start_cell if ch.isalpha())
+    sr = "".join(ch for ch in start_cell if ch.isdigit())
+    ec = "".join(ch for ch in end_cell if ch.isalpha())
+    er = "".join(ch for ch in end_cell if ch.isdigit())
+    return (_col_name2num(sc), int(sr)), (_col_name2num(ec), int(er))
+
+
+def _generate_cell_names(range_str: str):
+    if ":" not in range_str:
+        return [range_str]
+    (sc, sr), (ec, er) = _parse_range(range_str)
+    cols = [_col_num2name(i) for i in range(sc, ec + 1)]
+    return [f"{c}{r}" for c in cols for r in range(sr, er + 1)]
+
+
+def _cell_level_compare(wb_gt, wb_proc, sheet_name: str, cell_range: str):
+    if sheet_name not in wb_proc.sheetnames:
+        return False, f"worksheet not found: {sheet_name}"
+    ws_gt = wb_gt[sheet_name]
+    ws_proc = wb_proc[sheet_name]
+    for cn in _generate_cell_names(cell_range):
+        cg = ws_gt[cn]
+        cp = ws_proc[cn]
+        if not _compare_cell_value(cg.value, cp.value):
+            return False, f"value@{sheet_name}!{cn}: gt={cg.value!r} pred={cp.value!r}"
+    return True, ""
+
+
+# ---------- public API ----------
+
+def compare_workbooks(gt_file: str, proc_file: str, answer_position: str) -> tuple[bool, str]:
+    """Return (ok, msg). Single test-case comparison, official semantics."""
+    if not os.path.exists(proc_file):
+        return False, "file not exist"
+    try:
+        wb_gt = openpyxl.load_workbook(filename=gt_file, data_only=True)
+        wb_proc = openpyxl.load_workbook(filename=proc_file, data_only=True)
+    except Exception as e:  # noqa: BLE001
+        return False, f"load error: {e}"
+    try:
+        ok_all = True
+        msg_first = ""
+        for scr in (answer_position or "").split(","):
+            scr = scr.strip()
+            if not scr:
+                continue
+            if "!" in scr:
+                sheet_name, cell_range = scr.split("!", 1)
+                sheet_name = sheet_name.strip().strip("'\"")
+            else:
+                sheet_name = wb_gt.sheetnames[0]
+                cell_range = scr
+            cell_range = cell_range.strip().strip("'\"")
+            ok, msg = _cell_level_compare(wb_gt, wb_proc, sheet_name, cell_range)
+            if not ok:
+                ok_all = False
+                if not msg_first:
+                    msg_first = msg
+        return ok_all, msg_first
+    finally:
+        wb_gt.close()
+        wb_proc.close()
+
+
+def evaluate(pred_path: str, gold_path: str,
+             instruction_type: str, answer_position: str) -> dict:
+    """Single test-case evaluate. soft/hard aggregation happens in run_bench."""
+    ok, msg = compare_workbooks(gold_path, pred_path, answer_position)
+    return {
+        "ok": ok,
+        "reason": msg,
+        "instruction_type": instruction_type,
+    }

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/spreadsheetbench/executor.py

@@ -0,0 +1,67 @@
+"""Execute LLM-generated Python code against an input xlsx to produce an output xlsx."""
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import textwrap
+
+
+RUNNER_TEMPLATE = textwrap.dedent(
+    """
+    import os, sys, traceback
+    INPUT_PATH = {input_path!r}
+    OUTPUT_PATH = {output_path!r}
+    try:
+    {user_code_indented}
+    except Exception:
+        traceback.print_exc()
+        sys.exit(2)
+    """
+)
+
+# Regex to strip user-defined INPUT_PATH / OUTPUT_PATH assignments,
+# since the runner template injects the correct values.
+_PATH_ASSIGN_RE = re.compile(
+    r'^\s*(INPUT_PATH|OUTPUT_PATH)\s*=\s*.+$', re.MULTILINE
+)
+
+
+def _strip_path_assignments(code: str) -> str:
+    """Remove INPUT_PATH/OUTPUT_PATH assignments from user code."""
+    return _PATH_ASSIGN_RE.sub("", code)
+
+
+def run_generated_code(code: str, input_path: str, output_path: str, timeout: int | None = 120) -> tuple[bool, str]:
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    cleaned = _strip_path_assignments(code)
+    indented = textwrap.indent(cleaned, "    ")
+    script = RUNNER_TEMPLATE.format(
+        input_path=input_path,
+        output_path=output_path,
+        user_code_indented=indented,
+    )
+    with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f:
+        f.write(script)
+        tmp = f.name
+    try:
+        proc = subprocess.run(
+            [sys.executable, tmp],
+            capture_output=True,
+            text=True,
+            timeout=timeout if timeout and timeout > 0 else None,
+        )
+        if proc.returncode != 0:
+            return False, (proc.stdout + "\n" + proc.stderr).strip()
+        if not os.path.exists(output_path):
+            return False, "output file was not created"
+        return True, ""
+    except subprocess.TimeoutExpired:
+        return False, f"timeout after {timeout}s"
+    finally:
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/spreadsheetbench/prompts/analyst_error.md

@@ -0,0 +1,46 @@
+You are an expert failure-analysis agent for spreadsheet manipulation tasks.
+
+You will be given MULTIPLE failed agent trajectories from a single minibatch
+and the current skill document.
+Your job is to identify the most important COMMON failure patterns across
+the batch and propose a concise set of skill edits.
+
+## Failure Type Categories
+- **rule_missing**: the skill lacks a relevant rule for this type of task
+- **rule_wrong**: an existing skill rule is misleading or incorrect
+- **rule_ignored**: the skill has the right rule but the agent did not follow it
+- **data_exploration**: the agent did not read enough data from the spreadsheet
+- **code_error**: the agent's code has a bug unrelated to the skill
+- **other**: none of the above
+
+## Analysis Process
+1. Read ALL failed trajectories in the minibatch.
+2. Identify the most prevalent, systematic failure patterns across them.
+3. For each pattern, classify its failure type.
+4. Propose skill edits that address the COMMON patterns — not individual edge cases.
+5. Edits must be generalizable; do not hardcode task-specific values
+   (file paths, cell addresses, expected values).
+6. Only patch gaps in the skill — do not duplicate existing content.
+7. If the failure is because the agent did not read enough spreadsheet rows/columns
+   to understand the data, propose a patch encouraging broader data exploration.
+
+You will be told the maximum number of edits (the budget L). Produce AT MOST L edits,
+focusing on the highest-impact patterns. You may produce fewer if warranted.
+
+Respond ONLY with a valid JSON object (no markdown fences, no extra text):
+{
+  "batch_size": <number of trajectories analysed>,
+  "failure_summary": [
+    {"failure_type": "<type>", "count": <int>, "description": "<one-line>"}
+  ],
+  "patch": {
+    "reasoning": "<why these edits address the batch's common failures>",
+    "edits": [
+      {"op": "append",       "content": "<markdown to add at end of skill>"},
+      {"op": "insert_after", "target": "<exact heading/text to insert after>", "content": "<markdown>"},
+      {"op": "replace",      "target": "<exact text to replace>",              "content": "<replacement>"},
+      {"op": "delete",       "target": "<exact text to remove>"}
+    ]
+  }
+}
+Only include edits that are needed. "edits" can be an empty list if no patch is warranted.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/spreadsheetbench/prompts/analyst_success.md

@@ -0,0 +1,32 @@
+You are an expert success-pattern analyst for AI spreadsheet agents.
+
+You will be given MULTIPLE successful agent trajectories from a single minibatch
+and the current skill document. Your job is to identify generalizable behavior
+patterns that are COMMON across the batch and worth encoding in the skill.
+
+## Rules
+- Only propose patches for patterns NOT already covered in the skill.
+- Focus on patterns that appear across MULTIPLE trajectories in the batch.
+- Be concise. Patterns must generalize beyond specific tasks.
+- Prefer reinforcing existing sections over adding new top-level sections.
+- If the agents' success involved reading enough data rows or using a smart
+  exploration strategy, consider reinforcing that in the patch.
+
+You will be told the maximum number of edits (the budget L). Produce AT MOST L edits,
+focusing on the most broadly applicable patterns. You may produce fewer if warranted.
+
+Respond ONLY with a valid JSON object:
+{
+  "batch_size": <number of trajectories analysed>,
+  "success_patterns": ["<pattern 1>", "<pattern 2>"],
+  "patch": {
+    "reasoning": "<why these patterns are worth encoding>",
+    "edits": [
+      {"op": "append",       "content": "<markdown>"},
+      {"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
+      {"op": "replace",      "target": "<old text>",     "content": "<new text>"},
+      {"op": "delete",       "target": "<exact text to remove>"}
+    ]
+  }
+}
+"edits" may be empty if the skill already covers all observed patterns.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/spreadsheetbench/prompts/codegen_system.md

@@ -0,0 +1 @@
+You are an expert Python programmer specializing in spreadsheet manipulation. You will be given a user instruction together with a preview of an input .xlsx file. Your job is to write a single self-contained Python script that reads the input file at the path stored in the variable INPUT_PATH, performs the requested manipulation, and saves the result to OUTPUT_PATH. Use only the standard library, openpyxl, and pandas. Do not print anything. Do not use input(). Do not hardcode file paths. Return ONLY the Python code inside a single ```python ... ``` fenced block.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/spreadsheetbench/prompts/critical_rules.md

@@ -0,0 +1,9 @@
+## Critical Rules (MUST follow)
+1. NEVER write Excel formulas to cells that will be graded on their displayed value.
+   openpyxl does NOT compute formulas -- the evaluator will see None.
+   Instead, compute results in Python and write literal values (numbers/strings).
+2. After saving the workbook, ALWAYS reopen and verify the written values:
+   `wb2 = openpyxl.load_workbook(OUTPUT_PATH); print(wb2[sheet][cell].value)`
+3. Use the `write_file` tool to create solution.py -- it avoids shell escaping issues.
+   Do NOT use `echo "..." > solution.py` for multi-line scripts.
+

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/spreadsheetbench/prompts/react_system.md

@@ -0,0 +1,21 @@
+You are an expert spreadsheet manipulation agent.
+
+{critical_rules}{skill_section}## Tools
+You have two tools:
+- `bash` -- execute any shell command and receive its output.
+- `write_file` -- write content to a file (path, content). Use this for solution.py.
+
+## Protocol
+1. Explore the input spreadsheet to understand its structure (sheets, headers, row count).
+2. Use the `write_file` tool to create `solution.py` in the current directory.
+   solution.py MUST start with:
+       INPUT_PATH  = "<exact input path given in the task>"
+       OUTPUT_PATH = "<exact output path given in the task>"
+   Then perform the manipulation and save the result to OUTPUT_PATH.
+   Use only: standard library, openpyxl, pandas.
+3. Run `python solution.py` via `bash` and verify the output was created.
+4. Fix any errors and re-run until the output is correct.
+5. Once OUTPUT_PATH exists and is correct, stop calling tools.
+
+Do NOT use any libraries other than standard library, openpyxl, and pandas.
+Do NOT hardcode cell values from the preview -- iterate over actual rows.