self-evolve-framework 1.4.0 → 1.6.0

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/livemathematicianbench/rollout.py

@@ -0,0 +1,434 @@
+"""LiveMathematicianBench rollout — theorem-grounded math MCQ agent."""
+from __future__ import annotations
+
+import json
+import os
+import time
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+
+from skillopt.envs.livemathematicianbench.evaluator import evaluate
+from skillopt.model import chat_target, get_target_backend, is_target_exec_backend
+from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec
+from skillopt.prompts import load_prompt
+
+def _build_system(skill_content: str) -> str:
+    if skill_content.strip():
+        skill_section = f"## Skill\n{skill_content.strip()}\n\n"
+    else:
+        skill_section = ""
+    return load_prompt("rollout_system", env="livemathematicianbench").format(skill_section=skill_section)
+
+
+def _format_choices(choices: list[dict]) -> str:
+    return "\n".join(
+        f"{choice['label']}. {choice['text']}"
+        for choice in choices
+    )
+
+
+def _build_user(
+    item: dict,
+    *,
+    use_theorem: bool = False,
+    use_sketch: bool = False,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+) -> str:
+    parts = [f"## Question\n{item['question']}", f"## Choices\n{_format_choices(item['choices'])}"]
+    if use_theorem and item.get("theorem"):
+        parts.append(f"## Theorem\n{item['theorem']}")
+    if use_sketch and item.get("sketch"):
+        parts.append(f"## Proof Sketch\n{item['sketch']}")
+    if diagnostic_trace_context.strip():
+        parts.append(
+            "## Previous Codex Trace Snapshot\n"
+            "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n"
+            f"{diagnostic_trace_context.strip()}"
+        )
+    if diagnostic_mode and diagnostic_instruction.strip():
+        parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}")
+    return "\n\n".join(parts)
+
+
+def _build_codex_skill(skill_content: str) -> str:
+    return render_skill_md(
+        skill_content,
+        description="Dynamic ReflACT skill for solving the current LiveMathematicianBench multiple-choice question.",
+        preamble=(
+            "Use this skill when solving the current math multiple-choice question.\n"
+            "Inspect the option wording carefully and output only the final choice label inside <answer>...</answer>."
+        ),
+    )
+
+def _run_codex_once(
+    *,
+    pred_dir: str,
+    skill_content: str,
+    item: dict,
+    model: str,
+    timeout: int,
+    use_theorem: bool = False,
+    use_sketch: bool = False,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+    previous_response: str = "",
+) -> tuple[str, str, str, str]:
+    user = _build_user(
+        item,
+        use_theorem=use_theorem,
+        use_sketch=use_sketch,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        diagnostic_trace_context=diagnostic_trace_context,
+    )
+    task_parts = [user]
+    if previous_response:
+        task_parts.append(
+            "## Previous Attempt\n"
+            f"{previous_response}\n\n"
+            "Re-evaluate the exact option wording. If needed, correct it."
+        )
+    task_text = "\n\n".join(task_parts)
+    skill_md = _build_codex_skill(skill_content)
+    work_dir = os.path.join(pred_dir, "codex_exec")
+    prepare_workspace(work_dir=work_dir, skill_md=skill_md, task_text=task_text)
+    prompt = (
+        "Use the `skillopt-target` skill available in this workspace.\n"
+        "Read `task.md` and solve the multiple-choice problem.\n"
+        "Output only the final choice label inside <answer>...</answer>."
+    )
+    final_message, raw = run_target_exec(
+        work_dir=work_dir,
+        prompt=prompt,
+        model=model,
+        timeout=timeout,
+    )
+    return final_message or raw, raw, skill_md, task_text
+
+
+def process_one(
+    item: dict,
+    out_root: str,
+    skill_content: str,
+    *,
+    max_turns: int = 1,
+    use_theorem: bool = False,
+    use_sketch: bool = False,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+    exec_timeout: int | None = 300,
+    max_completion_tokens: int = 16384,
+) -> dict:
+    item_id = str(item["id"])
+    result = {
+        "id": item_id,
+        "question": item["question"],
+        "task_type": item.get("theorem_type", ["math_mcq"])[0] if item.get("theorem_type") else "math_mcq",
+        "hard": 0,
+        "soft": 0.0,
+        "predicted_answer": "",
+        "predicted_label": "",
+        "predicted_text": "",
+        "correct_label": item["correct_choice"]["label"],
+        "correct_text": item["correct_choice"]["text"],
+        "response": "",
+        "fail_reason": "",
+        "agent_ok": False,
+        "n_turns": 0,
+    }
+
+    try:
+        pred_dir = os.path.join(out_root, "predictions", item_id)
+        os.makedirs(pred_dir, exist_ok=True)
+        llm_timeout = int(exec_timeout) if exec_timeout and int(exec_timeout) > 0 else None
+
+        if is_target_exec_backend():
+            from skillopt.model import azure_openai as _llm
+
+            conversation: list[dict] = []
+            response = ""
+            system = ""
+            user = ""
+            for turn in range(max_turns):
+                response, raw, system, user = _run_codex_once(
+                    pred_dir=pred_dir,
+                    skill_content=skill_content,
+                    item=item,
+                    model=_llm.TARGET_DEPLOYMENT,
+                    timeout=llm_timeout,
+                    use_theorem=use_theorem,
+                    use_sketch=use_sketch,
+                    diagnostic_mode=diagnostic_mode if turn == 0 else False,
+                    diagnostic_instruction=diagnostic_instruction if turn == 0 else "",
+                    diagnostic_trace_context=diagnostic_trace_context if turn == 0 else "",
+                    previous_response=response if turn > 0 else "",
+                )
+                conversation.append({"type": "message", "turn": turn + 1, "content": response})
+                if "<answer>" in response.lower():
+                    break
+
+            result["response"] = response
+            result["agent_ok"] = True
+            result["n_turns"] = len(conversation)
+
+            with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f:
+                f.write(system)
+            with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f:
+                f.write(user)
+
+            eval_result = evaluate(response, item["correct_choice"], item["choices"])
+            result["hard"] = int(eval_result["em"])
+            result["soft"] = eval_result["f1"]
+            result["predicted_answer"] = eval_result["predicted_answer"]
+            result["predicted_label"] = eval_result["predicted_label"]
+            result["predicted_text"] = eval_result["predicted_text"]
+            if not result["hard"]:
+                result["fail_reason"] = (
+                    f"MCQ=0: predicted '{eval_result['predicted_label'] or eval_result['predicted_answer']}' "
+                    f"but expected '{eval_result['correct_label']}'"
+                )
+            eval_detail = (
+                f"[EVALUATION RESULT]\n"
+                f"Question: {item['question']}\n"
+                f"Predicted label: {eval_result['predicted_label']!r}\n"
+                f"Predicted text: {eval_result['predicted_text']!r}\n"
+                f"Correct label: {eval_result['correct_label']!r}\n"
+                f"Correct text: {eval_result['correct_text']!r}\n"
+                f"Exact Match: {eval_result['em']}"
+            )
+            conversation.append({"role": "system", "content": eval_detail})
+            with open(os.path.join(pred_dir, "conversation.json"), "w") as f:
+                json.dump(conversation, f, ensure_ascii=False, indent=2)
+            return result
+
+        system = _build_system(skill_content)
+        user = _build_user(
+            item,
+            use_theorem=use_theorem,
+            use_sketch=use_sketch,
+            diagnostic_mode=diagnostic_mode,
+            diagnostic_instruction=diagnostic_instruction,
+            diagnostic_trace_context=diagnostic_trace_context,
+        )
+        conversation: list[dict] = []
+        response = ""
+
+        for turn in range(max_turns):
+            if turn == 0:
+                resp_text, _ = chat_target(
+                    system=system,
+                    user=user,
+                    max_completion_tokens=max_completion_tokens,
+                    retries=5,
+                    stage="rollout",
+                    timeout=llm_timeout,
+                )
+            else:
+                refinement = (
+                    f"Your previous answer was:\n{response}\n\n"
+                    "Re-evaluate the exact option wording. If needed, correct it. "
+                    "Output only the final choice label inside <answer>...</answer>."
+                )
+                resp_text, _ = chat_target(
+                    system=system,
+                    user=refinement,
+                    max_completion_tokens=max_completion_tokens,
+                    retries=5,
+                    stage="rollout",
+                    timeout=llm_timeout,
+                )
+            response = resp_text
+            conversation.append({"type": "message", "turn": turn + 1, "content": resp_text})
+            if "<answer>" in resp_text.lower():
+                break
+
+        result["response"] = response
+        result["agent_ok"] = True
+        result["n_turns"] = len(conversation)
+
+        with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f:
+            f.write(system)
+        with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f:
+            f.write(user)
+
+        eval_result = evaluate(response, item["correct_choice"], item["choices"])
+        result["hard"] = int(eval_result["em"])
+        result["soft"] = eval_result["f1"]
+        result["predicted_answer"] = eval_result["predicted_answer"]
+        result["predicted_label"] = eval_result["predicted_label"]
+        result["predicted_text"] = eval_result["predicted_text"]
+
+        if not result["hard"]:
+            result["fail_reason"] = (
+                f"MCQ=0: predicted '{eval_result['predicted_label'] or eval_result['predicted_answer']}' "
+                f"but expected '{eval_result['correct_label']}'"
+            )
+
+        eval_detail = (
+            f"[EVALUATION RESULT]\n"
+            f"Question: {item['question']}\n"
+            f"Predicted label: {eval_result['predicted_label']!r}\n"
+            f"Predicted text: {eval_result['predicted_text']!r}\n"
+            f"Correct label: {eval_result['correct_label']!r}\n"
+            f"Correct text: {eval_result['correct_text']!r}\n"
+            f"Exact Match: {eval_result['em']}"
+        )
+        conversation.append({"role": "system", "content": eval_detail})
+
+        with open(os.path.join(pred_dir, "conversation.json"), "w") as f:
+            json.dump(conversation, f, ensure_ascii=False, indent=2)
+
+    except Exception as e:  # noqa: BLE001
+        result["fail_reason"] = f"error: {e}"
+
+    return result
+
+
+def run_batch(
+    items: list[dict],
+    out_root: str,
+    skill_content: str,
+    *,
+    max_turns: int = 1,
+    exec_timeout: int | None = 300,
+    workers: int = 64,
+    max_completion_tokens: int = 16384,
+    use_theorem: bool = False,
+    use_sketch: bool = False,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context_by_id: dict[str, str] | None = None,
+    task_timeout: int | None = 600,
+) -> list[dict]:
+    exec_timeout_value = int(exec_timeout) if exec_timeout and int(exec_timeout) > 0 else 0
+    task_timeout_value = int(task_timeout) if task_timeout and int(task_timeout) > 0 else 0
+    if exec_timeout_value <= 0 or task_timeout_value <= 0:
+        task_timeout = None
+    else:
+        task_timeout = max(task_timeout_value, exec_timeout_value + 60)
+    results_path = os.path.join(out_root, "results.jsonl")
+    os.makedirs(out_root, exist_ok=True)
+
+    done_ids: set[str] = set()
+    existing: list[dict] = []
+    if os.path.exists(results_path):
+        with open(results_path) as f:
+            for line in f:
+                try:
+                    r = json.loads(line)
+                    done_ids.add(str(r["id"]))
+                    existing.append(r)
+                except Exception:
+                    pass
+
+    pending = [it for it in items if str(it["id"]) not in done_ids]
+    if not pending:
+        return existing
+
+    total = len(existing) + len(pending)
+    completed = len(existing)
+    correct_count = sum(1 for r in existing if r.get("hard", 0))
+    if existing:
+        print(f"    [rollout] resuming: {completed}/{total} already done", flush=True)
+
+    results = list(existing)
+
+    started_at: dict[str, float] = {}
+
+    def _run_one(it: dict) -> dict:
+        started_at[str(it["id"])] = time.time()
+        return process_one(
+            it,
+            out_root,
+            skill_content,
+            max_turns=max_turns,
+            exec_timeout=exec_timeout,
+            max_completion_tokens=max_completion_tokens,
+            use_theorem=use_theorem,
+            use_sketch=use_sketch,
+            diagnostic_mode=diagnostic_mode,
+            diagnostic_instruction=diagnostic_instruction,
+            diagnostic_trace_context=(diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""),
+        )
+
+    def _timeout_result(it: dict) -> dict:
+        correct = it.get("correct_choice") or {}
+        return {
+            "id": str(it["id"]),
+            "question": it.get("question", ""),
+            "task_type": it.get("theorem_type", ["math_mcq"])[0] if it.get("theorem_type") else "math_mcq",
+            "hard": 0,
+            "soft": 0.0,
+            "predicted_answer": "",
+            "predicted_label": "",
+            "predicted_text": "",
+            "correct_label": correct.get("label", ""),
+            "correct_text": correct.get("text", ""),
+            "response": "",
+            "fail_reason": f"task-timeout-{task_timeout}s",
+            "agent_ok": False,
+            "n_turns": 0,
+        }
+
+    def _error_result(it: dict, exc: Exception) -> dict:
+        res = _timeout_result(it)
+        res["fail_reason"] = f"error: {type(exc).__name__}: {exc}"
+        return res
+
+    with open(results_path, "a") as outf:
+        ex = ThreadPoolExecutor(max_workers=workers)
+        try:
+            futs = {
+                ex.submit(_run_one, it): it
+                for it in pending
+            }
+            pending_futs = set(futs)
+            while pending_futs:
+                done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED)
+                now = time.time()
+                timed_out = [
+                    fut for fut in pending_futs - done
+                    if task_timeout is not None
+                    if str(futs[fut]["id"]) in started_at
+                    and now - started_at[str(futs[fut]["id"])] >= task_timeout
+                ]
+                for fut in done:
+                    pending_futs.remove(fut)
+                    item = futs[fut]
+                    try:
+                        res = fut.result()
+                    except Exception as e:  # noqa: BLE001
+                        res = _error_result(item, e)
+                    results.append(res)
+                    completed += 1
+                    if res.get("hard", 0):
+                        correct_count += 1
+                    acc = correct_count / completed if completed else 0
+                    print(
+                        f"    [rollout] {completed}/{total} "
+                        f"(acc={acc:.3f}) id={res['id']} "
+                        f"hard={res.get('hard', '?')}",
+                        flush=True,
+                    )
+                    outf.write(json.dumps(res, ensure_ascii=False) + "\n")
+                    outf.flush()
+                for fut in timed_out:
+                    pending_futs.remove(fut)
+                    res = _timeout_result(futs[fut])
+                    results.append(res)
+                    completed += 1
+                    acc = correct_count / completed if completed else 0
+                    print(
+                        f"    [rollout] {completed}/{total} "
+                        f"(acc={acc:.3f}) id={res['id']} TIMEOUT",
+                        flush=True,
+                    )
+                    outf.write(json.dumps(res, ensure_ascii=False) + "\n")
+                    outf.flush()
+        finally:
+            ex.shutdown(wait=False, cancel_futures=True)
+
+    return results

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/livemathematicianbench/skills/initial.md

@@ -0,0 +1,16 @@
+# Live Mathematical MCQ Heuristics
+
+## Option Comparison
+- Compare all options before committing. The correct choice is often the strongest statement justified by the question, while nearby distractors are weaker, overstrong, or miss an equality case.
+- Track exact quantifiers such as "there exists", "for every", "if and only if", and "exactly when".
+
+## Theorem-Level Precision
+- Check whether an option weakens the conclusion by dropping a characterization, equality clause, or full equivalence.
+- Check whether an option overstates the theorem by upgrading regularity, removing scale restrictions, or changing an existential statement into a universal one.
+
+## Hypotheses
+- Verify the hypotheses and domain carefully. Distractors often keep the theorem shape but alter the required assumptions.
+- Pay close attention to equality cases, extremal conditions, and whether a result applies to the full family or only a restricted subfamily.
+
+## Final Answer
+- Output the final answer as the single option label only.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/officeqa/__init__.py

@@ -0,0 +1 @@
+"""OfficeQA environment package for ReflACT."""

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/officeqa/adapter.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import os
+
+from skillopt.datasets.base import BatchSpec
+from skillopt.envs.base import EnvAdapter
+from skillopt.envs.officeqa.dataloader import OfficeQADataLoader
+from skillopt.envs.officeqa.rollout import run_batch
+
+
+class OfficeQAAdapter(EnvAdapter):
+    def __init__(
+        self,
+        split_dir: str = "",
+        data_path: str = "",
+        split_mode: str = "split_dir",
+        split_ratio: str = "2:1:7",
+        split_seed: int = 42,
+        split_output_dir: str = "",
+        workers: int = 8,
+        analyst_workers: int = 8,
+        failure_only: bool = False,
+        minibatch_size: int = 8,
+        edit_budget: int = 4,
+        seed: int = 42,
+        limit: int = 0,
+        max_tool_turns: int = 12,
+        max_completion_tokens: int = 16384,
+        search_mode: str = "offline",
+        max_queries_per_turn: int = 4,
+        search_api_url: str = os.environ.get("OFFICEQA_SEARCH_API_URL", "http://localhost:8080/search_tool/search"),
+        search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH",
+        search_provider: str = "duckduckgo",
+        search_max_num_results: int = 4,
+        search_timeout_seconds: int = 20,
+        use_local_tools: bool = True,
+        data_dirs: list[str] | str | None = None,
+        docs_dirs: list[str] | str | None = None,    ) -> None:
+        self.workers = workers
+        self.analyst_workers = analyst_workers
+        self.failure_only = failure_only
+        self.minibatch_size = minibatch_size
+        self.edit_budget = edit_budget
+        self.max_tool_turns = max_tool_turns
+        self.max_completion_tokens = int(max_completion_tokens)
+        self.search_mode = str(search_mode or "offline")
+        self.max_queries_per_turn = int(max_queries_per_turn)
+        self.search_api_url = str(search_api_url or "").strip()
+        self.search_auth_env = str(search_auth_env or "OFFICEQA_CUSTOM_SEARCH_AUTH").strip()
+        self.search_provider = str(search_provider or "duckduckgo").strip()
+        self.search_max_num_results = int(search_max_num_results)
+        self.search_timeout_seconds = int(search_timeout_seconds)
+        self.use_local_tools = bool(use_local_tools)
+        self.data_dirs = data_dirs if data_dirs is not None else docs_dirs
+        self.dataloader = OfficeQADataLoader(
+            split_dir=split_dir,
+            data_path=data_path,
+            split_mode=split_mode,
+            split_ratio=split_ratio,
+            split_seed=split_seed,
+            split_output_dir=split_output_dir,
+            seed=seed,
+            limit=limit,
+        )
+
+    def setup(self, cfg: dict) -> None:
+        super().setup(cfg)
+        self.dataloader.setup(cfg)
+
+    def get_dataloader(self):
+        return self.dataloader
+
+    def build_env_from_batch(self, batch: BatchSpec, **kwargs):
+        return list(batch.payload or [])
+
+    def build_train_env(self, batch_size: int, seed: int, **kwargs):
+        batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
+        batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]:
+        items: list[dict] = env_manager
+        return run_batch(
+            items=items,
+            out_root=out_dir,
+            skill_content=skill_content,
+            workers=self.workers,
+            max_tool_turns=self.max_tool_turns,
+            max_completion_tokens=self.max_completion_tokens,
+            search_mode=self.search_mode,
+            max_queries_per_turn=self.max_queries_per_turn,
+            search_api_url=self.search_api_url,
+            search_auth_env=self.search_auth_env,
+            search_provider=self.search_provider,
+            search_max_num_results=self.search_max_num_results,
+            search_timeout_seconds=self.search_timeout_seconds,
+            use_local_tools=self.use_local_tools,
+            data_dirs=self.data_dirs,
+            diagnostic_mode=kwargs.get("diagnostic_mode", False),
+            diagnostic_instruction=kwargs.get("diagnostic_instruction", ""),
+        )
+
+    def get_task_types(self) -> list[str]:
+        seen: list[str] = []
+        for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items:
+            task_type = str(item.get("task_type") or "officeqa")
+            if task_type not in seen:
+                seen.append(task_type)
+        return seen or ["officeqa"]

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/officeqa/dataloader.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import csv
+import json
+import os
+from pathlib import Path
+
+from skillopt.datasets.base import SplitDataLoader
+
+
+def _parse_list_field(value: str | list[str] | None) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [str(item).strip() for item in value if str(item).strip()]
+    text = str(value).strip()
+    if not text:
+        return []
+    try:
+        loaded = json.loads(text)
+    except json.JSONDecodeError:
+        loaded = None
+    if isinstance(loaded, list):
+        return [str(item).strip() for item in loaded if str(item).strip()]
+    if "\n" in text:
+        return [part.strip() for part in text.splitlines() if part.strip()]
+    if "," in text and not text.lower().endswith(".txt"):
+        return [part.strip() for part in text.split(",") if part.strip()]
+    return [text]
+
+
+def _normalize_row(row: dict[str, str]) -> dict:
+    item_id = str(row.get("uid") or row.get("id") or "").strip()
+    question = str(row.get("question") or "").strip()
+    ground_truth = str(row.get("ground_truth") or row.get("answer") or "").strip()
+    task_type = str(row.get("category") or row.get("difficulty") or "officeqa").strip() or "officeqa"
+    source_files = _parse_list_field(row.get("source_files"))
+    source_docs = _parse_list_field(row.get("source_docs"))
+    split = str(row.get("split") or "").strip()
+    return {
+        "id": item_id,
+        "uid": item_id,
+        "question": question,
+        "ground_truth": ground_truth,
+        "answers": [ground_truth] if ground_truth else [],
+        "task_type": task_type,
+        "category": task_type,
+        "source_files": source_files,
+        "source_docs": source_docs,
+        "split": split,
+    }
+
+
+class OfficeQADataLoader(SplitDataLoader):
+    def load_split_items(self, split_path: str) -> list[dict]:
+        path = Path(split_path)
+        csv_files = sorted(path.glob("*.csv"))
+        if csv_files:
+            with csv_files[0].open(encoding="utf-8", newline="") as f:
+                reader = csv.DictReader(f)
+                return [_normalize_row(row) for row in reader]
+
+        json_files = sorted(path.glob("*.json"))
+        if json_files:
+            with json_files[0].open(encoding="utf-8") as f:
+                data = json.load(f)
+            if not isinstance(data, list):
+                raise ValueError(f"Expected JSON array in {json_files[0]}")
+            return [_normalize_row(item) for item in data]
+
+        raise FileNotFoundError(f"No .csv or .json file found in {split_path}")

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/officeqa/evaluator.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import re
+import string
+from collections import Counter
+
+
+_NUMERIC_CHARS = set("0123456789.-")
+
+
+def normalize_answer(text: str) -> str:
+    text = text.lower().strip()
+    text = text.replace(",", "")
+    text = "".join(ch for ch in text if ch not in string.punctuation or ch in _NUMERIC_CHARS or ch == "%")
+    text = re.sub(r"\b(million|millions|billion|billions|dollars|dollar|nominal)\b", " ", text)
+    text = " ".join(text.split())
+    return text
+
+
+def exact_match(prediction: str, gold: str) -> float:
+    return 1.0 if normalize_answer(prediction) == normalize_answer(gold) else 0.0
+
+
+def token_f1(prediction: str, gold: str) -> float:
+    pred_tokens = normalize_answer(prediction).split()
+    gold_tokens = normalize_answer(gold).split()
+    if not pred_tokens or not gold_tokens:
+        return 1.0 if pred_tokens == gold_tokens else 0.0
+    common = Counter(pred_tokens) & Counter(gold_tokens)
+    n_common = sum(common.values())
+    if n_common == 0:
+        return 0.0
+    precision = n_common / len(pred_tokens)
+    recall = n_common / len(gold_tokens)
+    return 2 * precision * recall / (precision + recall)
+
+
+def evaluate(prediction: str, gold: str) -> dict:
+    em = exact_match(prediction, gold)
+    f1 = token_f1(prediction, gold)
+    return {
+        "em": em,
+        "f1": f1,
+        "predicted_answer": prediction.strip(),
+        "gold_answer": gold,
+    }