self-evolve-framework 1.6.0 → 1.6.1

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/livemathematicianbench/dataloader.py

@@ -1,308 +0,0 @@
-"""LiveMathematicianBench task dataloader."""
-from __future__ import annotations
-
-import glob
-import hashlib
-import json
-import os
-import random
-from typing import Any
-
-from skillopt.datasets.base import BatchSpec, SplitDataLoader
-
-
-# ── Raw data loading utilities (for preprocessing / standalone eval) ─────
-
-_CHOICE_LABELS = ["A", "B", "C", "D", "E", "F", "G"]
-
-
-def _load_json(path: str) -> Any:
-    with open(path) as f:
-        return json.load(f)
-
-
-def _iter_monthly_files(data_path: str) -> list[str]:
-    if not data_path:
-        return []
-    if os.path.isfile(data_path):
-        return [data_path]
-    if os.path.isdir(data_path):
-        nested = glob.glob(
-            os.path.join(data_path, "**", "qa_*_final.json"),
-            recursive=True,
-        )
-        flat = glob.glob(os.path.join(data_path, "qa_*_final.json"))
-        return sorted(set(nested + flat))
-    return []
-
-
-def _coerce_choices(raw_choices: Any) -> list[dict]:
-    if isinstance(raw_choices, list):
-        choices: list[dict] = []
-        for idx, item in enumerate(raw_choices):
-            if isinstance(item, dict):
-                label = str(item.get("label") or _CHOICE_LABELS[idx]).strip()
-                text = str(item.get("text") or item.get("content") or "").strip()
-            else:
-                label = _CHOICE_LABELS[idx]
-                text = str(item).strip()
-            if text:
-                choices.append({"label": label, "text": text})
-        return choices
-
-    if isinstance(raw_choices, dict):
-        labels = sorted(raw_choices.keys())
-        return [
-            {"label": str(label).strip(), "text": str(raw_choices[label]).strip()}
-            for label in labels
-            if str(raw_choices[label]).strip()
-        ]
-
-    return []
-
-
-def _coerce_theorem_types(raw: Any) -> list[str]:
-    if isinstance(raw, list):
-        return [str(x).strip() for x in raw if str(x).strip()]
-    if raw is None:
-        return []
-    text = str(raw).strip()
-    return [text] if text else []
-
-
-def _normalize_label(text: str) -> str:
-    return str(text).strip().upper().rstrip(".):")
-
-
-def _normalize_item(item: dict, row_idx: int, source_path: str) -> dict:
-    mcq = item.get("mcq", {}) if isinstance(item.get("mcq"), dict) else {}
-    question = str(mcq.get("question") or item.get("question") or "").strip()
-    choices = _coerce_choices(mcq.get("choices") or item.get("choices") or [])
-    correct = mcq.get("correct_choice") or item.get("correct_choice") or {}
-
-    if isinstance(correct, dict):
-        correct_label = _normalize_label(correct.get("label", ""))
-        correct_text = str(correct.get("text") or "").strip()
-    else:
-        correct_label = _normalize_label(correct)
-        correct_text = ""
-
-    choice_by_label = {
-        _normalize_label(choice["label"]): choice["text"]
-        for choice in choices
-    }
-    if correct_label and not correct_text:
-        correct_text = choice_by_label.get(correct_label, "")
-    if correct_label and correct_text and correct_label not in choice_by_label:
-        choices.append({"label": correct_label, "text": correct_text})
-        choices.sort(key=lambda choice: _CHOICE_LABELS.index(choice["label"]) if choice["label"] in _CHOICE_LABELS else len(_CHOICE_LABELS))
-        choice_by_label[correct_label] = correct_text
-
-    month = str(item.get("month") or "").strip()
-    item_no = item.get("no", row_idx + 1)
-    item_id = f"{month}:{item_no}" if month else str(item_no)
-
-    return {
-        "id": item_id,
-        "month": month,
-        "no": item_no,
-        "paper_link": str(item.get("paper_link") or "").strip(),
-        "theorem": str(item.get("theorem") or "").strip(),
-        "sketch": str(item.get("sketch") or "").strip(),
-        "theorem_type": _coerce_theorem_types(item.get("theorem_type")),
-        "question": question,
-        "choices": choices,
-        "correct_choice": {
-            "label": correct_label,
-            "text": correct_text,
-        },
-        "source_path": source_path,
-    }
-
-
-def load_items(data_path: str) -> list[dict]:
-    """Load and normalise LiveMathematicianBench items from JSON files."""
-    files = _iter_monthly_files(data_path)
-    if not files:
-        raise ValueError(
-            "LiveMathematicianBench requires data_path to be a qa_*_final.json file "
-            "or a directory containing monthly qa_*_final.json files."
-        )
-
-    items: list[dict] = []
-    for path in files:
-        raw = _load_json(path)
-        if not isinstance(raw, list):
-            raise ValueError(f"Expected JSON array in {path}, got {type(raw).__name__}")
-        for row_idx, item in enumerate(raw):
-            norm = _normalize_item(item, row_idx=row_idx, source_path=path)
-            if norm["question"] and norm["choices"] and norm["correct_choice"]["label"]:
-                items.append(norm)
-    if not items:
-        raise ValueError(f"No valid LiveMathematicianBench items loaded from {data_path}")
-    return items
-
-
-# ── Dataloader ───────────────────────────────────────────────────────────
-
-class LiveMathematicianBenchDataLoader(SplitDataLoader):
-    """LiveMathematicianBench dataloader with per-seed choice shuffling."""
-
-    def __init__(
-        self,
-        split_dir: str = "",
-        data_path: str = "",
-        split_mode: str = "ratio",
-        split_ratio: str = "2:1:7",
-        split_seed: int = 42,
-        split_output_dir: str = "",
-        seed: int = 42,
-        limit: int = 0,
-        shuffle_choices: bool = True,
-        **kwargs,
-    ) -> None:
-        super().__init__(
-            split_dir=split_dir,
-            data_path=data_path,
-            split_mode=split_mode,
-            split_ratio=split_ratio,
-            split_seed=split_seed,
-            split_output_dir=split_output_dir,
-            seed=seed,
-            limit=limit,
-        )
-        self.shuffle_choices = shuffle_choices
-        self._task_types: list[str] = []
-
-    def load_raw_items(self, data_path: str) -> list[dict]:
-        return load_items(data_path)
-
-    def setup(self, cfg: dict) -> None:
-        super().setup(cfg)
-        all_items = self.train_items + self.val_items + self.test_items
-        task_types: set[str] = set()
-        for item in all_items:
-            for name in item.get("theorem_type", []):
-                if name:
-                    task_types.add(name)
-        self._task_types = sorted(task_types)
-
-    def get_task_types(self) -> list[str]:
-        return list(self._task_types)
-
-    # ── Choice shuffling ─────────────────────────────────────────────────
-
-    @staticmethod
-    def _item_shuffle_seed(item_id: str, seed: int) -> int:
-        digest = hashlib.sha256(f"{seed}:{item_id}".encode("utf-8")).hexdigest()
-        return int(digest[:16], 16)
-
-    def _shuffle_item_choices(self, item: dict, seed: int) -> dict:
-        if not self.shuffle_choices:
-            return {
-                **item,
-                "choices": [dict(c) for c in item["choices"]],
-                "correct_choice": dict(item["correct_choice"]),
-            }
-
-        shuffled_choices = [dict(c) for c in item["choices"]]
-        rng = random.Random(self._item_shuffle_seed(str(item["id"]), seed))
-        rng.shuffle(shuffled_choices)
-
-        original_correct = _normalize_label(item["correct_choice"]["label"])
-        remapped_choices: list[dict] = []
-        new_correct_choice = dict(item["correct_choice"])
-
-        for idx, choice in enumerate(shuffled_choices):
-            new_label = _CHOICE_LABELS[idx]
-            old_label = _normalize_label(choice["label"])
-            remapped_choices.append({"label": new_label, "text": choice["text"]})
-            if old_label == original_correct:
-                new_correct_choice = {"label": new_label, "text": choice["text"]}
-
-        transformed = dict(item)
-        transformed["choices"] = remapped_choices
-        transformed["correct_choice"] = new_correct_choice
-        return transformed
-
-    def _materialize_batch(self, items: list[dict], seed: int) -> list[dict]:
-        return [self._shuffle_item_choices(item, seed) for item in items]
-
-    # ── Batch construction (override for choice shuffling) ───────────────
-
-    def plan_train_epoch(
-        self,
-        *,
-        epoch: int,
-        steps_per_epoch: int,
-        accumulation: int,
-        batch_size: int,
-        seed: int,
-        **kwargs,
-    ) -> list[BatchSpec]:
-        """Build a shuffled epoch while preserving per-batch choice shuffling."""
-        epoch_rng = random.Random(seed + epoch * 1000)
-        items = list(self.train_items)
-        epoch_rng.shuffle(items)
-
-        total_batches = steps_per_epoch * accumulation
-        if total_batches <= 0:
-            return []
-
-        batches: list[BatchSpec] = []
-        cursor = 0
-        for batch_idx in range(total_batches):
-            batch_seed = seed + epoch * 1000 + batch_idx + 1
-            batch_items = items[cursor: cursor + batch_size]
-            cursor += len(batch_items)
-
-            if not batch_items and items:
-                refill_rng = random.Random(batch_seed)
-                batch_items = list(items)
-                refill_rng.shuffle(batch_items)
-                batch_items = batch_items[:batch_size]
-
-            batch_items = self._materialize_batch(batch_items, batch_seed)
-            batches.append(
-                BatchSpec(
-                    phase="train",
-                    split="train",
-                    seed=batch_seed,
-                    batch_size=len(batch_items),
-                    payload=batch_items,
-                )
-            )
-
-        return batches
-
-    def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec:
-        rng = random.Random(seed)
-        items = list(self.train_items)
-        rng.shuffle(items)
-        items = self._materialize_batch(items[:batch_size], seed)
-        return BatchSpec(
-            phase="train",
-            split="train",
-            seed=seed,
-            batch_size=len(items),
-            payload=items,
-        )
-
-    def build_eval_batch(
-        self,
-        env_num: int,
-        split: str,
-        seed: int,
-        **kwargs,
-    ) -> BatchSpec:
-        items = self.get_split_items(split)
-        if env_num and env_num < len(items):
-            items = items[:env_num]
-        items = self._materialize_batch(items, seed)
-        return BatchSpec(
-            phase="eval",
-            split=split,
-            seed=seed,
-            batch_size=len(items),
-            payload=items,
-        )

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/livemathematicianbench/evaluator.py

@@ -1,62 +0,0 @@
-"""LiveMathematicianBench evaluation helpers."""
-from __future__ import annotations
-
-import re
-
-
-def extract_answer(text: str) -> str:
-    matches = re.findall(r"<answer>(.*?)</answer>", text, re.DOTALL | re.IGNORECASE)
-    if matches:
-        return matches[-1].strip()
-    lines = [ln.strip() for ln in text.strip().splitlines() if ln.strip()]
-    if lines:
-        return lines[-1]
-    return text.strip()
-
-
-def normalize_label(text: str) -> str:
-    return str(text).strip().upper().rstrip(".):")
-
-
-def parse_choice_label(prediction_text: str, choices: list[dict]) -> str:
-    answer = extract_answer(prediction_text)
-    label = normalize_label(answer)
-    valid_labels = {normalize_label(choice.get("label", "")) for choice in choices}
-    if label in valid_labels:
-        return label
-
-    answer_lower = answer.lower()
-    for choice in choices:
-        choice_label = normalize_label(choice.get("label", ""))
-        choice_text = str(choice.get("text", "")).strip()
-        if choice_text and choice_text.lower() == answer_lower:
-            return choice_label
-
-    first_token = normalize_label(answer.split()[0]) if answer.split() else ""
-    if first_token in valid_labels:
-        return first_token
-    return label
-
-
-def evaluate(prediction_text: str, correct_choice: dict, choices: list[dict]) -> dict:
-    predicted_label = parse_choice_label(prediction_text, choices)
-    correct_label = normalize_label(correct_choice.get("label", ""))
-    predicted_text = ""
-    correct_text = str(correct_choice.get("text", "")).strip()
-
-    for choice in choices:
-        if normalize_label(choice.get("label", "")) == predicted_label:
-            predicted_text = str(choice.get("text", "")).strip()
-            break
-
-    is_correct = float(predicted_label == correct_label)
-    return {
-        "em": is_correct,
-        "f1": is_correct,
-        "sub_em": is_correct,
-        "predicted_answer": predicted_label or extract_answer(prediction_text),
-        "predicted_label": predicted_label,
-        "predicted_text": predicted_text,
-        "correct_label": correct_label,
-        "correct_text": correct_text,
-    }

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/livemathematicianbench/prompts/analyst_error.md

@@ -1,37 +0,0 @@
-You are an expert failure-analysis agent for theorem-grounded mathematical multiple-choice questions.
-
-You will be given MULTIPLE failed trajectories from a single minibatch and the current skill document.
-Each trajectory includes the target's response and an evaluation result showing the predicted option
-versus the correct option.
-
-Your job is to identify COMMON reasoning failures across the batch and propose concise skill edits.
-
-## Failure Type Categories
-- **quantifier_miss**: the agent missed exact quantifiers, scope, or existence/uniqueness conditions
-- **strength_mismatch**: the agent preferred a weaker or stronger statement than what was proved
-- **condition_miss**: the agent ignored hypotheses, equality cases, or domain restrictions
-- **option_confusion**: the agent confused similar answer choices or failed to compare them exactly
-- **other**: none of the above
-
-## Rules
-1. Focus on patterns that recur across the minibatch.
-2. Prefer edits that improve exact choice discrimination, not theorem-specific memorization.
-3. Do not hardcode paper-specific content.
-4. Only patch gaps not already covered by the skill.
-
-Respond ONLY with a valid JSON object:
-{
-  "batch_size": <number>,
-  "failure_summary": [
-    {"failure_type": "<type>", "count": <int>, "description": "<one-line>"}
-  ],
-  "patch": {
-    "reasoning": "<why these edits address the common failures>",
-    "edits": [
-      {"op": "append",       "content": "<markdown>"},
-      {"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
-      {"op": "replace",      "target": "<old text>",     "content": "<new text>"},
-      {"op": "delete",       "target": "<exact text to remove>"}
-    ]
-  }
-}

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/livemathematicianbench/prompts/analyst_success.md

@@ -1,25 +0,0 @@
-You are an expert success-pattern analyst for theorem-grounded mathematical multiple-choice questions.
-
-You will be given MULTIPLE successful trajectories from a minibatch and the current skill document.
-Identify generalizable behavior patterns that are genuinely helping the agent choose the exact correct option.
-
-## Rules
-- Focus on broadly useful reasoning behaviors.
-- Prefer patterns about exact comparison of options, quantifiers, and equality conditions.
-- Do not add theorem-specific facts.
-- "edits" may be empty if the skill already captures the useful patterns.
-
-Respond ONLY with a valid JSON object:
-{
-  "batch_size": <number>,
-  "success_patterns": ["<pattern 1>", "<pattern 2>"],
-  "patch": {
-    "reasoning": "<why these patterns matter>",
-    "edits": [
-      {"op": "append",       "content": "<markdown>"},
-      {"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
-      {"op": "replace",      "target": "<old text>",     "content": "<new text>"},
-      {"op": "delete",       "target": "<exact text to remove>"}
-    ]
-  }
-}

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/livemathematicianbench/prompts/rollout_system.md

@@ -1,12 +0,0 @@
-You are an expert mathematical reasoning agent solving multiple-choice questions.
-
-{skill_section}## Task Format
-You will receive one mathematics multiple-choice question and its answer choices.
-Reason carefully about quantifiers, hypotheses, extremal wording, and exact equality conditions.
-
-## Answer Format
-Think step by step, then provide your final answer inside <answer>...</answer> tags.
-Inside the tags, output only the single choice label, such as A or C.
-
-Example:
-<answer>B</answer>

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/livemathematicianbench/reflect.py

@@ -1,4 +0,0 @@
-"""LiveMathematicianBench Reflect stage.
-
-Prompts are now loaded from .md files by the base adapter.
-"""