self-evolve-framework 1.6.0 → 1.6.1

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/dataloader.py

@@ -1,42 +0,0 @@
-"""SearchQA task dataloader."""
-from __future__ import annotations
-
-import json
-
-from skillopt.datasets.base import SplitDataLoader
-
-
-# ── Raw data loading utilities (for preprocessing / standalone eval) ─────
-
-def _load_items(path: str) -> list[dict]:
-    """Load items from JSON or JSONL file."""
-    with open(path) as f:
-        content = f.read().strip()
-    try:
-        data = json.loads(content)
-        if isinstance(data, list):
-            return data
-        if isinstance(data, dict):
-            return data.get("data") or list(data.values())
-    except json.JSONDecodeError:
-        pass
-
-    items = []
-    for line in content.splitlines():
-        line = line.strip()
-        if line:
-            items.append(json.loads(line))
-    return items
-
-
-# ── Dataloader ───────────────────────────────────────────────────────────
-
-class SearchQADataLoader(SplitDataLoader):
-    """SearchQA dataloader.
-
-    Each split directory (train/, val/, test/) contains a .json file —
-    a JSON array of question items.
-    """
-
-    def load_raw_items(self, data_path: str) -> list[dict]:
-        return _load_items(data_path)

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/evaluator.py

@@ -1,100 +0,0 @@
-"""SearchQA evaluation metrics: Exact Match, F1, and Substring Match.
-
-Normalization follows the SQuAD convention:
-  - lowercase
-  - remove punctuation
-  - remove articles (a, an, the)
-  - collapse whitespace
-
-Answer extraction looks for <answer>...</answer> XML tags,
-falling back to the last non-empty line of the response.
-"""
-from __future__ import annotations
-
-import re
-import string
-from collections import Counter
-
-
-def normalize_answer(s: str) -> str:
-    """Normalize answer string (SQuAD convention)."""
-    s = s.lower()
-    s = "".join(ch for ch in s if ch not in string.punctuation)
-    s = re.sub(r"\b(a|an|the)\b", " ", s)
-    s = " ".join(s.split())
-    return s.strip()
-
-
-def extract_answer(text: str) -> str:
-    """Extract answer from <answer>...</answer> tags.
-
-    Fallback: last non-empty line, then full response stripped.
-    """
-    matches = re.findall(r"<answer>(.*?)</answer>", text, re.DOTALL | re.IGNORECASE)
-    if matches:
-        return matches[-1].strip()
-    lines = [ln.strip() for ln in text.strip().splitlines() if ln.strip()]
-    if lines:
-        return lines[-1]
-    return text.strip()
-
-
-def exact_match(prediction: str, gold_answers: list[str]) -> float:
-    norm_pred = normalize_answer(prediction)
-    for gold in gold_answers:
-        if normalize_answer(gold) == norm_pred:
-            return 1.0
-    return 0.0
-
-
-def f1_score(prediction: str, gold_answers: list[str]) -> float:
-    """Token-level F1 (SQuAD-style), max across all gold answers."""
-    norm_pred = normalize_answer(prediction)
-    pred_tokens = norm_pred.split()
-
-    if not pred_tokens:
-        for gold in gold_answers:
-            if not normalize_answer(gold).split():
-                return 1.0
-        return 0.0
-
-    best_f1 = 0.0
-    for gold in gold_answers:
-        gold_tokens = normalize_answer(gold).split()
-        if not gold_tokens:
-            continue
-        common = Counter(pred_tokens) & Counter(gold_tokens)
-        n_common = sum(common.values())
-        if n_common == 0:
-            continue
-        precision = n_common / len(pred_tokens)
-        recall = n_common / len(gold_tokens)
-        f1 = 2 * precision * recall / (precision + recall)
-        best_f1 = max(best_f1, f1)
-
-    return best_f1
-
-
-def sub_em(prediction: str, gold_answers: list[str]) -> float:
-    """1.0 if any normalized gold is a substring of prediction, or vice versa."""
-    norm_pred = normalize_answer(prediction)
-    for gold in gold_answers:
-        norm_gold = normalize_answer(gold)
-        if norm_gold in norm_pred or norm_pred in norm_gold:
-            return 1.0
-    return 0.0
-
-
-def evaluate(prediction_text: str, gold_answers: list[str]) -> dict:
-    """Evaluate a single QA prediction against gold answers.
-
-    Returns dict with: em, f1, sub_em, predicted_answer, gold_answers.
-    """
-    answer = extract_answer(prediction_text)
-    return {
-        "em": exact_match(answer, gold_answers),
-        "f1": f1_score(answer, gold_answers),
-        "sub_em": sub_em(answer, gold_answers),
-        "predicted_answer": answer,
-        "gold_answers": gold_answers,
-    }

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/prompts/analyst_error.md

@@ -1,46 +0,0 @@
-You are an expert failure-analysis agent for question answering tasks.
-
-You will be given MULTIPLE failed QA agent responses from a single minibatch
-and the current skill document. Each trajectory includes the agent's response
-and an evaluation result showing the predicted answer vs. the gold answer(s).
-
-Your job is to identify the most important COMMON failure patterns across
-the batch and propose a concise set of skill edits.
-
-## Failure Type Categories
-- **rule_missing**: the skill lacks a relevant rule for this type of question
-- **rule_wrong**: an existing skill rule is misleading or incorrect
-- **rule_ignored**: the skill has the right rule but the agent did not follow it
-- **answer_format**: the agent found the right information but formatted it incorrectly
-- **other**: none of the above
-
-## Analysis Process
-1. Read ALL failed trajectories in the minibatch.
-2. Carefully compare each predicted answer against the gold answer(s) —
-   understand exactly WHY the Exact Match failed.
-3. Identify the most prevalent, systematic failure patterns across them.
-4. For each pattern, classify its failure type.
-5. Propose skill edits that address the COMMON patterns — not individual edge cases.
-6. Edits must be generalizable; do not hardcode question-specific values.
-7. Only patch gaps in the skill — do not duplicate existing content.
-
-You will be told the maximum number of edits (the budget L). Produce AT MOST L edits,
-focusing on the highest-impact patterns. You may produce fewer if warranted.
-
-Respond ONLY with a valid JSON object (no markdown fences, no extra text):
-{
-  "batch_size": <number of trajectories analysed>,
-  "failure_summary": [
-    {"failure_type": "<type>", "count": <int>, "description": "<one-line>"}
-  ],
-  "patch": {
-    "reasoning": "<why these edits address the batch's common failures>",
-    "edits": [
-      {"op": "append",       "content": "<markdown to add at end of skill>"},
-      {"op": "insert_after", "target": "<exact heading/text to insert after>", "content": "<markdown>"},
-      {"op": "replace",      "target": "<exact text to replace>",              "content": "<replacement>"},
-      {"op": "delete",       "target": "<exact text to remove>"}
-    ]
-  }
-}
-Only include edits that are needed. "edits" can be an empty list if no patch is warranted.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/prompts/analyst_success.md

@@ -1,32 +0,0 @@
-You are an expert success-pattern analyst for AI question answering agents.
-
-You will be given MULTIPLE successful QA agent responses from a single minibatch
-and the current skill document. Your job is to identify generalizable behavior
-patterns that are COMMON across the batch and worth encoding in the skill.
-
-## Rules
-- Only propose patches for patterns NOT already covered in the skill.
-- Focus on patterns that appear across MULTIPLE trajectories in the batch.
-- Be concise. Patterns must generalize beyond specific questions.
-- Prefer reinforcing existing sections over adding new top-level sections.
-- If the agents' success involved a smart reading strategy or disambiguation
-  approach, consider reinforcing that in the patch.
-
-You will be told the maximum number of edits (the budget L). Produce AT MOST L edits,
-focusing on the most broadly applicable patterns. You may produce fewer if warranted.
-
-Respond ONLY with a valid JSON object:
-{
-  "batch_size": <number of trajectories analysed>,
-  "success_patterns": ["<pattern 1>", "<pattern 2>"],
-  "patch": {
-    "reasoning": "<why these patterns are worth encoding>",
-    "edits": [
-      {"op": "append",       "content": "<markdown>"},
-      {"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
-      {"op": "replace",      "target": "<old text>",     "content": "<new text>"},
-      {"op": "delete",       "target": "<exact text to remove>"}
-    ]
-  }
-}
-"edits" may be empty if the skill already covers all observed patterns.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/prompts/rollout_system.md

@@ -1,13 +0,0 @@
-You are an expert question answering agent.
-
-{skill_section}## Task Format
-You will receive a CONTEXT containing document passages and a QUESTION.
-Read the context carefully and answer the question based on the information provided.
-
-## Answer Format
-Think step by step, then provide your final answer inside <answer>...</answer> tags.
-Keep your answer concise — typically a few words or a short phrase.
-Do not repeat the question. Do not include unnecessary explanation in the answer tags.
-
-Example:
-<answer>Abraham Lincoln</answer>

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/reflect.py

@@ -1,4 +0,0 @@
-"""SearchQA Reflect stage.
-
-Prompts are now loaded from .md files by the base adapter.
-"""