self-evolve-framework 1.5.0 → 1.6.0

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/dataloader.py

@@ -0,0 +1,42 @@
+"""SearchQA task dataloader."""
+from __future__ import annotations
+
+import json
+
+from skillopt.datasets.base import SplitDataLoader
+
+
+# ── Raw data loading utilities (for preprocessing / standalone eval) ─────
+
+def _load_items(path: str) -> list[dict]:
+    """Load items from JSON or JSONL file."""
+    with open(path) as f:
+        content = f.read().strip()
+    try:
+        data = json.loads(content)
+        if isinstance(data, list):
+            return data
+        if isinstance(data, dict):
+            return data.get("data") or list(data.values())
+    except json.JSONDecodeError:
+        pass
+
+    items = []
+    for line in content.splitlines():
+        line = line.strip()
+        if line:
+            items.append(json.loads(line))
+    return items
+
+
+# ── Dataloader ───────────────────────────────────────────────────────────
+
+class SearchQADataLoader(SplitDataLoader):
+    """SearchQA dataloader.
+
+    Each split directory (train/, val/, test/) contains a .json file —
+    a JSON array of question items.
+    """
+
+    def load_raw_items(self, data_path: str) -> list[dict]:
+        return _load_items(data_path)

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/evaluator.py

@@ -0,0 +1,100 @@
+"""SearchQA evaluation metrics: Exact Match, F1, and Substring Match.
+
+Normalization follows the SQuAD convention:
+  - lowercase
+  - remove punctuation
+  - remove articles (a, an, the)
+  - collapse whitespace
+
+Answer extraction looks for <answer>...</answer> XML tags,
+falling back to the last non-empty line of the response.
+"""
+from __future__ import annotations
+
+import re
+import string
+from collections import Counter
+
+
+def normalize_answer(s: str) -> str:
+    """Normalize answer string (SQuAD convention)."""
+    s = s.lower()
+    s = "".join(ch for ch in s if ch not in string.punctuation)
+    s = re.sub(r"\b(a|an|the)\b", " ", s)
+    s = " ".join(s.split())
+    return s.strip()
+
+
+def extract_answer(text: str) -> str:
+    """Extract answer from <answer>...</answer> tags.
+
+    Fallback: last non-empty line, then full response stripped.
+    """
+    matches = re.findall(r"<answer>(.*?)</answer>", text, re.DOTALL | re.IGNORECASE)
+    if matches:
+        return matches[-1].strip()
+    lines = [ln.strip() for ln in text.strip().splitlines() if ln.strip()]
+    if lines:
+        return lines[-1]
+    return text.strip()
+
+
+def exact_match(prediction: str, gold_answers: list[str]) -> float:
+    norm_pred = normalize_answer(prediction)
+    for gold in gold_answers:
+        if normalize_answer(gold) == norm_pred:
+            return 1.0
+    return 0.0
+
+
+def f1_score(prediction: str, gold_answers: list[str]) -> float:
+    """Token-level F1 (SQuAD-style), max across all gold answers."""
+    norm_pred = normalize_answer(prediction)
+    pred_tokens = norm_pred.split()
+
+    if not pred_tokens:
+        for gold in gold_answers:
+            if not normalize_answer(gold).split():
+                return 1.0
+        return 0.0
+
+    best_f1 = 0.0
+    for gold in gold_answers:
+        gold_tokens = normalize_answer(gold).split()
+        if not gold_tokens:
+            continue
+        common = Counter(pred_tokens) & Counter(gold_tokens)
+        n_common = sum(common.values())
+        if n_common == 0:
+            continue
+        precision = n_common / len(pred_tokens)
+        recall = n_common / len(gold_tokens)
+        f1 = 2 * precision * recall / (precision + recall)
+        best_f1 = max(best_f1, f1)
+
+    return best_f1
+
+
+def sub_em(prediction: str, gold_answers: list[str]) -> float:
+    """1.0 if any normalized gold is a substring of prediction, or vice versa."""
+    norm_pred = normalize_answer(prediction)
+    for gold in gold_answers:
+        norm_gold = normalize_answer(gold)
+        if norm_gold in norm_pred or norm_pred in norm_gold:
+            return 1.0
+    return 0.0
+
+
+def evaluate(prediction_text: str, gold_answers: list[str]) -> dict:
+    """Evaluate a single QA prediction against gold answers.
+
+    Returns dict with: em, f1, sub_em, predicted_answer, gold_answers.
+    """
+    answer = extract_answer(prediction_text)
+    return {
+        "em": exact_match(answer, gold_answers),
+        "f1": f1_score(answer, gold_answers),
+        "sub_em": sub_em(answer, gold_answers),
+        "predicted_answer": answer,
+        "gold_answers": gold_answers,
+    }

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/prompts/analyst_error.md

@@ -0,0 +1,46 @@
+You are an expert failure-analysis agent for question answering tasks.
+
+You will be given MULTIPLE failed QA agent responses from a single minibatch
+and the current skill document. Each trajectory includes the agent's response
+and an evaluation result showing the predicted answer vs. the gold answer(s).
+
+Your job is to identify the most important COMMON failure patterns across
+the batch and propose a concise set of skill edits.
+
+## Failure Type Categories
+- **rule_missing**: the skill lacks a relevant rule for this type of question
+- **rule_wrong**: an existing skill rule is misleading or incorrect
+- **rule_ignored**: the skill has the right rule but the agent did not follow it
+- **answer_format**: the agent found the right information but formatted it incorrectly
+- **other**: none of the above
+
+## Analysis Process
+1. Read ALL failed trajectories in the minibatch.
+2. Carefully compare each predicted answer against the gold answer(s) —
+   understand exactly WHY the Exact Match failed.
+3. Identify the most prevalent, systematic failure patterns across them.
+4. For each pattern, classify its failure type.
+5. Propose skill edits that address the COMMON patterns — not individual edge cases.
+6. Edits must be generalizable; do not hardcode question-specific values.
+7. Only patch gaps in the skill — do not duplicate existing content.
+
+You will be told the maximum number of edits (the budget L). Produce AT MOST L edits,
+focusing on the highest-impact patterns. You may produce fewer if warranted.
+
+Respond ONLY with a valid JSON object (no markdown fences, no extra text):
+{
+  "batch_size": <number of trajectories analysed>,
+  "failure_summary": [
+    {"failure_type": "<type>", "count": <int>, "description": "<one-line>"}
+  ],
+  "patch": {
+    "reasoning": "<why these edits address the batch's common failures>",
+    "edits": [
+      {"op": "append",       "content": "<markdown to add at end of skill>"},
+      {"op": "insert_after", "target": "<exact heading/text to insert after>", "content": "<markdown>"},
+      {"op": "replace",      "target": "<exact text to replace>",              "content": "<replacement>"},
+      {"op": "delete",       "target": "<exact text to remove>"}
+    ]
+  }
+}
+Only include edits that are needed. "edits" can be an empty list if no patch is warranted.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/prompts/analyst_success.md

@@ -0,0 +1,32 @@
+You are an expert success-pattern analyst for AI question answering agents.
+
+You will be given MULTIPLE successful QA agent responses from a single minibatch
+and the current skill document. Your job is to identify generalizable behavior
+patterns that are COMMON across the batch and worth encoding in the skill.
+
+## Rules
+- Only propose patches for patterns NOT already covered in the skill.
+- Focus on patterns that appear across MULTIPLE trajectories in the batch.
+- Be concise. Patterns must generalize beyond specific questions.
+- Prefer reinforcing existing sections over adding new top-level sections.
+- If the agents' success involved a smart reading strategy or disambiguation
+  approach, consider reinforcing that in the patch.
+
+You will be told the maximum number of edits (the budget L). Produce AT MOST L edits,
+focusing on the most broadly applicable patterns. You may produce fewer if warranted.
+
+Respond ONLY with a valid JSON object:
+{
+  "batch_size": <number of trajectories analysed>,
+  "success_patterns": ["<pattern 1>", "<pattern 2>"],
+  "patch": {
+    "reasoning": "<why these patterns are worth encoding>",
+    "edits": [
+      {"op": "append",       "content": "<markdown>"},
+      {"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
+      {"op": "replace",      "target": "<old text>",     "content": "<new text>"},
+      {"op": "delete",       "target": "<exact text to remove>"}
+    ]
+  }
+}
+"edits" may be empty if the skill already covers all observed patterns.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/prompts/rollout_system.md

@@ -0,0 +1,13 @@
+You are an expert question answering agent.
+
+{skill_section}## Task Format
+You will receive a CONTEXT containing document passages and a QUESTION.
+Read the context carefully and answer the question based on the information provided.
+
+## Answer Format
+Think step by step, then provide your final answer inside <answer>...</answer> tags.
+Keep your answer concise — typically a few words or a short phrase.
+Do not repeat the question. Do not include unnecessary explanation in the answer tags.
+
+Example:
+<answer>Abraham Lincoln</answer>

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/searchqa/reflect.py

@@ -0,0 +1,4 @@
+"""SearchQA Reflect stage.
+
+Prompts are now loaded from .md files by the base adapter.
+"""