self-evolve-framework 1.3.0 → 1.4.0

package/template/skills/skillopt-sleep/scripts/python/mine.py

@@ -0,0 +1,312 @@
+"""SkillOpt-Sleep — Stage 2: mine.
+
+Turn :class:`SessionDigest` objects into :class:`TaskRecord` training units.
+
+Two miners:
+  * heuristic_mine  — deterministic, no API. Detects retry chains (a prompt
+    re-asked after negative feedback => the early attempt failed), extracts
+    the user's recurring intents, and labels outcomes from feedback signals.
+  * llm_mine        — optional; uses an optimizer backend to produce richer
+    TaskRecords with checkable references. Falls back to heuristic on error.
+
+The heuristic miner is what makes the whole cycle runnable offline and is the
+basis of the deterministic experiment.
+"""
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+from collections import Counter
+from typing import Any, Callable, List, Optional, Set, Tuple
+
+from skillopt_sleep.types import SessionDigest, TaskRecord
+
+
+def _tid(project: str, intent: str) -> str:
+    h = hashlib.sha256((project + "::" + intent).encode("utf-8")).hexdigest()[:12]
+    return "task_" + h
+
+
+def _short(text: str, n: int = 600) -> str:
+    text = (text or "").strip()
+    return text if len(text) <= n else text[:n] + " …"
+
+
+def _looks_negative(signals: List[str]) -> bool:
+    return any(s.startswith("neg:") for s in signals)
+
+
+def _looks_positive(signals: List[str]) -> bool:
+    return any(s.startswith("pos:") for s in signals)
+
+
+_TARGET_STOPWORDS = {
+    "about", "after", "again", "agent", "agents", "all", "also", "always",
+    "and", "any", "are", "before", "being", "but", "can", "codex",
+    "current", "default", "docs", "does", "done", "each", "file", "files",
+    "for", "from", "have", "into", "keep", "must", "not", "only", "path",
+    "paths", "project", "read", "repo", "request", "requests", "rule",
+    "rules", "same", "should", "skill", "skills", "source", "start",
+    "task", "tasks", "that", "the", "their", "then", "this", "unless",
+    "update", "user", "users", "when", "with", "work", "workflow",
+}
+
+
+def _target_tokens(text: str) -> List[str]:
+    tokens: List[str] = []
+    for raw in re.findall(r"[\w][\w.-]*", (text or "").lower(), flags=re.UNICODE):
+        parts = [raw] + re.split(r"[\W_]+", raw, flags=re.UNICODE)
+        for part in parts:
+            if len(part) < 3 or part.isdigit() or part in _TARGET_STOPWORDS:
+                continue
+            tokens.append(part)
+    return tokens
+
+
+def _expand_target_keywords(keywords: Set[str]) -> None:
+    if "mcp" in keywords:
+        keywords.update({
+            "configure", "configuration", "connect", "connected", "enable",
+            "enabled", "install", "installed", "server", "servers",
+            "настрой", "настроить", "подключи", "подключить",
+        })
+    if {"conflict", "conflicts"} & keywords:
+        keywords.update({
+            "cherry", "conflict", "conflicts", "git", "merge", "rebase",
+            "unmerged", "конфликт", "конфликты",
+        })
+
+
+def target_task_keywords(
+    target_skill_text: str,
+    target_skill_path: str = "",
+    *,
+    limit: int = 180,
+) -> Tuple[Set[str], Set[str]]:
+    """Return (strong, weak) keywords that describe a target skill."""
+    path_text = (target_skill_path or "").replace(os.sep, " ")
+    headings = "\n".join(re.findall(r"(?m)^#+\s+(.+)$", target_skill_text or ""))
+    strong = set(_target_tokens(path_text + "\n" + headings))
+    weak = set(strong)
+    counts = Counter(_target_tokens(target_skill_text or ""))
+    for token, _count in counts.most_common(limit):
+        weak.add(token)
+    _expand_target_keywords(strong)
+    _expand_target_keywords(weak)
+    return strong, weak
+
+
+def _task_search_text(task: TaskRecord) -> str:
+    return "\n".join([
+        task.intent or "",
+        task.context_excerpt or "",
+        " ".join(task.tags or []),
+    ])
+
+
+def filter_tasks_for_target(
+    tasks: List[TaskRecord],
+    target_skill_text: str,
+    target_skill_path: str = "",
+) -> List[TaskRecord]:
+    """Prefer tasks whose language overlaps the explicit target skill.
+
+    If nothing matches, return the original list. This keeps a target run useful
+    even when transcripts are too sparse or the skill is too generic.
+    """
+    strong, weak = target_task_keywords(target_skill_text, target_skill_path)
+    if not tasks or not (strong or weak):
+        return tasks
+
+    ranked = []
+    for idx, task in enumerate(tasks):
+        tokens = set(_target_tokens(_task_search_text(task)))
+        strong_hits = tokens & strong
+        weak_hits = tokens & weak
+        if not strong_hits and len(weak_hits) < 2:
+            continue
+        score = len(strong_hits) * 3 + len(weak_hits)
+        ranked.append((score, idx, task))
+    if not ranked:
+        return tasks
+    ranked.sort(key=lambda item: (-item[0], item[1]))
+    return [task for _score, _idx, task in ranked]
+
+
+def heuristic_mine(
+    digests: List[SessionDigest],
+    *,
+    max_tasks: int = 40,
+) -> List[TaskRecord]:
+    """Deterministic miner — no API calls.
+
+    Strategy:
+      * Each session with >=1 real user prompt yields one TaskRecord whose
+        intent is the FIRST substantive prompt (the original ask).
+      * Outcome is inferred:
+          - negative feedback present and no later positive  -> "fail"
+          - positive feedback present                         -> "success"
+          - re-asks (multiple user turns) without resolution  -> "mixed"
+          - otherwise                                         -> "unknown"
+      * attempted_solution = the last assistant final (what was produced).
+      * reference_kind defaults to "none"; the consolidation step will use a
+        rubric judge for these. (Exact refs are added by the experiment data
+        or by the LLM miner when it can derive a checkable answer.)
+    """
+    tasks: List[TaskRecord] = []
+    for d in digests:
+        if not d.user_prompts:
+            continue
+        intent = d.user_prompts[0]
+        if len(intent.strip()) < 8:
+            continue
+        if _looks_positive(d.feedback_signals) and not _looks_negative(d.feedback_signals):
+            outcome = "success"
+        elif _looks_negative(d.feedback_signals):
+            outcome = "fail"
+        elif d.n_user_turns >= 3:
+            outcome = "mixed"
+        else:
+            outcome = "unknown"
+
+        attempted = d.assistant_finals[-1] if d.assistant_finals else ""
+        context = ""
+        if len(d.user_prompts) > 1:
+            # later prompts often carry the corrective detail / real constraints
+            context = "Follow-up constraints from the same session:\n- " + "\n- ".join(
+                _short(p, 200) for p in d.user_prompts[1:4]
+            )
+        tags = []
+        if d.tools_used:
+            tags.append("tools:" + "+".join(d.tools_used[:4]))
+        if d.git_branch:
+            tags.append("branch:" + d.git_branch)
+
+        tasks.append(
+            TaskRecord(
+                id=_tid(d.project, intent),
+                project=d.project,
+                intent=_short(intent, 800),
+                context_excerpt=_short(context, 600),
+                attempted_solution=_short(attempted, 600),
+                outcome=outcome,
+                reference_kind="none",
+                reference="",
+                tags=tags,
+                source_sessions=[d.session_id],
+            )
+        )
+        if len(tasks) >= max_tasks:
+            break
+    return tasks
+
+
+def dedup_tasks(tasks: List[TaskRecord]) -> List[TaskRecord]:
+    """Merge tasks sharing an id (same project+intent across sessions)."""
+    by_id: dict = {}
+    for t in tasks:
+        if t.id in by_id:
+            ex = by_id[t.id]
+            ex.source_sessions = list(dict.fromkeys(ex.source_sessions + t.source_sessions))
+            # prefer a resolved outcome if either session resolved it
+            order = {"success": 3, "fail": 2, "mixed": 1, "unknown": 0}
+            if order.get(t.outcome, 0) > order.get(ex.outcome, 0):
+                ex.outcome = t.outcome
+        else:
+            by_id[t.id] = t
+    return list(by_id.values())
+
+
+def assign_splits(
+    tasks: List[TaskRecord],
+    *,
+    val_fraction: float = 0.34,
+    test_fraction: float = 0.0,
+    holdout_fraction: float | None = None,  # legacy alias for val_fraction
+    seed: int = 42,
+) -> List[TaskRecord]:
+    """Deterministically split tasks into train / val / test.
+
+    Anti-overfitting contract (the user's design):
+      * ``val`` and ``test`` are drawn ONLY from REAL mined tasks (origin=='real')
+        and never overlap. val gates updates; test is the final held-out measure.
+      * ``train`` may include DREAM-augmented tasks (origin=='dream'); those are
+        NEVER placed in val/test.
+
+    A stable hash of the task id keeps the same real task in the same split across
+    nights (a fixed held-out gate, like SkillOpt's D_sel/D_test).
+
+    Back-compat: if ``test_fraction`` is 0 (default), this behaves like the old
+    two-way replay/holdout split — real tasks divide into train + val, no test.
+    ``holdout_fraction`` is accepted as an alias for ``val_fraction``.
+    """
+    if holdout_fraction is not None:
+        val_fraction = holdout_fraction
+
+    dream = [t for t in tasks if t.origin == "dream"]
+    real = [t for t in tasks if t.origin != "dream"]
+
+    # all dream tasks go to train, unconditionally
+    for t in dream:
+        t.split = "train"
+
+    val_cut = int(round(val_fraction * 100))
+    test_cut = val_cut + int(round(test_fraction * 100))
+    for t in real:
+        bucket = int(hashlib.sha256((str(seed) + t.id).encode()).hexdigest(), 16) % 100
+        if bucket < val_cut:
+            t.split = "val"
+        elif bucket < test_cut:
+            t.split = "test"
+        else:
+            t.split = "train"
+
+    # guarantee val (the gate) is non-empty when we have >=2 real tasks
+    real_splits = {t.split for t in real}
+    if len(real) >= 2 and "val" not in real_splits:
+        real[-1].split = "val"
+    # guarantee a train pool exists (dream or real) when possible
+    if not any(t.split == "train" for t in tasks) and len(real) >= 2:
+        real[0].split = "train"
+    # if test was requested but ended up empty with >=3 real tasks, carve one
+    if test_fraction > 0 and len(real) >= 3 and not any(t.split == "test" for t in real):
+        for t in real:
+            if t.split == "train":
+                t.split = "test"
+                break
+    return tasks
+
+
+def normalize_legacy_split(value: str) -> str:
+    """Map old split names to the new vocabulary."""
+    return {"replay": "train", "holdout": "val"}.get(value, value)
+
+
+def mine(
+    digests: List[SessionDigest],
+    *,
+    max_tasks: int = 40,
+    candidate_limit: int = 0,
+    holdout_fraction: float = 0.34,
+    seed: int = 42,
+    llm_miner: Optional[Callable[[List[SessionDigest]], List[TaskRecord]]] = None,
+    target_skill_text: str = "",
+    target_skill_path: str = "",
+) -> List[TaskRecord]:
+    """Top-level miner. Uses ``llm_miner`` if provided, else heuristic."""
+    candidate_limit = candidate_limit or max_tasks
+    tasks: List[TaskRecord] = []
+    if llm_miner is not None:
+        try:
+            tasks = llm_miner(digests) or []
+        except Exception:
+            tasks = []
+    if not tasks:
+        tasks = heuristic_mine(digests, max_tasks=candidate_limit)
+    tasks = dedup_tasks(tasks)
+    if target_skill_text or target_skill_path:
+        tasks = filter_tasks_for_target(tasks, target_skill_text, target_skill_path)
+    tasks = tasks[:max_tasks]
+    tasks = assign_splits(tasks, holdout_fraction=holdout_fraction, seed=seed)
+    return tasks

package/template/skills/skillopt-sleep/scripts/python/replay.py

@@ -0,0 +1,146 @@
+"""SkillOpt-Sleep — Stage 3: replay.
+
+Re-run mined TaskRecords offline under a given (skill, memory) and score
+them, producing the (hard, soft) signal SkillOpt's gate consumes.
+
+Single-shot text replay by default. Tasks whose rule judge requires a tool
+call (gbrain's `tool_called`) are run through the backend's real tool loop
+(attempt_with_tools), so tool use is verified honestly rather than self-reported.
+"""
+from __future__ import annotations
+
+from typing import List, Tuple
+
+from skillopt_sleep.backend import Backend
+from skillopt_sleep.types import ReplayResult, TaskRecord
+
+
+def _required_tools(task: TaskRecord) -> List[str]:
+    """Tool names a rule judge requires (op == 'tool_called')."""
+    if task.reference_kind != "rule" or not task.judge:
+        return []
+    tools = []
+    for c in task.judge.get("checks", []) or []:
+        if isinstance(c, dict) and c.get("op") == "tool_called" and c.get("arg"):
+            tools.append(str(c["arg"]))
+    return tools
+
+
+def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str,
+               sample_id: int = 0) -> ReplayResult:
+    """``sample_id`` distinguishes repeated dream rollouts of the same
+    (task, skill, memory) in the attempt cache — without it all K rollouts
+    collapse to one cached response and the contrastive signal is always 0."""
+    import time
+    tools = _required_tools(task)
+    tools_called: List[str] = []
+    t0 = time.time()
+    tok_before = backend.tokens_used()
+    if tools:
+        response, tools_called = backend.attempt_with_tools(task, skill, memory, tools)
+    else:
+        response = backend.attempt(task, skill, memory, sample_id=sample_id)
+    latency_ms = (time.time() - t0) * 1000.0
+    tokens = max(0, backend.tokens_used() - tok_before)
+    # if the backend doesn't track tokens (e.g. mock), approximate from text length
+    if tokens == 0:
+        tokens = (len(skill) + len(memory) + len(task.intent) + len(response)) // 4
+
+    # rule judges may need the detected tool calls; score locally when possible
+    if task.reference_kind == "rule" and task.judge:
+        from skillopt_sleep.judges import score_rule_judge
+        hard, soft, rationale = score_rule_judge(task.judge, response, tools_called)
+    else:
+        hard, soft, rationale = backend.judge(task, response)
+
+    return ReplayResult(
+        id=task.id,
+        hard=float(hard),
+        soft=float(soft),
+        response=response,
+        fail_reason="" if hard >= 1.0 else (rationale or "below threshold"),
+        task_type=(task.tags[0] if task.tags else "task"),
+        judge_rationale=rationale,
+        tools_called=tools_called,
+        tokens=int(tokens),
+        latency_ms=round(latency_ms, 1),
+    )
+
+
+import os
+from concurrent.futures import ThreadPoolExecutor
+
+
+def replay_batch(
+    backend: Backend,
+    tasks: List[TaskRecord],
+    skill: str,
+    memory: str,
+    *,
+    workers: int = 0,
+) -> List[Tuple[TaskRecord, ReplayResult]]:
+    """Replay tasks, optionally in parallel.
+
+    Real backends are network-bound, so a thread pool gives a large speedup on
+    big test sets (like the research harness's --workers). ``workers`` defaults
+    to env SKILLOPT_SLEEP_WORKERS or 1 (sequential). Mock stays sequential
+    (deterministic) unless asked otherwise.
+    """
+    if workers <= 0:
+        workers = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1") or "1")
+    if workers <= 1 or len(tasks) <= 1:
+        return [(t, replay_one(backend, t, skill, memory)) for t in tasks]
+    results: List = [None] * len(tasks)
+    with ThreadPoolExecutor(max_workers=min(workers, len(tasks))) as ex:
+        futs = {ex.submit(replay_one, backend, t, skill, memory): i
+                for i, t in enumerate(tasks)}
+        for fut in futs:
+            i = futs[fut]
+            results[i] = (tasks[i], fut.result())
+    return results
+
+
+def aggregate_scores(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]:
+    if not pairs:
+        return 0.0, 0.0
+    hard = sum(r.hard for _t, r in pairs) / len(pairs)
+    soft = sum(r.soft for _t, r in pairs) / len(pairs)
+    return hard, soft
+
+
+def aggregate_cost(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]:
+    """Mean (tokens, latency_ms) per task — the cost objectives."""
+    if not pairs:
+        return 0.0, 0.0
+    tok = sum(r.tokens for _t, r in pairs) / len(pairs)
+    lat = sum(r.latency_ms for _t, r in pairs) / len(pairs)
+    return tok, lat
+
+
+def multi_objective_reward(
+    pairs: List[Tuple[TaskRecord, ReplayResult]],
+    *,
+    w_acc: float = 1.0,
+    w_tokens: float = 0.0,
+    w_latency: float = 0.0,
+    token_ref: float = 2000.0,
+    latency_ref_ms: float = 15000.0,
+) -> float:
+    """Weighted reward = accuracy↑, tokens↓, latency↓.
+
+    Cost terms are normalized against a reference and clamped to [0,1], so a
+    response at/under the reference cost contributes ~1.0 and an expensive one
+    less. Weights let the user trade off (default = accuracy only, backward
+    compatible).
+    """
+    if not pairs:
+        return 0.0
+    acc, _soft = aggregate_scores(pairs)
+    tok, lat = aggregate_cost(pairs)
+    tok_score = max(0.0, 1.0 - tok / max(1.0, token_ref)) if token_ref else 0.0
+    lat_score = max(0.0, 1.0 - lat / max(1.0, latency_ref_ms)) if latency_ref_ms else 0.0
+    total_w = w_acc + w_tokens + w_latency
+    if total_w <= 0:
+        return acc
+    return (w_acc * acc + w_tokens * tok_score + w_latency * lat_score) / total_w
+

package/template/skills/skillopt-sleep/scripts/python/rollout.py

@@ -0,0 +1,153 @@
+"""SkillOpt-Sleep — multi-rollout + contrastive reflection (the imagination core).
+
+The core idea: let the agent re-run the SAME task many times, then look at
+which rollouts went well vs badly and distill a rule from the *contrast*. This
+is a much stronger learning signal than a single failure, and it is the essence
+of the offline "dream/imagination" process — train-time rollouts are synthetic,
+so doing many is fine.
+
+Pieces:
+  * multi_rollout   — run one task K times under (skill, memory), return scored attempts
+  * contrastive_reflect — given good vs bad attempts of the same tasks, ask the
+    optimizer what distinguishes them and propose a general rule
+
+Driven through the Backend abstraction (mock/claude/codex), import-light.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Tuple
+
+from skillopt_sleep.backend import Backend, _extract_json
+from skillopt_sleep.replay import replay_one
+from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord
+
+
+@dataclass
+class RolloutSet:
+    """K scored attempts at one task under a fixed (skill, memory)."""
+    task: TaskRecord
+    attempts: List[ReplayResult] = field(default_factory=list)
+
+    @property
+    def best(self) -> Optional[ReplayResult]:
+        return max(self.attempts, key=lambda r: r.hard, default=None)
+
+    @property
+    def worst(self) -> Optional[ReplayResult]:
+        return min(self.attempts, key=lambda r: r.hard, default=None)
+
+    @property
+    def spread(self) -> float:
+        if not self.attempts:
+            return 0.0
+        hs = [r.hard for r in self.attempts]
+        return max(hs) - min(hs)
+
+    @property
+    def pass_rate(self) -> float:
+        if not self.attempts:
+            return 0.0
+        return sum(1 for r in self.attempts if r.hard >= 1.0) / len(self.attempts)
+
+
+def multi_rollout(
+    backend: Backend,
+    task: TaskRecord,
+    skill: str,
+    memory: str,
+    *,
+    k: int = 3,
+    workers: int = 0,
+) -> RolloutSet:
+    """Run ``task`` K times. replay_one is deterministic for mock; for real
+    backends the model's own sampling yields variation across attempts.
+
+    The K attempts are independent, so they run concurrently (this is the dream
+    phase's dominant cost). ``workers`` defaults to the SKILLOPT_SLEEP_WORKERS
+    env (capped at k); set to 1 to force serial (used by the mock tests).
+    """
+    import os
+    rs = RolloutSet(task=task)
+    k = max(1, k)
+    if workers <= 0:
+        try:
+            workers = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1"))
+        except ValueError:
+            workers = 1
+    workers = max(1, min(workers, k))
+    if workers == 1:
+        for i in range(k):
+            rs.attempts.append(replay_one(backend, task, skill, memory, sample_id=i))
+        return rs
+    from concurrent.futures import ThreadPoolExecutor
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        futs = [ex.submit(replay_one, backend, task, skill, memory, sample_id=i)
+                for i in range(k)]
+        for f in futs:
+            rs.attempts.append(f.result())
+    return rs
+
+
+def contrastive_reflect(
+    backend: Backend,
+    rollout_sets: List[RolloutSet],
+    skill: str,
+    memory: str,
+    *,
+    edit_budget: int = 4,
+    target: str = "skill",
+) -> List[EditRecord]:
+    """Distill a rule from the contrast between good and bad attempts.
+
+    We pick tasks with the highest score *spread* (some attempts passed, some
+    failed) — those are the most informative — and show the optimizer a
+    high-scoring vs a low-scoring attempt of each, asking what general rule makes
+    the good behavior reliable.
+    """
+    informative = [rs for rs in rollout_sets if rs.spread > 0 and rs.best and rs.worst]
+    informative.sort(key=lambda rs: rs.spread, reverse=True)
+    informative = informative[:6]
+    if not informative:
+        return []
+
+    blocks = []
+    for rs in informative:
+        blocks.append(
+            f"## Task: {rs.task.intent[:160]}\n"
+            f"- GOOD attempt (score {rs.best.hard:.1f}): {rs.best.response[:200]}\n"
+            f"- BAD  attempt (score {rs.worst.hard:.1f}): {rs.worst.response[:200]}\n"
+            f"  (bad failed: {rs.worst.fail_reason[:100]})"
+        )
+    # the output contract the proposed rules must not violate (same guardrail the
+    # single-shot reflect uses — prevents harness-violating rules like "return VBA"
+    # or "ask the user for the range" on SpreadsheetBench).
+    from skillopt_sleep.backend import _task_guardrail
+    guard = _task_guardrail([(rs.task, rs.best) for rs in informative])
+    prompt = (
+        "You are SkillOpt's optimizer doing CONTRASTIVE reflection. For each task "
+        "below the agent was run multiple times; some attempts succeeded and some "
+        "failed. Identify what the GOOD attempts did that the BAD ones did not, "
+        f"and propose at most {edit_budget} SHORT, GENERAL, reusable rules for the "
+        f"{target} that would make the good behavior reliable every time. Quote "
+        "concrete thresholds/formats verbatim; do not paraphrase vaguely. "
+        "Every rule MUST obey the task output contract (if shown) — never propose "
+        "a rule that changes the required output format/language or tells the agent "
+        "to ask the user a question; such a rule scores ZERO.\n"
+        f"{guard}"
+        'Return ONLY a JSON array: '
+        '[{"op":"add","content":"<rule>","rationale":"<what good did that bad didnt>"}].\n\n'
+        + "\n\n".join(blocks)
+    )
+    raw = backend._call(prompt, max_tokens=1024)  # type: ignore[attr-defined]
+    arr = _extract_json(raw, "array")
+    edits: List[EditRecord] = []
+    if isinstance(arr, list):
+        for e in arr[:edit_budget]:
+            if isinstance(e, dict) and str(e.get("content", "")).strip():
+                edits.append(EditRecord(
+                    target=target, op=str(e.get("op", "add")).strip().lower(),
+                    content=str(e["content"]).strip(),
+                    rationale=str(e.get("rationale", "")).strip(),
+                ))
+    return edits