self-evolve-framework 1.6.0 → 1.6.1

package/template/skills/skillopt-sleep/scripts/python/mine.py

@@ -1,312 +0,0 @@
-"""SkillOpt-Sleep — Stage 2: mine.
-
-Turn :class:`SessionDigest` objects into :class:`TaskRecord` training units.
-
-Two miners:
-  * heuristic_mine  — deterministic, no API. Detects retry chains (a prompt
-    re-asked after negative feedback => the early attempt failed), extracts
-    the user's recurring intents, and labels outcomes from feedback signals.
-  * llm_mine        — optional; uses an optimizer backend to produce richer
-    TaskRecords with checkable references. Falls back to heuristic on error.
-
-The heuristic miner is what makes the whole cycle runnable offline and is the
-basis of the deterministic experiment.
-"""
-from __future__ import annotations
-
-import hashlib
-import os
-import re
-from collections import Counter
-from typing import Any, Callable, List, Optional, Set, Tuple
-
-from skillopt_sleep.types import SessionDigest, TaskRecord
-
-
-def _tid(project: str, intent: str) -> str:
-    h = hashlib.sha256((project + "::" + intent).encode("utf-8")).hexdigest()[:12]
-    return "task_" + h
-
-
-def _short(text: str, n: int = 600) -> str:
-    text = (text or "").strip()
-    return text if len(text) <= n else text[:n] + " …"
-
-
-def _looks_negative(signals: List[str]) -> bool:
-    return any(s.startswith("neg:") for s in signals)
-
-
-def _looks_positive(signals: List[str]) -> bool:
-    return any(s.startswith("pos:") for s in signals)
-
-
-_TARGET_STOPWORDS = {
-    "about", "after", "again", "agent", "agents", "all", "also", "always",
-    "and", "any", "are", "before", "being", "but", "can", "codex",
-    "current", "default", "docs", "does", "done", "each", "file", "files",
-    "for", "from", "have", "into", "keep", "must", "not", "only", "path",
-    "paths", "project", "read", "repo", "request", "requests", "rule",
-    "rules", "same", "should", "skill", "skills", "source", "start",
-    "task", "tasks", "that", "the", "their", "then", "this", "unless",
-    "update", "user", "users", "when", "with", "work", "workflow",
-}
-
-
-def _target_tokens(text: str) -> List[str]:
-    tokens: List[str] = []
-    for raw in re.findall(r"[\w][\w.-]*", (text or "").lower(), flags=re.UNICODE):
-        parts = [raw] + re.split(r"[\W_]+", raw, flags=re.UNICODE)
-        for part in parts:
-            if len(part) < 3 or part.isdigit() or part in _TARGET_STOPWORDS:
-                continue
-            tokens.append(part)
-    return tokens
-
-
-def _expand_target_keywords(keywords: Set[str]) -> None:
-    if "mcp" in keywords:
-        keywords.update({
-            "configure", "configuration", "connect", "connected", "enable",
-            "enabled", "install", "installed", "server", "servers",
-            "настрой", "настроить", "подключи", "подключить",
-        })
-    if {"conflict", "conflicts"} & keywords:
-        keywords.update({
-            "cherry", "conflict", "conflicts", "git", "merge", "rebase",
-            "unmerged", "конфликт", "конфликты",
-        })
-
-
-def target_task_keywords(
-    target_skill_text: str,
-    target_skill_path: str = "",
-    *,
-    limit: int = 180,
-) -> Tuple[Set[str], Set[str]]:
-    """Return (strong, weak) keywords that describe a target skill."""
-    path_text = (target_skill_path or "").replace(os.sep, " ")
-    headings = "\n".join(re.findall(r"(?m)^#+\s+(.+)$", target_skill_text or ""))
-    strong = set(_target_tokens(path_text + "\n" + headings))
-    weak = set(strong)
-    counts = Counter(_target_tokens(target_skill_text or ""))
-    for token, _count in counts.most_common(limit):
-        weak.add(token)
-    _expand_target_keywords(strong)
-    _expand_target_keywords(weak)
-    return strong, weak
-
-
-def _task_search_text(task: TaskRecord) -> str:
-    return "\n".join([
-        task.intent or "",
-        task.context_excerpt or "",
-        " ".join(task.tags or []),
-    ])
-
-
-def filter_tasks_for_target(
-    tasks: List[TaskRecord],
-    target_skill_text: str,
-    target_skill_path: str = "",
-) -> List[TaskRecord]:
-    """Prefer tasks whose language overlaps the explicit target skill.
-
-    If nothing matches, return the original list. This keeps a target run useful
-    even when transcripts are too sparse or the skill is too generic.
-    """
-    strong, weak = target_task_keywords(target_skill_text, target_skill_path)
-    if not tasks or not (strong or weak):
-        return tasks
-
-    ranked = []
-    for idx, task in enumerate(tasks):
-        tokens = set(_target_tokens(_task_search_text(task)))
-        strong_hits = tokens & strong
-        weak_hits = tokens & weak
-        if not strong_hits and len(weak_hits) < 2:
-            continue
-        score = len(strong_hits) * 3 + len(weak_hits)
-        ranked.append((score, idx, task))
-    if not ranked:
-        return tasks
-    ranked.sort(key=lambda item: (-item[0], item[1]))
-    return [task for _score, _idx, task in ranked]
-
-
-def heuristic_mine(
-    digests: List[SessionDigest],
-    *,
-    max_tasks: int = 40,
-) -> List[TaskRecord]:
-    """Deterministic miner — no API calls.
-
-    Strategy:
-      * Each session with >=1 real user prompt yields one TaskRecord whose
-        intent is the FIRST substantive prompt (the original ask).
-      * Outcome is inferred:
-          - negative feedback present and no later positive  -> "fail"
-          - positive feedback present                         -> "success"
-          - re-asks (multiple user turns) without resolution  -> "mixed"
-          - otherwise                                         -> "unknown"
-      * attempted_solution = the last assistant final (what was produced).
-      * reference_kind defaults to "none"; the consolidation step will use a
-        rubric judge for these. (Exact refs are added by the experiment data
-        or by the LLM miner when it can derive a checkable answer.)
-    """
-    tasks: List[TaskRecord] = []
-    for d in digests:
-        if not d.user_prompts:
-            continue
-        intent = d.user_prompts[0]
-        if len(intent.strip()) < 8:
-            continue
-        if _looks_positive(d.feedback_signals) and not _looks_negative(d.feedback_signals):
-            outcome = "success"
-        elif _looks_negative(d.feedback_signals):
-            outcome = "fail"
-        elif d.n_user_turns >= 3:
-            outcome = "mixed"
-        else:
-            outcome = "unknown"
-
-        attempted = d.assistant_finals[-1] if d.assistant_finals else ""
-        context = ""
-        if len(d.user_prompts) > 1:
-            # later prompts often carry the corrective detail / real constraints
-            context = "Follow-up constraints from the same session:\n- " + "\n- ".join(
-                _short(p, 200) for p in d.user_prompts[1:4]
-            )
-        tags = []
-        if d.tools_used:
-            tags.append("tools:" + "+".join(d.tools_used[:4]))
-        if d.git_branch:
-            tags.append("branch:" + d.git_branch)
-
-        tasks.append(
-            TaskRecord(
-                id=_tid(d.project, intent),
-                project=d.project,
-                intent=_short(intent, 800),
-                context_excerpt=_short(context, 600),
-                attempted_solution=_short(attempted, 600),
-                outcome=outcome,
-                reference_kind="none",
-                reference="",
-                tags=tags,
-                source_sessions=[d.session_id],
-            )
-        )
-        if len(tasks) >= max_tasks:
-            break
-    return tasks
-
-
-def dedup_tasks(tasks: List[TaskRecord]) -> List[TaskRecord]:
-    """Merge tasks sharing an id (same project+intent across sessions)."""
-    by_id: dict = {}
-    for t in tasks:
-        if t.id in by_id:
-            ex = by_id[t.id]
-            ex.source_sessions = list(dict.fromkeys(ex.source_sessions + t.source_sessions))
-            # prefer a resolved outcome if either session resolved it
-            order = {"success": 3, "fail": 2, "mixed": 1, "unknown": 0}
-            if order.get(t.outcome, 0) > order.get(ex.outcome, 0):
-                ex.outcome = t.outcome
-        else:
-            by_id[t.id] = t
-    return list(by_id.values())
-
-
-def assign_splits(
-    tasks: List[TaskRecord],
-    *,
-    val_fraction: float = 0.34,
-    test_fraction: float = 0.0,
-    holdout_fraction: float | None = None,  # legacy alias for val_fraction
-    seed: int = 42,
-) -> List[TaskRecord]:
-    """Deterministically split tasks into train / val / test.
-
-    Anti-overfitting contract (the user's design):
-      * ``val`` and ``test`` are drawn ONLY from REAL mined tasks (origin=='real')
-        and never overlap. val gates updates; test is the final held-out measure.
-      * ``train`` may include DREAM-augmented tasks (origin=='dream'); those are
-        NEVER placed in val/test.
-
-    A stable hash of the task id keeps the same real task in the same split across
-    nights (a fixed held-out gate, like SkillOpt's D_sel/D_test).
-
-    Back-compat: if ``test_fraction`` is 0 (default), this behaves like the old
-    two-way replay/holdout split — real tasks divide into train + val, no test.
-    ``holdout_fraction`` is accepted as an alias for ``val_fraction``.
-    """
-    if holdout_fraction is not None:
-        val_fraction = holdout_fraction
-
-    dream = [t for t in tasks if t.origin == "dream"]
-    real = [t for t in tasks if t.origin != "dream"]
-
-    # all dream tasks go to train, unconditionally
-    for t in dream:
-        t.split = "train"
-
-    val_cut = int(round(val_fraction * 100))
-    test_cut = val_cut + int(round(test_fraction * 100))
-    for t in real:
-        bucket = int(hashlib.sha256((str(seed) + t.id).encode()).hexdigest(), 16) % 100
-        if bucket < val_cut:
-            t.split = "val"
-        elif bucket < test_cut:
-            t.split = "test"
-        else:
-            t.split = "train"
-
-    # guarantee val (the gate) is non-empty when we have >=2 real tasks
-    real_splits = {t.split for t in real}
-    if len(real) >= 2 and "val" not in real_splits:
-        real[-1].split = "val"
-    # guarantee a train pool exists (dream or real) when possible
-    if not any(t.split == "train" for t in tasks) and len(real) >= 2:
-        real[0].split = "train"
-    # if test was requested but ended up empty with >=3 real tasks, carve one
-    if test_fraction > 0 and len(real) >= 3 and not any(t.split == "test" for t in real):
-        for t in real:
-            if t.split == "train":
-                t.split = "test"
-                break
-    return tasks
-
-
-def normalize_legacy_split(value: str) -> str:
-    """Map old split names to the new vocabulary."""
-    return {"replay": "train", "holdout": "val"}.get(value, value)
-
-
-def mine(
-    digests: List[SessionDigest],
-    *,
-    max_tasks: int = 40,
-    candidate_limit: int = 0,
-    holdout_fraction: float = 0.34,
-    seed: int = 42,
-    llm_miner: Optional[Callable[[List[SessionDigest]], List[TaskRecord]]] = None,
-    target_skill_text: str = "",
-    target_skill_path: str = "",
-) -> List[TaskRecord]:
-    """Top-level miner. Uses ``llm_miner`` if provided, else heuristic."""
-    candidate_limit = candidate_limit or max_tasks
-    tasks: List[TaskRecord] = []
-    if llm_miner is not None:
-        try:
-            tasks = llm_miner(digests) or []
-        except Exception:
-            tasks = []
-    if not tasks:
-        tasks = heuristic_mine(digests, max_tasks=candidate_limit)
-    tasks = dedup_tasks(tasks)
-    if target_skill_text or target_skill_path:
-        tasks = filter_tasks_for_target(tasks, target_skill_text, target_skill_path)
-    tasks = tasks[:max_tasks]
-    tasks = assign_splits(tasks, holdout_fraction=holdout_fraction, seed=seed)
-    return tasks

package/template/skills/skillopt-sleep/scripts/python/replay.py

@@ -1,146 +0,0 @@
-"""SkillOpt-Sleep — Stage 3: replay.
-
-Re-run mined TaskRecords offline under a given (skill, memory) and score
-them, producing the (hard, soft) signal SkillOpt's gate consumes.
-
-Single-shot text replay by default. Tasks whose rule judge requires a tool
-call (gbrain's `tool_called`) are run through the backend's real tool loop
-(attempt_with_tools), so tool use is verified honestly rather than self-reported.
-"""
-from __future__ import annotations
-
-from typing import List, Tuple
-
-from skillopt_sleep.backend import Backend
-from skillopt_sleep.types import ReplayResult, TaskRecord
-
-
-def _required_tools(task: TaskRecord) -> List[str]:
-    """Tool names a rule judge requires (op == 'tool_called')."""
-    if task.reference_kind != "rule" or not task.judge:
-        return []
-    tools = []
-    for c in task.judge.get("checks", []) or []:
-        if isinstance(c, dict) and c.get("op") == "tool_called" and c.get("arg"):
-            tools.append(str(c["arg"]))
-    return tools
-
-
-def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str,
-               sample_id: int = 0) -> ReplayResult:
-    """``sample_id`` distinguishes repeated dream rollouts of the same
-    (task, skill, memory) in the attempt cache — without it all K rollouts
-    collapse to one cached response and the contrastive signal is always 0."""
-    import time
-    tools = _required_tools(task)
-    tools_called: List[str] = []
-    t0 = time.time()
-    tok_before = backend.tokens_used()
-    if tools:
-        response, tools_called = backend.attempt_with_tools(task, skill, memory, tools)
-    else:
-        response = backend.attempt(task, skill, memory, sample_id=sample_id)
-    latency_ms = (time.time() - t0) * 1000.0
-    tokens = max(0, backend.tokens_used() - tok_before)
-    # if the backend doesn't track tokens (e.g. mock), approximate from text length
-    if tokens == 0:
-        tokens = (len(skill) + len(memory) + len(task.intent) + len(response)) // 4
-
-    # rule judges may need the detected tool calls; score locally when possible
-    if task.reference_kind == "rule" and task.judge:
-        from skillopt_sleep.judges import score_rule_judge
-        hard, soft, rationale = score_rule_judge(task.judge, response, tools_called)
-    else:
-        hard, soft, rationale = backend.judge(task, response)
-
-    return ReplayResult(
-        id=task.id,
-        hard=float(hard),
-        soft=float(soft),
-        response=response,
-        fail_reason="" if hard >= 1.0 else (rationale or "below threshold"),
-        task_type=(task.tags[0] if task.tags else "task"),
-        judge_rationale=rationale,
-        tools_called=tools_called,
-        tokens=int(tokens),
-        latency_ms=round(latency_ms, 1),
-    )
-
-
-import os
-from concurrent.futures import ThreadPoolExecutor
-
-
-def replay_batch(
-    backend: Backend,
-    tasks: List[TaskRecord],
-    skill: str,
-    memory: str,
-    *,
-    workers: int = 0,
-) -> List[Tuple[TaskRecord, ReplayResult]]:
-    """Replay tasks, optionally in parallel.
-
-    Real backends are network-bound, so a thread pool gives a large speedup on
-    big test sets (like the research harness's --workers). ``workers`` defaults
-    to env SKILLOPT_SLEEP_WORKERS or 1 (sequential). Mock stays sequential
-    (deterministic) unless asked otherwise.
-    """
-    if workers <= 0:
-        workers = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1") or "1")
-    if workers <= 1 or len(tasks) <= 1:
-        return [(t, replay_one(backend, t, skill, memory)) for t in tasks]
-    results: List = [None] * len(tasks)
-    with ThreadPoolExecutor(max_workers=min(workers, len(tasks))) as ex:
-        futs = {ex.submit(replay_one, backend, t, skill, memory): i
-                for i, t in enumerate(tasks)}
-        for fut in futs:
-            i = futs[fut]
-            results[i] = (tasks[i], fut.result())
-    return results
-
-
-def aggregate_scores(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]:
-    if not pairs:
-        return 0.0, 0.0
-    hard = sum(r.hard for _t, r in pairs) / len(pairs)
-    soft = sum(r.soft for _t, r in pairs) / len(pairs)
-    return hard, soft
-
-
-def aggregate_cost(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]:
-    """Mean (tokens, latency_ms) per task — the cost objectives."""
-    if not pairs:
-        return 0.0, 0.0
-    tok = sum(r.tokens for _t, r in pairs) / len(pairs)
-    lat = sum(r.latency_ms for _t, r in pairs) / len(pairs)
-    return tok, lat
-
-
-def multi_objective_reward(
-    pairs: List[Tuple[TaskRecord, ReplayResult]],
-    *,
-    w_acc: float = 1.0,
-    w_tokens: float = 0.0,
-    w_latency: float = 0.0,
-    token_ref: float = 2000.0,
-    latency_ref_ms: float = 15000.0,
-) -> float:
-    """Weighted reward = accuracy↑, tokens↓, latency↓.
-
-    Cost terms are normalized against a reference and clamped to [0,1], so a
-    response at/under the reference cost contributes ~1.0 and an expensive one
-    less. Weights let the user trade off (default = accuracy only, backward
-    compatible).
-    """
-    if not pairs:
-        return 0.0
-    acc, _soft = aggregate_scores(pairs)
-    tok, lat = aggregate_cost(pairs)
-    tok_score = max(0.0, 1.0 - tok / max(1.0, token_ref)) if token_ref else 0.0
-    lat_score = max(0.0, 1.0 - lat / max(1.0, latency_ref_ms)) if latency_ref_ms else 0.0
-    total_w = w_acc + w_tokens + w_latency
-    if total_w <= 0:
-        return acc
-    return (w_acc * acc + w_tokens * tok_score + w_latency * lat_score) / total_w
-

package/template/skills/skillopt-sleep/scripts/python/rollout.py

@@ -1,153 +0,0 @@
-"""SkillOpt-Sleep — multi-rollout + contrastive reflection (the imagination core).
-
-The core idea: let the agent re-run the SAME task many times, then look at
-which rollouts went well vs badly and distill a rule from the *contrast*. This
-is a much stronger learning signal than a single failure, and it is the essence
-of the offline "dream/imagination" process — train-time rollouts are synthetic,
-so doing many is fine.
-
-Pieces:
-  * multi_rollout   — run one task K times under (skill, memory), return scored attempts
-  * contrastive_reflect — given good vs bad attempts of the same tasks, ask the
-    optimizer what distinguishes them and propose a general rule
-
-Driven through the Backend abstraction (mock/claude/codex), import-light.
-"""
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import List, Optional, Tuple
-
-from skillopt_sleep.backend import Backend, _extract_json
-from skillopt_sleep.replay import replay_one
-from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord
-
-
-@dataclass
-class RolloutSet:
-    """K scored attempts at one task under a fixed (skill, memory)."""
-    task: TaskRecord
-    attempts: List[ReplayResult] = field(default_factory=list)
-
-    @property
-    def best(self) -> Optional[ReplayResult]:
-        return max(self.attempts, key=lambda r: r.hard, default=None)
-
-    @property
-    def worst(self) -> Optional[ReplayResult]:
-        return min(self.attempts, key=lambda r: r.hard, default=None)
-
-    @property
-    def spread(self) -> float:
-        if not self.attempts:
-            return 0.0
-        hs = [r.hard for r in self.attempts]
-        return max(hs) - min(hs)
-
-    @property
-    def pass_rate(self) -> float:
-        if not self.attempts:
-            return 0.0
-        return sum(1 for r in self.attempts if r.hard >= 1.0) / len(self.attempts)
-
-
-def multi_rollout(
-    backend: Backend,
-    task: TaskRecord,
-    skill: str,
-    memory: str,
-    *,
-    k: int = 3,
-    workers: int = 0,
-) -> RolloutSet:
-    """Run ``task`` K times. replay_one is deterministic for mock; for real
-    backends the model's own sampling yields variation across attempts.
-
-    The K attempts are independent, so they run concurrently (this is the dream
-    phase's dominant cost). ``workers`` defaults to the SKILLOPT_SLEEP_WORKERS
-    env (capped at k); set to 1 to force serial (used by the mock tests).
-    """
-    import os
-    rs = RolloutSet(task=task)
-    k = max(1, k)
-    if workers <= 0:
-        try:
-            workers = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1"))
-        except ValueError:
-            workers = 1
-    workers = max(1, min(workers, k))
-    if workers == 1:
-        for i in range(k):
-            rs.attempts.append(replay_one(backend, task, skill, memory, sample_id=i))
-        return rs
-    from concurrent.futures import ThreadPoolExecutor
-    with ThreadPoolExecutor(max_workers=workers) as ex:
-        futs = [ex.submit(replay_one, backend, task, skill, memory, sample_id=i)
-                for i in range(k)]
-        for f in futs:
-            rs.attempts.append(f.result())
-    return rs
-
-
-def contrastive_reflect(
-    backend: Backend,
-    rollout_sets: List[RolloutSet],
-    skill: str,
-    memory: str,
-    *,
-    edit_budget: int = 4,
-    target: str = "skill",
-) -> List[EditRecord]:
-    """Distill a rule from the contrast between good and bad attempts.
-
-    We pick tasks with the highest score *spread* (some attempts passed, some
-    failed) — those are the most informative — and show the optimizer a
-    high-scoring vs a low-scoring attempt of each, asking what general rule makes
-    the good behavior reliable.
-    """
-    informative = [rs for rs in rollout_sets if rs.spread > 0 and rs.best and rs.worst]
-    informative.sort(key=lambda rs: rs.spread, reverse=True)
-    informative = informative[:6]
-    if not informative:
-        return []
-
-    blocks = []
-    for rs in informative:
-        blocks.append(
-            f"## Task: {rs.task.intent[:160]}\n"
-            f"- GOOD attempt (score {rs.best.hard:.1f}): {rs.best.response[:200]}\n"
-            f"- BAD  attempt (score {rs.worst.hard:.1f}): {rs.worst.response[:200]}\n"
-            f"  (bad failed: {rs.worst.fail_reason[:100]})"
-        )
-    # the output contract the proposed rules must not violate (same guardrail the
-    # single-shot reflect uses — prevents harness-violating rules like "return VBA"
-    # or "ask the user for the range" on SpreadsheetBench).
-    from skillopt_sleep.backend import _task_guardrail
-    guard = _task_guardrail([(rs.task, rs.best) for rs in informative])
-    prompt = (
-        "You are SkillOpt's optimizer doing CONTRASTIVE reflection. For each task "
-        "below the agent was run multiple times; some attempts succeeded and some "
-        "failed. Identify what the GOOD attempts did that the BAD ones did not, "
-        f"and propose at most {edit_budget} SHORT, GENERAL, reusable rules for the "
-        f"{target} that would make the good behavior reliable every time. Quote "
-        "concrete thresholds/formats verbatim; do not paraphrase vaguely. "
-        "Every rule MUST obey the task output contract (if shown) — never propose "
-        "a rule that changes the required output format/language or tells the agent "
-        "to ask the user a question; such a rule scores ZERO.\n"
-        f"{guard}"
-        'Return ONLY a JSON array: '
-        '[{"op":"add","content":"<rule>","rationale":"<what good did that bad didnt>"}].\n\n'
-        + "\n\n".join(blocks)
-    )
-    raw = backend._call(prompt, max_tokens=1024)  # type: ignore[attr-defined]
-    arr = _extract_json(raw, "array")
-    edits: List[EditRecord] = []
-    if isinstance(arr, list):
-        for e in arr[:edit_budget]:
-            if isinstance(e, dict) and str(e.get("content", "")).strip():
-                edits.append(EditRecord(
-                    target=target, op=str(e.get("op", "add")).strip().lower(),
-                    content=str(e["content"]).strip(),
-                    rationale=str(e.get("rationale", "")).strip(),
-                ))
-    return edits