self-evolve-framework 1.6.0 → 1.6.1

package/template/skills/skillopt-sleep/scripts/python/experiments/sweep.py

@@ -1,164 +0,0 @@
-"""SkillOpt-Sleep — benchmark sweep driver.
-
-Runs many (backend, model, seed, transfer-pair) configurations SEQUENTIALLY in
-one process, appending each result to a JSONL file as it finishes. Designed to
-run unattended in the background; safe to interrupt (already-written rows
-survive) and resume (skip configs whose row already exists).
-
-Then `report.py` turns the JSONL into a presented Markdown scorecard.
-
-Usage:
-  python -m skillopt_sleep.experiments.sweep --plan quick   --out docs/sleep/sweep.jsonl
-  python -m skillopt_sleep.experiments.sweep --plan full    --out docs/sleep/sweep.jsonl
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import sys
-import time
-from typing import Any, Dict, List
-
-from skillopt_sleep.backend import build_backend, get_backend
-from skillopt_sleep.experiments.gbrain_bench import find_data_root, load_seed
-from skillopt_sleep.experiments.run_gbrain import run_seed as bench_seed
-from skillopt_sleep.experiments.run_transfer import run_seed as transfer_seed
-
-
-# Plans: lists of config dicts. Kept small per-run to bound cost/latency.
-def _direct_cfg(backend, model, seed, nights=2):
-    return {"kind": "direct", "backend": backend, "model": model, "seed": seed, "nights": nights}
-
-
-def _dual_cfg(opt_backend, opt_model, tgt_backend, tgt_model, seed, nights=2):
-    # a 'direct' run on a DualBackend: strong optimizer proposes, weak target runs
-    return {"kind": "dual", "optimizer_backend": opt_backend, "optimizer_model": opt_model,
-            "target_backend": tgt_backend, "target_model": tgt_model, "seed": seed, "nights": nights}
-
-
-def _transfer_cfg(sb, sm, tb, tm, seed, nights=2):
-    return {"kind": "transfer", "source_backend": sb, "source_model": sm,
-            "target_backend": tb, "target_model": tm, "seed": seed, "nights": nights}
-
-
-PLANS: Dict[str, List[Dict[str, Any]]] = {
-    # one cheap seed each, both backends — fast sanity
-    "quick": [
-        _direct_cfg("claude", "haiku", "brief-writer", 1),
-        _direct_cfg("codex", "", "brief-writer", 2),
-    ],
-    # SkillOpt-faithful: STRONG optimizer (sonnet) proposes, WEAK target (haiku)
-    # runs — the reliable config. Plus Codex self-optimized. All 4 gbrain seeds,
-    # including quick-answerer (real tool loop).
-    "direct": [
-        _dual_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"),
-        _dual_cfg("claude", "sonnet", "claude", "haiku", "advisor"),
-        _dual_cfg("claude", "sonnet", "claude", "haiku", "thorough-analyst"),
-        _dual_cfg("claude", "sonnet", "claude", "haiku", "quick-answerer"),
-        _direct_cfg("codex", "", "brief-writer"),
-        _direct_cfg("codex", "", "advisor"),
-        _direct_cfg("codex", "", "quick-answerer"),
-    ],
-    # the price-difference story: optimize cheap, deploy expensive (and reverse)
-    "transfer": [
-        _transfer_cfg("claude", "haiku", "claude", "sonnet", "brief-writer"),
-        _transfer_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"),
-        _transfer_cfg("codex", "", "claude", "haiku", "brief-writer"),
-        _transfer_cfg("claude", "haiku", "codex", "", "brief-writer"),
-    ],
-}
-PLANS["full"] = PLANS["direct"] + PLANS["transfer"]
-
-
-def _cfg_key(c: Dict[str, Any]) -> str:
-    return json.dumps({k: c[k] for k in sorted(c)}, ensure_ascii=False)
-
-
-def _load_done(out_path: str) -> set:
-    done = set()
-    if os.path.exists(out_path):
-        with open(out_path) as f:
-            for line in f:
-                try:
-                    row = json.loads(line)
-                    if "cfg_key" in row:
-                        done.add(row["cfg_key"])
-                except Exception:
-                    pass
-    return done
-
-
-def _append(out_path: str, row: Dict[str, Any]) -> None:
-    os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
-    with open(out_path, "a") as f:
-        f.write(json.dumps(row, ensure_ascii=False) + "\n")
-
-
-def run_one(cfg: Dict[str, Any], data_root: str, codex_path: str,
-            limit_replay: int, limit_holdout: int) -> Dict[str, Any]:
-    seed = cfg["seed"]
-    skill, tasks = load_seed(data_root, seed)
-    t0 = time.time()
-    if cfg["kind"] in ("direct", "dual"):
-        if cfg["kind"] == "dual":
-            be = build_backend(
-                optimizer_backend=cfg["optimizer_backend"], optimizer_model=cfg.get("optimizer_model", ""),
-                target_backend=cfg["target_backend"], target_model=cfg.get("target_model", ""),
-                codex_path=codex_path,
-            )
-        else:
-            be = get_backend(cfg["backend"], model=cfg.get("model", ""), codex_path=codex_path)
-        r = bench_seed(be, seed, skill, tasks, nights=cfg["nights"],
-                       limit_replay=limit_replay, limit_holdout=limit_holdout)
-        out = {"baseline": r["held_out_before"], "after": r["held_out_after"],
-               "improved": r["improved"], "tokens": be.tokens_used()}
-    else:
-        src = get_backend(cfg["source_backend"], model=cfg.get("source_model", ""), codex_path=codex_path)
-        tgt = get_backend(cfg["target_backend"], model=cfg.get("target_model", ""), codex_path=codex_path)
-        r = transfer_seed(seed, skill, tasks, source=src, target=tgt, nights=cfg["nights"],
-                          edit_budget=4, limit_replay=limit_replay, limit_holdout=limit_holdout,
-                          do_direct=False)
-        out = {"baseline_target": r["baseline_target"], "transferred": r["transferred"],
-               "transfer_gain": r["transfer_gain"],
-               "tokens": src.tokens_used() + tgt.tokens_used()}
-    out.update({"cfg": cfg, "cfg_key": _cfg_key(cfg), "elapsed_s": round(time.time() - t0, 1)})
-    return out
-
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser(description="SkillOpt-Sleep benchmark sweep")
-    ap.add_argument("--plan", default="quick", choices=list(PLANS.keys()))
-    ap.add_argument("--out", default="docs/sleep/sweep.jsonl")
-    ap.add_argument("--data-root", default="")
-    ap.add_argument("--codex-path", default="")
-    ap.add_argument("--limit-replay", type=int, default=3)
-    ap.add_argument("--limit-holdout", type=int, default=3)
-    args = ap.parse_args(argv)
-
-    data_root = find_data_root(args.data_root)
-    if not data_root:
-        print("ERROR: gbrain-evals data not found; pass --data-root", file=sys.stderr)
-        return 2
-
-    plan = PLANS[args.plan]
-    done = _load_done(args.out)
-    print(f"[sweep] plan={args.plan} configs={len(plan)} already_done={len(done)} -> {args.out}")
-    for i, cfg in enumerate(plan, 1):
-        key = _cfg_key(cfg)
-        if key in done:
-            print(f"[sweep] ({i}/{len(plan)}) skip (done): {cfg}")
-            continue
-        print(f"[sweep] ({i}/{len(plan)}) running: {cfg}", flush=True)
-        try:
-            row = run_one(cfg, data_root, args.codex_path, args.limit_replay, args.limit_holdout)
-        except Exception as e:  # never let one config kill the sweep
-            row = {"cfg": cfg, "cfg_key": key, "error": f"{type(e).__name__}: {e}"}
-        _append(args.out, row)
-        print(f"[sweep]   -> {json.dumps({k: v for k, v in row.items() if k not in ('cfg','cfg_key')})}", flush=True)
-    print(f"[sweep] done. rows in {args.out}: {len(_load_done(args.out))}")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/template/skills/skillopt-sleep/scripts/python/gate.py

@@ -1,50 +0,0 @@
-"""SkillOpt-Sleep — vendored validation gate.
-
-This is a self-contained copy of the SkillOpt validation gate so the sleep
-engine has ZERO dependency on the research package (skillopt/*). The research
-repo's ``skillopt.evaluation.gate`` is the reference implementation and the two
-are kept behaviourally identical; vendoring keeps this open-source tool
-decoupled from the paper's experiment code.
-"""
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-
-@dataclass(frozen=True)
-class GateResult:
-    action: str            # "accept_new_best" | "accept" | "reject"
-    current_skill: str
-    current_score: float
-    best_skill: str
-    best_score: float
-    best_step: int
-
-
-def select_gate_score(hard: float, soft: float, metric: str = "hard",
-                      mixed_weight: float = 0.5) -> float:
-    """Project (hard, soft) onto a single comparison metric."""
-    if metric == "hard":
-        return float(hard)
-    if metric == "soft":
-        return float(soft)
-    if metric == "mixed":
-        w = max(0.0, min(1.0, float(mixed_weight)))
-        return (1.0 - w) * float(hard) + w * float(soft)
-    raise ValueError(f"unknown gate metric {metric!r}; expected hard/soft/mixed")
-
-
-def evaluate_gate(candidate_skill: str, cand_hard: float, current_skill: str,
-                  current_score: float, best_skill: str, best_score: float,
-                  best_step: int, global_step: int, *, cand_soft: float = 0.0,
-                  metric: str = "hard", mixed_weight: float = 0.5) -> GateResult:
-    """Pure gate decision: compare candidate score to current/best."""
-    cand_score = select_gate_score(cand_hard, cand_soft, metric, mixed_weight)
-    if cand_score > current_score:
-        if cand_score > best_score:
-            return GateResult("accept_new_best", candidate_skill, cand_score,
-                              candidate_skill, cand_score, global_step)
-        return GateResult("accept", candidate_skill, cand_score,
-                          best_skill, best_score, best_step)
-    return GateResult("reject", current_skill, current_score,
-                      best_skill, best_score, best_step)

package/template/skills/skillopt-sleep/scripts/python/harvest.py

@@ -1,304 +0,0 @@
-"""SkillOpt-Sleep — Stage 1: harvest.
-
-Read the user's local Claude Code records (read-only) and normalize them
-into :class:`SessionDigest` objects.
-
-Sources (verified schema):
-  * ~/.claude/history.jsonl        — one JSON/line:
-        {"display": <prompt text>, "pastedContents": {...},
-         "timestamp": <epoch ms>, "project": <abs path>}
-  * ~/.claude/projects/<slug>/<sessionId>.jsonl — one record/line; the
-    records we care about have type "user"/"assistant" and carry:
-        message{role, content}, cwd, gitBranch, timestamp, sessionId, version
-
-This module performs NO writes and NO network calls.
-"""
-from __future__ import annotations
-
-import json
-import os
-from datetime import datetime
-from typing import Any, Dict, Iterable, List, Optional
-
-from skillopt_sleep.types import SessionDigest
-
-
-# Heuristic phrases that signal the user (dis)approving of prior output.
-# English-only by default. Users whose sessions are in another language can add
-# their own phrases via the SKILLOPT_SLEEP_NEG_FEEDBACK / _POS_FEEDBACK env vars
-# (comma-separated), so the capability is extensible without hardcoding locales.
-_NEGATIVE_FEEDBACK = (
-    "still broken", "still not", "still wrong", "doesn't work", "does not work",
-    "not working", "that's wrong", "thats wrong", "incorrect", "wrong",
-    "no,", "nope", "fix it", "didn't", "did not", "broken", "error again",
-    "still failing", "still fails", "not fixed", "revert", "undo",
-)
-_POSITIVE_FEEDBACK = (
-    "thanks", "thank you", "perfect", "great", "works now", "fixed",
-    "that works", "lgtm", "looks good", "nice", "awesome", "correct",
-)
-
-
-def _extra_phrases(env_var: str) -> tuple:
-    raw = os.environ.get(env_var, "")
-    return tuple(p.strip().lower() for p in raw.split(",") if p.strip())
-
-
-_NEGATIVE_FEEDBACK = _NEGATIVE_FEEDBACK + _extra_phrases("SKILLOPT_SLEEP_NEG_FEEDBACK")
-_POSITIVE_FEEDBACK = _POSITIVE_FEEDBACK + _extra_phrases("SKILLOPT_SLEEP_POS_FEEDBACK")
-
-
-def _iter_jsonl(path: str) -> Iterable[Dict[str, Any]]:
-    try:
-        with open(path, encoding="utf-8") as f:
-            for line in f:
-                line = line.strip()
-                if not line:
-                    continue
-                try:
-                    yield json.loads(line)
-                except Exception:
-                    continue
-    except (FileNotFoundError, IsADirectoryError, PermissionError):
-        return
-
-
-def _text_from_content(content: Any) -> str:
-    """Flatten a message.content (str or list of blocks) into text."""
-    if isinstance(content, str):
-        return content
-    if isinstance(content, list):
-        parts: List[str] = []
-        for b in content:
-            if isinstance(b, dict):
-                if b.get("type") == "text" and b.get("text"):
-                    parts.append(str(b["text"]))
-        return "\n".join(parts)
-    return ""
-
-
-def _tool_names_from_content(content: Any) -> List[str]:
-    names: List[str] = []
-    if isinstance(content, list):
-        for b in content:
-            if isinstance(b, dict) and b.get("type") == "tool_use" and b.get("name"):
-                names.append(str(b["name"]))
-    return names
-
-
-def _detect_feedback(text: str) -> List[str]:
-    low = text.lower()
-    sig: List[str] = []
-    for ph in _NEGATIVE_FEEDBACK:
-        if ph in low:
-            sig.append("neg:" + ph)
-    for ph in _POSITIVE_FEEDBACK:
-        if ph in low:
-            sig.append("pos:" + ph)
-    return sig
-
-
-def _is_meta_prompt(text: str) -> bool:
-    """Skip slash-commands / system noise that aren't real user intents."""
-    t = text.strip()
-    if not t:
-        return True
-    if t.startswith("<") and t.endswith(">"):
-        return True
-    if t.startswith("/") and len(t.split()) <= 3:
-        return True
-    if t.startswith("[Pasted text") or t.startswith("Caveat:"):
-        return True
-    return False
-
-
-# ── Issue #62: filter headless replay sessions ─────────────────────────
-
-# Prompt markers generated by the engine's own headless `claude -p` calls
-# (judge, reflect, attempt). If the sole user prompt in a single-turn
-# session matches any of these, the session is engine-generated, not a
-# real user task.
-_REPLAY_PROMPT_MARKERS = (
-    "## CURRENT SKILL",
-    "## FAILED TASKS",
-    "## SUCCESSFUL TASKS",
-    "## OUTPUT FORMAT",
-    "You are a strict grader",
-    "Score the response 0.0-1.0",
-    "You are SkillOpt-Sleep",
-    "## TASK\n",
-    "## SKILL\n",
-)
-
-
-def _is_headless_replay(digest: "SessionDigest") -> bool:
-    """Detect sessions created by the engine's own headless replay calls.
-
-    Heuristics (conservatively applied):
-    1. Session has exactly 1 user turn AND
-    2. The sole prompt matches engine-generated patterns (grader/reflect),
-       OR the session lasted < 3 seconds (programmatic, not interactive).
-    Multi-turn sessions are always kept (interactive by definition).
-    """
-    if digest.n_user_turns > 1:
-        return False
-    if digest.n_user_turns == 0:
-        return True
-    prompt = digest.user_prompts[0] if digest.user_prompts else ""
-    for marker in _REPLAY_PROMPT_MARKERS:
-        if marker in prompt:
-            return True
-    # Sub-3-second single-turn sessions with short prompts are almost
-    # certainly programmatic (engine grader/judge calls).  We require the
-    # prompt to also be short (<200 chars) to avoid false-positives on
-    # real one-shot questions that Claude happens to answer quickly.
-    if digest.started_at and digest.ended_at and len(prompt) < 200:
-        try:
-            fmt = "%Y-%m-%dT%H:%M:%S"
-            start = datetime.strptime(digest.started_at[:19], fmt)
-            end = datetime.strptime(digest.ended_at[:19], fmt)
-            if (end - start).total_seconds() < 3:
-                return True
-        except (ValueError, TypeError):
-            pass
-    return False
-
-
-def digest_transcript(path: str) -> Optional[SessionDigest]:
-    """Build a SessionDigest from one ``<sessionId>.jsonl`` transcript."""
-    session_id = os.path.splitext(os.path.basename(path))[0]
-    project = ""
-    git_branch = ""
-    started = ""
-    ended = ""
-    user_prompts: List[str] = []
-    assistant_finals: List[str] = []
-    tools: List[str] = []
-    files: List[str] = []
-    feedback: List[str] = []
-    n_user = 0
-    n_asst = 0
-
-    for rec in _iter_jsonl(path):
-        rtype = rec.get("type")
-        ts = rec.get("timestamp")
-        if isinstance(ts, str) and ts:
-            if not started:
-                started = ts
-            ended = ts
-        if rec.get("cwd") and not project:
-            project = str(rec.get("cwd"))
-        if rec.get("gitBranch") and not git_branch:
-            git_branch = str(rec.get("gitBranch"))
-        if rtype == "file-history-snapshot":
-            snap = rec.get("snapshot") or rec.get("files") or {}
-            if isinstance(snap, dict):
-                files.extend([str(k) for k in list(snap.keys())[:20]])
-        msg = rec.get("message")
-        if not isinstance(msg, dict):
-            continue
-        role = msg.get("role")
-        content = msg.get("content")
-        if role == "user":
-            text = _text_from_content(content)
-            if text and not _is_meta_prompt(text):
-                n_user += 1
-                user_prompts.append(text.strip())
-                feedback.extend(_detect_feedback(text))
-        elif role == "assistant":
-            n_asst += 1
-            tools.extend(_tool_names_from_content(content))
-            text = _text_from_content(content)
-            if text.strip():
-                assistant_finals.append(text.strip())
-
-    if n_user == 0 and n_asst == 0:
-        return None
-
-    # de-dup tools/files preserving order
-    def _dedup(xs: List[str]) -> List[str]:
-        seen = set()
-        out = []
-        for x in xs:
-            if x not in seen:
-                seen.add(x)
-                out.append(x)
-        return out
-
-    return SessionDigest(
-        session_id=session_id,
-        project=project,
-        git_branch=git_branch,
-        started_at=started,
-        ended_at=ended,
-        user_prompts=user_prompts,
-        assistant_finals=assistant_finals[-5:],  # last few finals are the useful ones
-        tools_used=_dedup(tools),
-        files_touched=_dedup(files),
-        feedback_signals=feedback,
-        n_user_turns=n_user,
-        n_assistant_turns=n_asst,
-        raw_path=path,
-    )
-
-
-def _project_matches(project: str, scope: Any, invoked: str) -> bool:
-    if scope == "all":
-        return True
-    if isinstance(scope, (list, tuple)):
-        return any(os.path.abspath(project) == os.path.abspath(p) for p in scope)
-    # "invoked": match the invoked project (or a subdir of it)
-    if not invoked:
-        return True
-    a = os.path.abspath(project)
-    b = os.path.abspath(invoked)
-    return a == b or a.startswith(b + os.sep) or b.startswith(a + os.sep)
-
-
-def harvest(
-    transcripts_dir: str,
-    *,
-    scope: Any = "all",
-    invoked_project: str = "",
-    since_iso: Optional[str] = None,
-    limit: int = 0,
-) -> List[SessionDigest]:
-    """Walk ~/.claude/projects and return digests matching scope/time.
-
-    Parameters
-    ----------
-    transcripts_dir : str    ~/.claude/projects
-    scope : "all" | "invoked" | list[path]
-    invoked_project : str    used when scope == "invoked"
-    since_iso : str|None      ISO8601; only sessions ending after this are kept
-    limit : int               cap number of digests (0 = no cap)
-    """
-    digests: List[SessionDigest] = []
-    if not os.path.isdir(transcripts_dir):
-        return digests
-
-    paths: List[str] = []
-    for root, _dirs, files in os.walk(transcripts_dir):
-        for fn in files:
-            if fn.endswith(".jsonl"):
-                paths.append(os.path.join(root, fn))
-    # newest first by mtime
-    paths.sort(key=lambda p: os.path.getmtime(p), reverse=True)
-
-    for p in paths:
-        d = digest_transcript(p)
-        if d is None:
-            continue
-        if _is_headless_replay(d):
-            continue  # Issue #62: skip engine's own headless replay sessions
-        if not _project_matches(d.project or "", scope, invoked_project):
-            continue
-        if since_iso and d.ended_at and d.ended_at < since_iso:
-            # Note: files are sorted by mtime but we compare the embedded
-            # ended_at timestamp — mtime can diverge (copy/touch), so we
-            # cannot break here; we must continue to check all files.
-            continue
-        digests.append(d)
-        if limit and len(digests) >= limit:
-            break
-    return digests