self-evolve-framework 1.3.0 → 1.5.0

package/template/skills/skillopt-sleep/scripts/python/experiments/sweep.py

@@ -0,0 +1,164 @@
+"""SkillOpt-Sleep — benchmark sweep driver.
+
+Runs many (backend, model, seed, transfer-pair) configurations SEQUENTIALLY in
+one process, appending each result to a JSONL file as it finishes. Designed to
+run unattended in the background; safe to interrupt (already-written rows
+survive) and resume (skip configs whose row already exists).
+
+Then `report.py` turns the JSONL into a presented Markdown scorecard.
+
+Usage:
+  python -m skillopt_sleep.experiments.sweep --plan quick   --out docs/sleep/sweep.jsonl
+  python -m skillopt_sleep.experiments.sweep --plan full    --out docs/sleep/sweep.jsonl
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+from typing import Any, Dict, List
+
+from skillopt_sleep.backend import build_backend, get_backend
+from skillopt_sleep.experiments.gbrain_bench import find_data_root, load_seed
+from skillopt_sleep.experiments.run_gbrain import run_seed as bench_seed
+from skillopt_sleep.experiments.run_transfer import run_seed as transfer_seed
+
+
+# Plans: lists of config dicts. Kept small per-run to bound cost/latency.
+def _direct_cfg(backend, model, seed, nights=2):
+    return {"kind": "direct", "backend": backend, "model": model, "seed": seed, "nights": nights}
+
+
+def _dual_cfg(opt_backend, opt_model, tgt_backend, tgt_model, seed, nights=2):
+    # a 'direct' run on a DualBackend: strong optimizer proposes, weak target runs
+    return {"kind": "dual", "optimizer_backend": opt_backend, "optimizer_model": opt_model,
+            "target_backend": tgt_backend, "target_model": tgt_model, "seed": seed, "nights": nights}
+
+
+def _transfer_cfg(sb, sm, tb, tm, seed, nights=2):
+    return {"kind": "transfer", "source_backend": sb, "source_model": sm,
+            "target_backend": tb, "target_model": tm, "seed": seed, "nights": nights}
+
+
+PLANS: Dict[str, List[Dict[str, Any]]] = {
+    # one cheap seed each, both backends — fast sanity
+    "quick": [
+        _direct_cfg("claude", "haiku", "brief-writer", 1),
+        _direct_cfg("codex", "", "brief-writer", 2),
+    ],
+    # SkillOpt-faithful: STRONG optimizer (sonnet) proposes, WEAK target (haiku)
+    # runs — the reliable config. Plus Codex self-optimized. All 4 gbrain seeds,
+    # including quick-answerer (real tool loop).
+    "direct": [
+        _dual_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"),
+        _dual_cfg("claude", "sonnet", "claude", "haiku", "advisor"),
+        _dual_cfg("claude", "sonnet", "claude", "haiku", "thorough-analyst"),
+        _dual_cfg("claude", "sonnet", "claude", "haiku", "quick-answerer"),
+        _direct_cfg("codex", "", "brief-writer"),
+        _direct_cfg("codex", "", "advisor"),
+        _direct_cfg("codex", "", "quick-answerer"),
+    ],
+    # the price-difference story: optimize cheap, deploy expensive (and reverse)
+    "transfer": [
+        _transfer_cfg("claude", "haiku", "claude", "sonnet", "brief-writer"),
+        _transfer_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"),
+        _transfer_cfg("codex", "", "claude", "haiku", "brief-writer"),
+        _transfer_cfg("claude", "haiku", "codex", "", "brief-writer"),
+    ],
+}
+PLANS["full"] = PLANS["direct"] + PLANS["transfer"]
+
+
+def _cfg_key(c: Dict[str, Any]) -> str:
+    return json.dumps({k: c[k] for k in sorted(c)}, ensure_ascii=False)
+
+
+def _load_done(out_path: str) -> set:
+    done = set()
+    if os.path.exists(out_path):
+        with open(out_path) as f:
+            for line in f:
+                try:
+                    row = json.loads(line)
+                    if "cfg_key" in row:
+                        done.add(row["cfg_key"])
+                except Exception:
+                    pass
+    return done
+
+
+def _append(out_path: str, row: Dict[str, Any]) -> None:
+    os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
+    with open(out_path, "a") as f:
+        f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def run_one(cfg: Dict[str, Any], data_root: str, codex_path: str,
+            limit_replay: int, limit_holdout: int) -> Dict[str, Any]:
+    seed = cfg["seed"]
+    skill, tasks = load_seed(data_root, seed)
+    t0 = time.time()
+    if cfg["kind"] in ("direct", "dual"):
+        if cfg["kind"] == "dual":
+            be = build_backend(
+                optimizer_backend=cfg["optimizer_backend"], optimizer_model=cfg.get("optimizer_model", ""),
+                target_backend=cfg["target_backend"], target_model=cfg.get("target_model", ""),
+                codex_path=codex_path,
+            )
+        else:
+            be = get_backend(cfg["backend"], model=cfg.get("model", ""), codex_path=codex_path)
+        r = bench_seed(be, seed, skill, tasks, nights=cfg["nights"],
+                       limit_replay=limit_replay, limit_holdout=limit_holdout)
+        out = {"baseline": r["held_out_before"], "after": r["held_out_after"],
+               "improved": r["improved"], "tokens": be.tokens_used()}
+    else:
+        src = get_backend(cfg["source_backend"], model=cfg.get("source_model", ""), codex_path=codex_path)
+        tgt = get_backend(cfg["target_backend"], model=cfg.get("target_model", ""), codex_path=codex_path)
+        r = transfer_seed(seed, skill, tasks, source=src, target=tgt, nights=cfg["nights"],
+                          edit_budget=4, limit_replay=limit_replay, limit_holdout=limit_holdout,
+                          do_direct=False)
+        out = {"baseline_target": r["baseline_target"], "transferred": r["transferred"],
+               "transfer_gain": r["transfer_gain"],
+               "tokens": src.tokens_used() + tgt.tokens_used()}
+    out.update({"cfg": cfg, "cfg_key": _cfg_key(cfg), "elapsed_s": round(time.time() - t0, 1)})
+    return out
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="SkillOpt-Sleep benchmark sweep")
+    ap.add_argument("--plan", default="quick", choices=list(PLANS.keys()))
+    ap.add_argument("--out", default="docs/sleep/sweep.jsonl")
+    ap.add_argument("--data-root", default="")
+    ap.add_argument("--codex-path", default="")
+    ap.add_argument("--limit-replay", type=int, default=3)
+    ap.add_argument("--limit-holdout", type=int, default=3)
+    args = ap.parse_args(argv)
+
+    data_root = find_data_root(args.data_root)
+    if not data_root:
+        print("ERROR: gbrain-evals data not found; pass --data-root", file=sys.stderr)
+        return 2
+
+    plan = PLANS[args.plan]
+    done = _load_done(args.out)
+    print(f"[sweep] plan={args.plan} configs={len(plan)} already_done={len(done)} -> {args.out}")
+    for i, cfg in enumerate(plan, 1):
+        key = _cfg_key(cfg)
+        if key in done:
+            print(f"[sweep] ({i}/{len(plan)}) skip (done): {cfg}")
+            continue
+        print(f"[sweep] ({i}/{len(plan)}) running: {cfg}", flush=True)
+        try:
+            row = run_one(cfg, data_root, args.codex_path, args.limit_replay, args.limit_holdout)
+        except Exception as e:  # never let one config kill the sweep
+            row = {"cfg": cfg, "cfg_key": key, "error": f"{type(e).__name__}: {e}"}
+        _append(args.out, row)
+        print(f"[sweep]   -> {json.dumps({k: v for k, v in row.items() if k not in ('cfg','cfg_key')})}", flush=True)
+    print(f"[sweep] done. rows in {args.out}: {len(_load_done(args.out))}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/template/skills/skillopt-sleep/scripts/python/gate.py

@@ -0,0 +1,50 @@
+"""SkillOpt-Sleep — vendored validation gate.
+
+This is a self-contained copy of the SkillOpt validation gate so the sleep
+engine has ZERO dependency on the research package (skillopt/*). The research
+repo's ``skillopt.evaluation.gate`` is the reference implementation and the two
+are kept behaviourally identical; vendoring keeps this open-source tool
+decoupled from the paper's experiment code.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class GateResult:
+    action: str            # "accept_new_best" | "accept" | "reject"
+    current_skill: str
+    current_score: float
+    best_skill: str
+    best_score: float
+    best_step: int
+
+
+def select_gate_score(hard: float, soft: float, metric: str = "hard",
+                      mixed_weight: float = 0.5) -> float:
+    """Project (hard, soft) onto a single comparison metric."""
+    if metric == "hard":
+        return float(hard)
+    if metric == "soft":
+        return float(soft)
+    if metric == "mixed":
+        w = max(0.0, min(1.0, float(mixed_weight)))
+        return (1.0 - w) * float(hard) + w * float(soft)
+    raise ValueError(f"unknown gate metric {metric!r}; expected hard/soft/mixed")
+
+
+def evaluate_gate(candidate_skill: str, cand_hard: float, current_skill: str,
+                  current_score: float, best_skill: str, best_score: float,
+                  best_step: int, global_step: int, *, cand_soft: float = 0.0,
+                  metric: str = "hard", mixed_weight: float = 0.5) -> GateResult:
+    """Pure gate decision: compare candidate score to current/best."""
+    cand_score = select_gate_score(cand_hard, cand_soft, metric, mixed_weight)
+    if cand_score > current_score:
+        if cand_score > best_score:
+            return GateResult("accept_new_best", candidate_skill, cand_score,
+                              candidate_skill, cand_score, global_step)
+        return GateResult("accept", candidate_skill, cand_score,
+                          best_skill, best_score, best_step)
+    return GateResult("reject", current_skill, current_score,
+                      best_skill, best_score, best_step)

package/template/skills/skillopt-sleep/scripts/python/harvest.py

@@ -0,0 +1,304 @@
+"""SkillOpt-Sleep — Stage 1: harvest.
+
+Read the user's local Claude Code records (read-only) and normalize them
+into :class:`SessionDigest` objects.
+
+Sources (verified schema):
+  * ~/.claude/history.jsonl        — one JSON/line:
+        {"display": <prompt text>, "pastedContents": {...},
+         "timestamp": <epoch ms>, "project": <abs path>}
+  * ~/.claude/projects/<slug>/<sessionId>.jsonl — one record/line; the
+    records we care about have type "user"/"assistant" and carry:
+        message{role, content}, cwd, gitBranch, timestamp, sessionId, version
+
+This module performs NO writes and NO network calls.
+"""
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime
+from typing import Any, Dict, Iterable, List, Optional
+
+from skillopt_sleep.types import SessionDigest
+
+
+# Heuristic phrases that signal the user (dis)approving of prior output.
+# English-only by default. Users whose sessions are in another language can add
+# their own phrases via the SKILLOPT_SLEEP_NEG_FEEDBACK / _POS_FEEDBACK env vars
+# (comma-separated), so the capability is extensible without hardcoding locales.
+_NEGATIVE_FEEDBACK = (
+    "still broken", "still not", "still wrong", "doesn't work", "does not work",
+    "not working", "that's wrong", "thats wrong", "incorrect", "wrong",
+    "no,", "nope", "fix it", "didn't", "did not", "broken", "error again",
+    "still failing", "still fails", "not fixed", "revert", "undo",
+)
+_POSITIVE_FEEDBACK = (
+    "thanks", "thank you", "perfect", "great", "works now", "fixed",
+    "that works", "lgtm", "looks good", "nice", "awesome", "correct",
+)
+
+
+def _extra_phrases(env_var: str) -> tuple:
+    raw = os.environ.get(env_var, "")
+    return tuple(p.strip().lower() for p in raw.split(",") if p.strip())
+
+
+_NEGATIVE_FEEDBACK = _NEGATIVE_FEEDBACK + _extra_phrases("SKILLOPT_SLEEP_NEG_FEEDBACK")
+_POSITIVE_FEEDBACK = _POSITIVE_FEEDBACK + _extra_phrases("SKILLOPT_SLEEP_POS_FEEDBACK")
+
+
+def _iter_jsonl(path: str) -> Iterable[Dict[str, Any]]:
+    try:
+        with open(path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    yield json.loads(line)
+                except Exception:
+                    continue
+    except (FileNotFoundError, IsADirectoryError, PermissionError):
+        return
+
+
+def _text_from_content(content: Any) -> str:
+    """Flatten a message.content (str or list of blocks) into text."""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: List[str] = []
+        for b in content:
+            if isinstance(b, dict):
+                if b.get("type") == "text" and b.get("text"):
+                    parts.append(str(b["text"]))
+        return "\n".join(parts)
+    return ""
+
+
+def _tool_names_from_content(content: Any) -> List[str]:
+    names: List[str] = []
+    if isinstance(content, list):
+        for b in content:
+            if isinstance(b, dict) and b.get("type") == "tool_use" and b.get("name"):
+                names.append(str(b["name"]))
+    return names
+
+
+def _detect_feedback(text: str) -> List[str]:
+    low = text.lower()
+    sig: List[str] = []
+    for ph in _NEGATIVE_FEEDBACK:
+        if ph in low:
+            sig.append("neg:" + ph)
+    for ph in _POSITIVE_FEEDBACK:
+        if ph in low:
+            sig.append("pos:" + ph)
+    return sig
+
+
+def _is_meta_prompt(text: str) -> bool:
+    """Skip slash-commands / system noise that aren't real user intents."""
+    t = text.strip()
+    if not t:
+        return True
+    if t.startswith("<") and t.endswith(">"):
+        return True
+    if t.startswith("/") and len(t.split()) <= 3:
+        return True
+    if t.startswith("[Pasted text") or t.startswith("Caveat:"):
+        return True
+    return False
+
+
+# ── Issue #62: filter headless replay sessions ─────────────────────────
+
+# Prompt markers generated by the engine's own headless `claude -p` calls
+# (judge, reflect, attempt). If the sole user prompt in a single-turn
+# session matches any of these, the session is engine-generated, not a
+# real user task.
+_REPLAY_PROMPT_MARKERS = (
+    "## CURRENT SKILL",
+    "## FAILED TASKS",
+    "## SUCCESSFUL TASKS",
+    "## OUTPUT FORMAT",
+    "You are a strict grader",
+    "Score the response 0.0-1.0",
+    "You are SkillOpt-Sleep",
+    "## TASK\n",
+    "## SKILL\n",
+)
+
+
+def _is_headless_replay(digest: "SessionDigest") -> bool:
+    """Detect sessions created by the engine's own headless replay calls.
+
+    Heuristics (conservatively applied):
+    1. Session has exactly 1 user turn AND
+    2. The sole prompt matches engine-generated patterns (grader/reflect),
+       OR the session lasted < 3 seconds (programmatic, not interactive).
+    Multi-turn sessions are always kept (interactive by definition).
+    """
+    if digest.n_user_turns > 1:
+        return False
+    if digest.n_user_turns == 0:
+        return True
+    prompt = digest.user_prompts[0] if digest.user_prompts else ""
+    for marker in _REPLAY_PROMPT_MARKERS:
+        if marker in prompt:
+            return True
+    # Sub-3-second single-turn sessions with short prompts are almost
+    # certainly programmatic (engine grader/judge calls).  We require the
+    # prompt to also be short (<200 chars) to avoid false-positives on
+    # real one-shot questions that Claude happens to answer quickly.
+    if digest.started_at and digest.ended_at and len(prompt) < 200:
+        try:
+            fmt = "%Y-%m-%dT%H:%M:%S"
+            start = datetime.strptime(digest.started_at[:19], fmt)
+            end = datetime.strptime(digest.ended_at[:19], fmt)
+            if (end - start).total_seconds() < 3:
+                return True
+        except (ValueError, TypeError):
+            pass
+    return False
+
+
+def digest_transcript(path: str) -> Optional[SessionDigest]:
+    """Build a SessionDigest from one ``<sessionId>.jsonl`` transcript."""
+    session_id = os.path.splitext(os.path.basename(path))[0]
+    project = ""
+    git_branch = ""
+    started = ""
+    ended = ""
+    user_prompts: List[str] = []
+    assistant_finals: List[str] = []
+    tools: List[str] = []
+    files: List[str] = []
+    feedback: List[str] = []
+    n_user = 0
+    n_asst = 0
+
+    for rec in _iter_jsonl(path):
+        rtype = rec.get("type")
+        ts = rec.get("timestamp")
+        if isinstance(ts, str) and ts:
+            if not started:
+                started = ts
+            ended = ts
+        if rec.get("cwd") and not project:
+            project = str(rec.get("cwd"))
+        if rec.get("gitBranch") and not git_branch:
+            git_branch = str(rec.get("gitBranch"))
+        if rtype == "file-history-snapshot":
+            snap = rec.get("snapshot") or rec.get("files") or {}
+            if isinstance(snap, dict):
+                files.extend([str(k) for k in list(snap.keys())[:20]])
+        msg = rec.get("message")
+        if not isinstance(msg, dict):
+            continue
+        role = msg.get("role")
+        content = msg.get("content")
+        if role == "user":
+            text = _text_from_content(content)
+            if text and not _is_meta_prompt(text):
+                n_user += 1
+                user_prompts.append(text.strip())
+                feedback.extend(_detect_feedback(text))
+        elif role == "assistant":
+            n_asst += 1
+            tools.extend(_tool_names_from_content(content))
+            text = _text_from_content(content)
+            if text.strip():
+                assistant_finals.append(text.strip())
+
+    if n_user == 0 and n_asst == 0:
+        return None
+
+    # de-dup tools/files preserving order
+    def _dedup(xs: List[str]) -> List[str]:
+        seen = set()
+        out = []
+        for x in xs:
+            if x not in seen:
+                seen.add(x)
+                out.append(x)
+        return out
+
+    return SessionDigest(
+        session_id=session_id,
+        project=project,
+        git_branch=git_branch,
+        started_at=started,
+        ended_at=ended,
+        user_prompts=user_prompts,
+        assistant_finals=assistant_finals[-5:],  # last few finals are the useful ones
+        tools_used=_dedup(tools),
+        files_touched=_dedup(files),
+        feedback_signals=feedback,
+        n_user_turns=n_user,
+        n_assistant_turns=n_asst,
+        raw_path=path,
+    )
+
+
+def _project_matches(project: str, scope: Any, invoked: str) -> bool:
+    if scope == "all":
+        return True
+    if isinstance(scope, (list, tuple)):
+        return any(os.path.abspath(project) == os.path.abspath(p) for p in scope)
+    # "invoked": match the invoked project (or a subdir of it)
+    if not invoked:
+        return True
+    a = os.path.abspath(project)
+    b = os.path.abspath(invoked)
+    return a == b or a.startswith(b + os.sep) or b.startswith(a + os.sep)
+
+
+def harvest(
+    transcripts_dir: str,
+    *,
+    scope: Any = "all",
+    invoked_project: str = "",
+    since_iso: Optional[str] = None,
+    limit: int = 0,
+) -> List[SessionDigest]:
+    """Walk ~/.claude/projects and return digests matching scope/time.
+
+    Parameters
+    ----------
+    transcripts_dir : str    ~/.claude/projects
+    scope : "all" | "invoked" | list[path]
+    invoked_project : str    used when scope == "invoked"
+    since_iso : str|None      ISO8601; only sessions ending after this are kept
+    limit : int               cap number of digests (0 = no cap)
+    """
+    digests: List[SessionDigest] = []
+    if not os.path.isdir(transcripts_dir):
+        return digests
+
+    paths: List[str] = []
+    for root, _dirs, files in os.walk(transcripts_dir):
+        for fn in files:
+            if fn.endswith(".jsonl"):
+                paths.append(os.path.join(root, fn))
+    # newest first by mtime
+    paths.sort(key=lambda p: os.path.getmtime(p), reverse=True)
+
+    for p in paths:
+        d = digest_transcript(p)
+        if d is None:
+            continue
+        if _is_headless_replay(d):
+            continue  # Issue #62: skip engine's own headless replay sessions
+        if not _project_matches(d.project or "", scope, invoked_project):
+            continue
+        if since_iso and d.ended_at and d.ended_at < since_iso:
+            # Note: files are sorted by mtime but we compare the embedded
+            # ended_at timestamp — mtime can diverge (copy/touch), so we
+            # cannot break here; we must continue to check all files.
+            continue
+        digests.append(d)
+        if limit and len(digests) >= limit:
+            break
+    return digests