self-evolve-framework 1.6.0 → 1.6.1

package/template/skills/skillopt-sleep/scripts/python/budget.py

@@ -1,75 +0,0 @@
-"""SkillOpt-Sleep — budget controller.
-
-Lets the user say how much they're willing to spend on a night's "dreaming",
-in tokens or wall-clock minutes, and the engine schedules depth (how many
-rollouts × how many nights) within that budget. Stops cleanly when exhausted
-and reports what it skipped (no silent truncation).
-"""
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import Optional
-
-
-@dataclass
-class Budget:
-    max_tokens: Optional[int] = None      # None = unlimited
-    max_minutes: Optional[float] = None   # None = unlimited
-    _start_time: Optional[float] = None
-    _tokens_at_start: int = 0
-
-    def start(self, clock_fn, tokens_now: int) -> None:
-        self._start_time = clock_fn()
-        self._tokens_at_start = tokens_now
-
-    def tokens_spent(self, tokens_now: int) -> int:
-        return max(0, tokens_now - self._tokens_at_start)
-
-    def minutes_elapsed(self, clock_fn) -> float:
-        if self._start_time is None:
-            return 0.0
-        return (clock_fn() - self._start_time) / 60.0
-
-    def remaining_fraction(self, *, tokens_now: int, clock_fn) -> float:
-        """Smallest remaining fraction across all active limits (1.0 = fresh)."""
-        fracs = [1.0]
-        if self.max_tokens:
-            fracs.append(max(0.0, 1.0 - self.tokens_spent(tokens_now) / self.max_tokens))
-        if self.max_minutes:
-            fracs.append(max(0.0, 1.0 - self.minutes_elapsed(clock_fn) / self.max_minutes))
-        return min(fracs)
-
-    def exhausted(self, *, tokens_now: int, clock_fn) -> bool:
-        if self.max_tokens and self.tokens_spent(tokens_now) >= self.max_tokens:
-            return True
-        if self.max_minutes and self.minutes_elapsed(clock_fn) >= self.max_minutes:
-            return True
-        return False
-
-    def status(self, *, tokens_now: int, clock_fn) -> str:
-        parts = []
-        if self.max_tokens:
-            parts.append(f"tokens {self.tokens_spent(tokens_now)}/{self.max_tokens}")
-        if self.max_minutes:
-            parts.append(f"minutes {self.minutes_elapsed(clock_fn):.1f}/{self.max_minutes}")
-        return ", ".join(parts) or "unbounded"
-
-
-def plan_depth(budget: Budget, *, n_tasks: int,
-               default_nights: int = 2, default_k: int = 1) -> tuple:
-    """Heuristically choose (nights, rollouts_per_task) from a token budget.
-
-    Rough cost model: one rollout ≈ 1 unit; a night does ~n_tasks*k rollouts
-    plus reflect/gate (~2*n_tasks). We scale k and nights up with more budget.
-    Returns (nights, k). With no budget set, returns the defaults.
-    """
-    if not budget.max_tokens:
-        return default_nights, default_k
-    # assume ~1.5k tokens per rollout as a planning constant
-    rollouts_affordable = budget.max_tokens / 1500.0
-    per_night = max(1, n_tasks) * 3  # rollouts + reflect + gate, k=1
-    nights = max(1, min(4, int(rollouts_affordable // per_night)))
-    # spend surplus on more rollouts-per-task (contrastive signal)
-    surplus = rollouts_affordable - nights * per_night
-    k = max(1, min(5, 1 + int(surplus // max(1, n_tasks))))
-    return nights, k

package/template/skills/skillopt-sleep/scripts/python/config.py

@@ -1,162 +0,0 @@
-"""SkillOpt-Sleep — configuration.
-
-Config is JSON-first (yaml optional) so the engine and the deterministic
-experiment run with zero external dependencies. Defaults are safe:
-review-gated adoption, single-project scope, bounded token/task budgets.
-
-Resolution order (later wins):
-  1. built-in DEFAULTS
-  2. ~/.skillopt-sleep/config.json  (or .yaml if PyYAML available)
-  3. explicit overrides passed to load_config(**overrides)
-"""
-from __future__ import annotations
-
-import json
-import os
-from dataclasses import dataclass, field
-from typing import Any, Dict, Optional
-
-HOME_STATE_DIR = os.path.expanduser("~/.skillopt-sleep")
-CLAUDE_HOME = os.path.expanduser("~/.claude")
-CODEX_HOME = os.path.expanduser("~/.codex")
-
-
-DEFAULTS: Dict[str, Any] = {
-    # ── scope ──────────────────────────────────────────────────────────────
-    "claude_home": CLAUDE_HOME,
-    "codex_home": CODEX_HOME,
-    "transcript_source": "claude",  # "claude" | "codex" | "auto"
-    "projects": "invoked",        # "invoked" | "all" | [list of abs paths]
-    "invoked_project": "",        # filled at runtime (cwd) when projects == "invoked"
-    "lookback_hours": 72,         # harvest window when no prior sleep recorded
-    # ── budgets ────────────────────────────────────────────────────────────
-    "max_tasks_per_night": 40,
-    "max_tokens_per_night": 400_000,
-    "holdout_fraction": 0.34,     # legacy alias for val_fraction
-    "val_fraction": 0.34,         # real tasks reserved to gate updates
-    "test_fraction": 0.0,         # real tasks reserved as the final held-out measure
-    # ── optimizer ──────────────────────────────────────────────────────────
-    "backend": "mock",            # "mock" | "claude" | "codex" | "copilot"
-    "model": "",                  # backend-specific; "" => backend default
-    "gate_mode": "on",            # "on" (validation-gated) | "off" (greedy, no hard filter)
-    "codex_path": "",             # "" => auto-detect the real @openai/codex binary
-    "edit_budget": 4,             # textual learning rate (max edits/night)
-    "gate_metric": "mixed",       # hard | soft | mixed (mixed best for tiny holdouts)
-    "gate_mixed_weight": 0.5,
-    "replay_mode": "mock",        # "mock" (sandboxed prompt) | "fresh" (worktree)
-    # ── dream + recall (opt-in; defaults reproduce the prior single-shot loop) ─
-    "dream_rollouts": 1,          # >1 => multi-rollout contrastive reflection per task
-    "dream_factor": 0,            # >0 => add N synthetic variants of each task to the dream
-    "recall_k": 0,                # >0 => recall the K most-similar past tasks into the dream
-    "evolve_memory": True,        # consolidate CLAUDE.md
-    "evolve_skill": True,         # consolidate the managed SKILL.md
-    "llm_mine": True,             # use the backend to mine checkable tasks (real backends)
-    "target_skill_path": "",      # explicit SKILL.md target for repo-scoped agents
-    "target_task_filter": True,   # prefer mined tasks matching target_skill_path/text
-    "progress": False,            # print phase progress to stderr
-    # ── adoption / safety ──────────────────────────────────────────────────
-    "auto_adopt": False,          # default: stage + require explicit `adopt`
-    "managed_skill_name": "skillopt-sleep-learned",
-    "redact_secrets": True,
-    "seed": 42,
-}
-
-
-@dataclass
-class SleepConfig:
-    data: Dict[str, Any] = field(default_factory=lambda: dict(DEFAULTS))
-
-    # convenient attribute access -------------------------------------------
-    def __getattr__(self, name: str) -> Any:
-        # only called when normal attribute lookup fails
-        data = object.__getattribute__(self, "data")
-        if name in data:
-            return data[name]
-        raise AttributeError(name)
-
-    def get(self, key: str, default: Any = None) -> Any:
-        return self.data.get(key, default)
-
-    def to_dict(self) -> Dict[str, Any]:
-        return dict(self.data)
-
-    # paths ------------------------------------------------------------------
-    @property
-    def state_dir(self) -> str:
-        # Allow full isolation: if the caller overrides state_dir explicitly,
-        # honor it; else derive from claude_home's parent so a single
-        # --claude-home flag isolates transcripts AND state together; else the
-        # default ~/.skillopt-sleep.
-        explicit = self.data.get("state_dir")
-        if explicit:
-            return explicit
-        ch = self.data.get("claude_home", CLAUDE_HOME)
-        if os.path.abspath(ch) != os.path.abspath(CLAUDE_HOME):
-            return os.path.join(os.path.dirname(os.path.abspath(ch)), ".skillopt-sleep")
-        return HOME_STATE_DIR
-
-    @property
-    def state_path(self) -> str:
-        return os.path.join(self.state_dir, "state.json")
-
-    @property
-    def transcripts_dir(self) -> str:
-        return os.path.join(self.data["claude_home"], "projects")
-
-    @property
-    def codex_archived_sessions_dir(self) -> str:
-        return os.path.join(self.data["codex_home"], "archived_sessions")
-
-    @property
-    def history_path(self) -> str:
-        return os.path.join(self.data["claude_home"], "history.jsonl")
-
-    @property
-    def skills_dir(self) -> str:
-        return os.path.join(self.data["claude_home"], "skills")
-
-    def managed_skill_path(self) -> str:
-        target = self.data.get("target_skill_path") or ""
-        if target:
-            target = os.path.expanduser(str(target))
-            if not os.path.isabs(target):
-                base = self.data.get("invoked_project") or os.getcwd()
-                target = os.path.join(base, target)
-            return os.path.abspath(target)
-        return os.path.join(
-            self.skills_dir, self.data["managed_skill_name"], "SKILL.md"
-        )
-
-
-def _user_config_path() -> Optional[str]:
-    for name in ("config.json", "config.yaml", "config.yml"):
-        p = os.path.join(HOME_STATE_DIR, name)
-        if os.path.exists(p):
-            return p
-    return None
-
-
-def _load_file(path: str) -> Dict[str, Any]:
-    if path.endswith((".yaml", ".yml")):
-        try:
-            import yaml  # optional
-            with open(path) as f:
-                return yaml.safe_load(f) or {}
-        except Exception:
-            return {}
-    with open(path) as f:
-        return json.load(f)
-
-
-def load_config(**overrides: Any) -> SleepConfig:
-    data = dict(DEFAULTS)
-    path = _user_config_path()
-    if path:
-        try:
-            data.update(_load_file(path) or {})
-        except Exception:
-            pass
-    data.update({k: v for k, v in overrides.items() if v is not None})
-    if data.get("projects") == "invoked" and not data.get("invoked_project"):
-        data["invoked_project"] = os.getcwd()
-    return SleepConfig(data=data)

package/template/skills/skillopt-sleep/scripts/python/consolidate.py

@@ -1,238 +0,0 @@
-"""SkillOpt-Sleep — Stage 4: consolidate (one SkillOpt epoch).
-
-This is the core that makes nightly evolution *safe*: it proposes bounded
-edits from replayed failures, applies them to a candidate skill/memory, then
-**gates** the candidate on a held-out slice of the user's own tasks. Only a
-candidate that strictly improves the held-out score is accepted — the SkillOpt
-validation gate, vendored self-contained in ``skillopt_sleep.gate``.
-"""
-from __future__ import annotations
-
-import os
-from dataclasses import dataclass
-from typing import List, Optional, Tuple
-
-from skillopt_sleep.backend import Backend
-from skillopt_sleep.memory import apply_edits
-from skillopt_sleep.replay import aggregate_scores, replay_batch
-from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord
-
-
-# Self-contained validation gate (vendored from SkillOpt; zero dependency on the
-# research package, so this open-source tool stays decoupled from the paper code).
-from skillopt_sleep.gate import evaluate_gate, select_gate_score
-_HAVE_REPO_GATE = True
-
-
-@dataclass
-class ConsolidationResult:
-    accepted: bool
-    gate_action: str
-    baseline_score: float
-    candidate_score: float
-    new_skill: str
-    new_memory: str
-    applied_edits: List[EditRecord]
-    rejected_edits: List[EditRecord]
-    holdout_baseline: float
-    holdout_candidate: float
-
-
-def _split(tasks: List[TaskRecord]) -> Tuple[List[TaskRecord], List[TaskRecord]]:
-    """Return (train_tasks, val_tasks).
-
-    train drives reflect; val gates updates. test is held out entirely from
-    consolidation and is scored by the caller. Accepts legacy split names
-    (replay->train, holdout->val) for robustness.
-    """
-    def _norm(s: str) -> str:
-        return {"replay": "train", "holdout": "val"}.get(s, s)
-
-    train = [t for t in tasks if _norm(t.split) == "train"]
-    val = [t for t in tasks if _norm(t.split) == "val"]
-    # be robust if a split is empty: fall back so a night still does something,
-    # but never silently use test as val.
-    test = [t for t in tasks if _norm(t.split) == "test"]
-    if not val:
-        # prefer train as the gate reference over nothing; last resort all-but-test
-        val = train or [t for t in tasks if _norm(t.split) != "test"] or tasks
-    if not train:
-        train = val
-    return train, val
-
-
-def consolidate(
-    backend: Backend,
-    tasks: List[TaskRecord],
-    skill: str,
-    memory: str,
-    *,
-    edit_budget: int = 4,
-    gate_metric: str = "mixed",
-    gate_mixed_weight: float = 0.5,
-    gate_mode: str = "on",       # "on" (hard/soft per gate_metric) | "off" (greedy)
-    rollouts_k: int = 1,         # >1 => multi-rollout contrastive reflection
-    evolve_skill: bool = True,
-    evolve_memory: bool = True,
-    night: int = 1,
-) -> ConsolidationResult:
-    """Run one consolidation epoch: reflect -> bounded edit -> gate.
-
-    train tasks drive reflect; val tasks gate the update (test is held out by the
-    caller). With ``gate_mode='off'`` edits are accepted greedily (no val-improve
-    requirement) — the user opts out of hard filtering — but val scores are still
-    recorded so the report shows whether quality moved.
-
-    Skill and memory are evolved in sequence (skill first if both enabled).
-    """
-    train_tasks, val_tasks = _split(tasks)
-    gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"}
-
-    # ── baseline on the VAL slice (the gate reference) ────────────────────
-    # When the gate is OFF the user has opted out of holding out a validation set
-    # (the daily-use design): we accept edits greedily and judge quality only on
-    # the real test set, scored by the caller. So we SKIP all val scoring — it is
-    # both wasted cost and contrary to the "no val set required" design.
-    if gate_off:
-        base_hard, base_soft = 0.0, 0.0
-    else:
-        base_pairs = replay_batch(backend, val_tasks, skill, memory)
-        base_hard, base_soft = aggregate_scores(base_pairs)
-    base_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
-
-    # ── reflect over TRAIN-split failures/successes ───────────────────────
-    train_pairs = replay_batch(backend, train_tasks, skill, memory)
-    failures = [(t, r) for (t, r) in train_pairs if r.hard < 1.0]
-    successes = [(t, r) for (t, r) in train_pairs if r.hard >= 1.0]
-
-    cand_skill, cand_memory = skill, memory
-    all_applied: List[EditRecord] = []
-    all_rejected: List[EditRecord] = []
-
-    def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str:
-        nonlocal cand_skill, cand_memory, base_score, all_applied, all_rejected
-        if not edits:
-            return doc
-        new_doc, applied = apply_edits(doc, edits)
-        if not applied:
-            return doc
-        # gate OFF: accept greedily with NO val scoring (the daily-use path)
-        if gate_off:
-            all_applied.extend(applied)
-            return new_doc
-        # gate ON: score the candidate on the VAL slice, keep only if it improves
-        trial_skill = new_doc if which == "skill" else cand_skill
-        trial_memory = new_doc if which == "memory" else cand_memory
-        pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory)
-        h, s = aggregate_scores(pairs)
-        cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight)
-        if cand_score > base_score:
-            base_score = max(base_score, cand_score)
-            all_applied.extend(applied)
-            return new_doc
-        all_rejected.extend(applied)
-        return doc
-
-    if evolve_skill:
-        if rollouts_k > 1:
-            # multi-rollout contrastive reflection: run each train task K times
-            # and distill a rule from the good-vs-bad contrast (the imagination signal).
-            from skillopt_sleep.rollout import multi_rollout, contrastive_reflect
-            # Parallelize across tasks (each multi_rollout also parallelizes its K
-            # attempts). This dream phase is the dominant cost; serial execution
-            # times out on real backends. Cap total in-flight at the worker env.
-            import os
-            from concurrent.futures import ThreadPoolExecutor
-            try:
-                _w = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1"))
-            except ValueError:
-                _w = 1
-            if _w > 1 and len(train_tasks) > 1:
-                # split the worker budget between task-parallelism and per-task K
-                task_workers = max(1, min(len(train_tasks), _w))
-                per_task = max(1, _w // task_workers)
-                with ThreadPoolExecutor(max_workers=task_workers) as ex:
-                    sets = list(ex.map(
-                        lambda t: multi_rollout(backend, t, cand_skill, cand_memory,
-                                                k=rollouts_k, workers=per_task),
-                        train_tasks))
-            else:
-                sets = [multi_rollout(backend, t, cand_skill, cand_memory,
-                                      k=rollouts_k, workers=1)
-                        for t in train_tasks]
-            edits = contrastive_reflect(
-                backend, sets, cand_skill, cand_memory,
-                edit_budget=edit_budget, target="skill",
-            )
-            # fall back to single-shot reflect if contrast yielded nothing
-            if not edits:
-                edits = backend.reflect(
-                    failures, successes, cand_skill, cand_memory,
-                    edit_budget=edit_budget, evolve_skill=True, evolve_memory=False,
-                )
-        else:
-            edits = backend.reflect(
-                failures, successes, cand_skill, cand_memory,
-                edit_budget=edit_budget, evolve_skill=True, evolve_memory=False,
-            )
-        cand_skill = _gate_apply(cand_skill, edits, "skill")
-
-    if evolve_memory:
-        # re-evaluate failures under the (possibly improved) skill
-        train_pairs2 = replay_batch(backend, train_tasks, cand_skill, cand_memory)
-        failures2 = [(t, r) for (t, r) in train_pairs2 if r.hard < 1.0]
-        successes2 = [(t, r) for (t, r) in train_pairs2 if r.hard >= 1.0]
-        edits_m = backend.reflect(
-            failures2, successes2, cand_skill, cand_memory,
-            edit_budget=edit_budget, evolve_skill=False, evolve_memory=True,
-        )
-        cand_memory = _gate_apply(cand_memory, edits_m, "memory")
-
-    # ── final decision ────────────────────────────────────────────────────
-    if gate_off:
-        # greedy mode: no val scoring at all. Keep whatever edits we applied; the
-        # caller measures real quality on the test set. We report holdout_candidate
-        # as 0.0 (val intentionally not computed in this variant).
-        final_hard, final_soft = 0.0, 0.0
-        final_score = 0.0
-        accepted = bool(all_applied)
-        action = "greedy_applied" if all_applied else "greedy_noop"
-        base_gate_score = 0.0
-    else:
-        # scored on the VAL slice (the gate reference)
-        final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory)
-        final_hard, final_soft = aggregate_scores(final_pairs)
-        final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight)
-        base_gate_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
-        if _HAVE_REPO_GATE:
-            gate = evaluate_gate(
-                candidate_skill=cand_skill,
-                cand_hard=final_hard,
-                current_skill=skill,
-                current_score=base_gate_score,
-                best_skill=skill,
-                best_score=base_gate_score,
-                best_step=night - 1,
-                global_step=night,
-                cand_soft=final_soft,
-                metric=gate_metric,
-                mixed_weight=gate_mixed_weight,
-            )
-            action = gate.action
-            accepted = bool(all_applied) and final_score > base_gate_score
-        else:
-            action = "accept" if final_score > base_gate_score else "reject"
-            accepted = bool(all_applied) and final_score > base_gate_score
-
-    return ConsolidationResult(
-        accepted=accepted,
-        gate_action=action,
-        baseline_score=base_gate_score,
-        candidate_score=final_score,
-        new_skill=cand_skill if accepted else skill,
-        new_memory=cand_memory if accepted else memory,
-        applied_edits=all_applied,
-        rejected_edits=all_rejected,
-        holdout_baseline=base_hard,
-        holdout_candidate=final_hard,
-    )