self-evolve-framework 1.6.0 → 1.6.1

package/template/skills/skillopt-sleep/scripts/python/scheduler.py

@@ -1,138 +0,0 @@
-"""SkillOpt-Sleep — built-in nightly scheduler.
-
-Installs/removes a crontab entry that runs the sleep cycle automatically, so the
-user doesn't have to wire cron themselves. Idempotent: a managed block delimited
-by marker comments is added/replaced/removed in the user's crontab.
-
-Design choices:
-  * Off-:00 minute (3:17 local by default) so many users don't all hit the API
-    at the same instant.
-  * The entry runs `python -m skillopt_sleep run` for a specific project and
-    appends to <project>/.skillopt-sleep/cron.log.
-  * `schedule` is additive per project (keyed by project path); `unschedule`
-    removes the project's line (or the whole managed block with --all).
-
-cron is the portable mechanism on Linux/macOS. On systems without `crontab`,
-`schedule` prints the line and instructions instead of failing.
-"""
-from __future__ import annotations
-
-import os
-import shutil
-import subprocess
-import sys
-from typing import List, Optional, Tuple
-
-_BEGIN = "# >>> skillopt-sleep (managed) >>>"
-_END = "# <<< skillopt-sleep (managed) <<<"
-
-
-def _have_crontab() -> bool:
-    return shutil.which("crontab") is not None
-
-
-def _read_crontab() -> str:
-    try:
-        proc = subprocess.run(["crontab", "-l"], capture_output=True, text=True)
-        return proc.stdout if proc.returncode == 0 else ""
-    except Exception:
-        return ""
-
-
-def _write_crontab(content: str) -> bool:
-    try:
-        proc = subprocess.run(["crontab", "-"], input=content, text=True,
-                              capture_output=True)
-        return proc.returncode == 0
-    except Exception:
-        return False
-
-
-def _split_managed(crontab: str) -> Tuple[str, List[str]]:
-    """Return (text_outside_block, managed_lines_inside_block)."""
-    lines = crontab.splitlines()
-    outside: List[str] = []
-    managed: List[str] = []
-    in_block = False
-    for ln in lines:
-        if ln.strip() == _BEGIN:
-            in_block = True
-            continue
-        if ln.strip() == _END:
-            in_block = False
-            continue
-        (managed if in_block else outside).append(ln)
-    return "\n".join(outside).rstrip(), managed
-
-
-def _runner_cmd(project: str, backend: str, extra: str, python: str) -> str:
-    logdir = os.path.join(project, ".skillopt-sleep")
-    log = os.path.join(logdir, "cron.log")
-    # use absolute python + -m so cron's minimal env still works
-    cmd = (f'{python} -m skillopt_sleep run --project "{project}" '
-           f'--scope invoked --backend {backend} {extra}'.rstrip())
-    return f'mkdir -p "{logdir}"; cd "{_repo_root()}" && {cmd} >> "{log}" 2>&1'
-
-
-def _repo_root() -> str:
-    # the package lives at <repo>/skillopt_sleep/; repo root is its parent
-    return os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
-
-
-def _project_marker(project: str) -> str:
-    return f"# project={os.path.abspath(project)}"
-
-
-def schedule(project: str, *, backend: str = "mock", hour: int = 3, minute: int = 17,
-             extra: str = "", python: Optional[str] = None) -> Tuple[bool, str]:
-    """Install (or replace) the nightly entry for ``project``.
-
-    Returns (installed, message). If crontab is unavailable, installed=False and
-    the message contains the line to add manually.
-    """
-    project = os.path.abspath(project)
-    python = python or sys.executable or "python3"
-    cron_line = f"{minute} {hour} * * *  {_runner_cmd(project, backend, extra, python)}  {_project_marker(project)}"
-
-    if not _have_crontab():
-        return False, ("crontab not found on this system. Add this line to your "
-                       "scheduler manually:\n" + cron_line)
-
-    outside, managed = _split_managed(_read_crontab())
-    # drop any existing line for this project, then add the new one
-    marker = _project_marker(project)
-    managed = [ln for ln in managed if marker not in ln and ln.strip()]
-    managed.append(cron_line)
-
-    block = _BEGIN + "\n" + "\n".join(managed) + "\n" + _END
-    new_crontab = (outside + "\n\n" + block + "\n").lstrip("\n")
-    ok = _write_crontab(new_crontab)
-    if ok:
-        return True, (f"Scheduled nightly at {hour:02d}:{minute:02d} for {project} "
-                      f"(backend={backend}). Logs -> {project}/.skillopt-sleep/cron.log\n"
-                      f"Runs `skillopt_sleep run`; it only STAGES a proposal — adopt is still manual.")
-    return False, "Failed to write crontab. Line to add manually:\n" + cron_line
-
-
-def unschedule(project: Optional[str] = None, *, all_projects: bool = False) -> Tuple[bool, str]:
-    """Remove the entry for ``project`` (or the whole managed block with all_projects)."""
-    if not _have_crontab():
-        return False, "crontab not found; nothing to remove."
-    outside, managed = _split_managed(_read_crontab())
-    if all_projects:
-        managed = []
-    elif project:
-        marker = _project_marker(project)
-        managed = [ln for ln in managed if marker not in ln and ln.strip()]
-    if managed:
-        block = _BEGIN + "\n" + "\n".join(managed) + "\n" + _END
-        new_crontab = (outside + "\n\n" + block + "\n").lstrip("\n")
-    else:
-        new_crontab = outside.rstrip() + "\n"
-    ok = _write_crontab(new_crontab)
-    return ok, ("Removed." if ok else "Failed to update crontab.")
-
-
-def list_scheduled() -> List[str]:
-    _outside, managed = _split_managed(_read_crontab())
-    return [ln for ln in managed if ln.strip()]

package/template/skills/skillopt-sleep/scripts/python/slow_update.py

@@ -1,142 +0,0 @@
-"""SkillOpt-Sleep — slow update (cross-night long-term memory).
-
-This is the deployment-time analogue of SkillOpt's epoch-wise slow/meta update
-(paper §3.6). Step-level edits (consolidate) learn from one night's batch; the
-slow update learns across nights and writes a durable "longitudinal guidance"
-block into a PROTECTED field of the skill that step-level edits never touch.
-
-It reuses the exact protected-field marker convention from the main repo
-(``skillopt/optimizer/slow_update.py``) so the artifact is compatible:
-
-    <!-- SLOW_UPDATE_START --> ... <!-- SLOW_UPDATE_END -->
-
-Why it matters: even when the user turns the validation gate OFF (greedy mode),
-the slow update still runs at the end of the run, so short-term nightly
-experience is consolidated into long-term memory rather than lost. The cross-night
-content is carried in ``state.slow_memory``.
-
-Driven through the Backend abstraction (mock/claude/codex), so it stays
-import-light — no `openai` dependency.
-"""
-from __future__ import annotations
-
-import re
-from typing import List, Optional, Tuple
-
-from skillopt_sleep.backend import Backend, _extract_json
-from skillopt_sleep.types import ReplayResult, TaskRecord
-
-
-SLOW_UPDATE_START = "<!-- SLOW_UPDATE_START -->"
-SLOW_UPDATE_END = "<!-- SLOW_UPDATE_END -->"
-
-
-# ── protected-field helpers (mirror skillopt/optimizer/slow_update.py) ─────────
-
-def has_slow_field(skill: str) -> bool:
-    return SLOW_UPDATE_START in skill and SLOW_UPDATE_END in skill
-
-
-def extract_slow_field(skill: str) -> str:
-    s = skill.find(SLOW_UPDATE_START)
-    e = skill.find(SLOW_UPDATE_END)
-    if s == -1 or e == -1:
-        return ""
-    return skill[s + len(SLOW_UPDATE_START):e].strip()
-
-
-def _strip_slow_fields(skill: str) -> str:
-    while True:
-        s = skill.find(SLOW_UPDATE_START)
-        if s == -1:
-            break
-        e = skill.find(SLOW_UPDATE_END, s)
-        if e == -1:
-            skill = skill[:s]
-            break
-        skill = skill[:s] + skill[e + len(SLOW_UPDATE_END):]
-    skill = skill.replace(SLOW_UPDATE_END, "")
-    while "\n\n\n" in skill:
-        skill = skill.replace("\n\n\n", "\n\n")
-    return skill.rstrip()
-
-
-def replace_slow_field(skill: str, content: str) -> str:
-    """Set the protected slow-update field to ``content`` (exactly one block)."""
-    base = _strip_slow_fields(skill)
-    if not content.strip():
-        return base
-    block = f"\n\n{SLOW_UPDATE_START}\n{content.strip()}\n{SLOW_UPDATE_END}\n"
-    return base + block
-
-
-# ── the slow-update synthesis ──────────────────────────────────────────────────
-
-def _summarize_pairs(
-    prev_pairs: List[Tuple[TaskRecord, ReplayResult]],
-    curr_pairs: List[Tuple[TaskRecord, ReplayResult]],
-) -> str:
-    """Group adjacent-version outcomes into improved/regressed/persistent/stable."""
-    prev_by = {t.id: r for t, r in prev_pairs}
-    lines: List[str] = []
-    counts = {"improved": 0, "regressed": 0, "persistent_fail": 0, "stable_success": 0}
-    for t, r in curr_pairs:
-        p = prev_by.get(t.id)
-        if p is None:
-            continue
-        a, b = p.hard, r.hard
-        if b > a:
-            cat = "improved"
-        elif b < a:
-            cat = "regressed"
-        elif b >= 1.0:
-            cat = "stable_success"
-        else:
-            cat = "persistent_fail"
-        counts[cat] += 1
-        if cat in ("regressed", "persistent_fail") and len(lines) < 8:
-            lines.append(f"- [{cat}] {t.intent[:120]} (why: {r.fail_reason[:80]})")
-    head = ", ".join(f"{k}={v}" for k, v in counts.items())
-    return head + ("\n" + "\n".join(lines) if lines else ""), counts  # type: ignore[return-value]
-
-
-def run_slow_update(
-    backend: Backend,
-    *,
-    prev_skill: str,
-    curr_skill: str,
-    prev_pairs: List[Tuple[TaskRecord, ReplayResult]],
-    curr_pairs: List[Tuple[TaskRecord, ReplayResult]],
-    prev_slow_content: str = "",
-) -> Optional[str]:
-    """Produce durable longitudinal guidance text (or None).
-
-    Compares behavior under the previous vs current skill across the same tasks
-    and asks the optimizer to distill a short, durable guidance block — what to
-    keep doing, what regressions to avoid — refining any prior slow-update text.
-    """
-    summary, counts = _summarize_pairs(prev_pairs, curr_pairs)  # type: ignore[misc]
-    # nothing changed and no prior guidance to refine → skip
-    if counts["regressed"] == 0 and counts["persistent_fail"] == 0 and not prev_slow_content:
-        return None
-
-    prompt = (
-        "You are SkillOpt's SLOW UPDATE — the long-term memory pass that runs "
-        "across nights. Write a SHORT, durable guidance block (2-5 bullet "
-        "points) capturing the longitudinal lessons: behaviors that reliably "
-        "help and should be preserved, and regressions/persistent failures to "
-        "avoid. Keep it GENERAL and stable (not tied to one task). If prior "
-        "guidance is given, refine it rather than restate it.\n"
-        'Return ONLY JSON: {"guidance": "<bullet list as one string>"}.\n\n'
-        f"# Cross-night outcome summary\n{summary}\n\n"
-        f"# Prior long-term guidance (refine this)\n{prev_slow_content or '(none)'}"
-    )
-    raw = backend._call(prompt, max_tokens=600)  # type: ignore[attr-defined]
-    obj = _extract_json(raw, "object")
-    if isinstance(obj, dict):
-        g = str(obj.get("guidance", "")).strip()
-        if g:
-            return g
-    # fallback: if the model returned prose, keep the first ~400 chars
-    text = (raw or "").strip()
-    return text[:400] if text else None

package/template/skills/skillopt-sleep/scripts/python/staging.py

@@ -1,103 +0,0 @@
-"""SkillOpt-Sleep — Stage 5/6: staging and adoption.
-
-Implements the Dreams safety contract: the cycle never mutates the user's
-live CLAUDE.md / SKILL.md. It writes proposals + a human-readable report into
-a staging directory; a separate, explicit `adopt` step copies them over the
-live files after taking a backup.
-"""
-from __future__ import annotations
-
-import json
-import os
-import shutil
-import time
-from typing import List, Optional
-
-from skillopt_sleep.types import SleepReport
-
-
-def _ts_dir() -> str:
-    return time.strftime("%Y%m%d-%H%M%S", time.localtime())
-
-
-def staging_root(project: str) -> str:
-    return os.path.join(project, ".skillopt-sleep", "staging")
-
-
-def latest_staging(project: str) -> Optional[str]:
-    root = staging_root(project)
-    if not os.path.isdir(root):
-        return None
-    subs = sorted(
-        (os.path.join(root, d) for d in os.listdir(root)),
-        key=lambda p: os.path.getmtime(p),
-        reverse=True,
-    )
-    return subs[0] if subs else None
-
-
-def write_staging(
-    project: str,
-    *,
-    report: SleepReport,
-    proposed_skill: Optional[str],
-    proposed_memory: Optional[str],
-    live_skill_path: str,
-    live_memory_path: str,
-    report_md: str,
-) -> str:
-    """Write proposals + report into staging/<ts>/ and return that path."""
-    out = os.path.join(staging_root(project), _ts_dir())
-    os.makedirs(out, exist_ok=True)
-
-    manifest = {
-        "live_skill_path": live_skill_path,
-        "live_memory_path": live_memory_path,
-        "has_skill": proposed_skill is not None,
-        "has_memory": proposed_memory is not None,
-        "accepted": report.accepted,
-    }
-    if proposed_skill is not None:
-        with open(os.path.join(out, "proposed_SKILL.md"), "w", encoding="utf-8") as f:
-            f.write(proposed_skill)
-    if proposed_memory is not None:
-        with open(os.path.join(out, "proposed_CLAUDE.md"), "w", encoding="utf-8") as f:
-            f.write(proposed_memory)
-    with open(os.path.join(out, "report.json"), "w", encoding="utf-8") as f:
-        json.dump(report.to_dict(), f, ensure_ascii=False, indent=2)
-    with open(os.path.join(out, "report.md"), "w", encoding="utf-8") as f:
-        f.write(report_md)
-    with open(os.path.join(out, "manifest.json"), "w", encoding="utf-8") as f:
-        json.dump(manifest, f, ensure_ascii=False, indent=2)
-    return out
-
-
-def _backup(path: str, backup_dir: str) -> None:
-    if os.path.exists(path):
-        os.makedirs(backup_dir, exist_ok=True)
-        shutil.copy2(path, os.path.join(backup_dir, os.path.basename(path)))
-
-
-def adopt(staging_dir: str) -> List[str]:
-    """Copy staged proposals over the live files, backing up first.
-
-    Returns the list of live paths that were updated.
-    """
-    with open(os.path.join(staging_dir, "manifest.json")) as f:
-        manifest = json.load(f)
-    backup_dir = os.path.join(staging_dir, "backup")
-    updated: List[str] = []
-
-    if manifest.get("has_skill"):
-        live = manifest["live_skill_path"]
-        os.makedirs(os.path.dirname(live), exist_ok=True)
-        _backup(live, backup_dir)
-        shutil.copy2(os.path.join(staging_dir, "proposed_SKILL.md"), live)
-        updated.append(live)
-    if manifest.get("has_memory"):
-        live = manifest["live_memory_path"]
-        os.makedirs(os.path.dirname(live), exist_ok=True)
-        _backup(live, backup_dir)
-        shutil.copy2(os.path.join(staging_dir, "proposed_CLAUDE.md"), live)
-        updated.append(live)
-    return updated

package/template/skills/skillopt-sleep/scripts/python/state.py

@@ -1,96 +0,0 @@
-"""SkillOpt-Sleep — persistent cross-night state.
-
-state.json lives in ~/.skillopt-sleep and is the "long-term" store that
-turns nightly episodes into durable competence (the Agent-Sleep paper's
-short-term -> long-term transfer). It records:
-
-  - night counter
-  - last harvest timestamp per project (so each night only sees new data)
-  - cross-night "slow/meta" memory (lessons that persisted across nights)
-  - per-night history (scores, accept/reject) for trend reporting
-"""
-from __future__ import annotations
-
-import json
-import os
-from typing import Any, Dict, List, Optional
-
-
-def _now_iso(clock: Optional[float] = None) -> str:
-    # caller passes a timestamp; we avoid importing time at module import
-    import time as _t
-    return _t.strftime("%Y-%m-%dT%H:%M:%S", _t.localtime(clock if clock is not None else _t.time()))
-
-
-DEFAULT_STATE: Dict[str, Any] = {
-    "version": 1,
-    "night": 0,
-    "last_harvest": {},     # project -> iso timestamp of last harvested record
-    "slow_memory": "",      # cross-night consolidated lessons (meta-skill analogue)
-    "history": [],          # list of per-night summaries
-    "task_archive": [],     # capped list of past mined tasks (for associative recall)
-}
-
-
-class SleepState:
-    def __init__(self, path: str, data: Optional[Dict[str, Any]] = None) -> None:
-        self.path = path
-        self.data = data if data is not None else dict(DEFAULT_STATE)
-
-    # io ---------------------------------------------------------------------
-    @classmethod
-    def load(cls, path: str) -> "SleepState":
-        if os.path.exists(path):
-            try:
-                with open(path) as f:
-                    data = json.load(f)
-                merged = dict(DEFAULT_STATE)
-                merged.update(data if isinstance(data, dict) else {})
-                return cls(path, merged)
-            except Exception:
-                pass
-        return cls(path, dict(DEFAULT_STATE))
-
-    def save(self) -> None:
-        os.makedirs(os.path.dirname(self.path), exist_ok=True)
-        tmp = self.path + ".tmp"
-        with open(tmp, "w") as f:
-            json.dump(self.data, f, ensure_ascii=False, indent=2)
-        os.replace(tmp, self.path)
-
-    # accessors --------------------------------------------------------------
-    @property
-    def night(self) -> int:
-        return int(self.data.get("night", 0))
-
-    def last_harvest_for(self, project: str) -> Optional[str]:
-        return self.data.get("last_harvest", {}).get(project)
-
-    def set_last_harvest(self, project: str, iso_ts: str) -> None:
-        self.data.setdefault("last_harvest", {})[project] = iso_ts
-
-    @property
-    def slow_memory(self) -> str:
-        return str(self.data.get("slow_memory", ""))
-
-    def set_slow_memory(self, content: str) -> None:
-        self.data["slow_memory"] = content
-
-    def begin_night(self, clock: Optional[float] = None) -> int:
-        self.data["night"] = self.night + 1
-        return self.night
-
-    def record_night(self, summary: Dict[str, Any]) -> None:
-        self.data.setdefault("history", []).append(summary)
-
-    # ── task archive (associative-recall memory) ──────────────────────────
-    def task_archive(self) -> list:
-        """Past mined tasks as plain dicts (newest last)."""
-        return list(self.data.get("task_archive", []))
-
-    def add_to_archive(self, task_dicts: list, cap: int = 300) -> None:
-        """Append tonight's tasks; keep only the most recent ``cap``."""
-        arc = self.data.setdefault("task_archive", [])
-        arc.extend(task_dicts)
-        if len(arc) > cap:
-            self.data["task_archive"] = arc[-cap:]

package/template/skills/skillopt-sleep/scripts/python/tasks_file.py

@@ -1,81 +0,0 @@
-"""Reviewed task-file helpers for privacy-safe SkillOpt-Sleep runs."""
-from __future__ import annotations
-
-import json
-import os
-from typing import Any, Dict, List, Tuple
-
-from skillopt_sleep.mine import assign_splits, normalize_legacy_split
-from skillopt_sleep.types import TaskRecord
-
-
-def make_tasks_payload(
-    tasks: List[TaskRecord],
-    *,
-    project: str,
-    transcript_source: str = "",
-    n_sessions: int = 0,
-    target_skill_path: str = "",
-) -> Dict[str, Any]:
-    return {
-        "format": "skillopt_sleep.tasks.v1",
-        "project": project,
-        "transcript_source": transcript_source,
-        "n_sessions": n_sessions,
-        "target_skill_path": target_skill_path,
-        "reviewed": False,
-        "tasks": [t.to_dict() for t in tasks],
-    }
-
-
-def write_tasks_file(path: str, payload: Dict[str, Any]) -> str:
-    out = os.path.abspath(os.path.expanduser(path))
-    parent = os.path.dirname(out)
-    if parent:
-        os.makedirs(parent, exist_ok=True)
-    with open(out, "w", encoding="utf-8") as f:
-        json.dump(payload, f, ensure_ascii=False, indent=2)
-        f.write("\n")
-    return out
-
-
-def _normalize_tasks(
-    tasks: List[TaskRecord],
-    *,
-    holdout_fraction: float,
-    seed: int,
-) -> List[TaskRecord]:
-    for task in tasks:
-        task.split = normalize_legacy_split(task.split or "train")
-    if len(tasks) >= 2 and not any(task.split in {"val", "test"} for task in tasks):
-        tasks = assign_splits(tasks, holdout_fraction=holdout_fraction, seed=seed)
-    return tasks
-
-
-def load_tasks_file(
-    path: str,
-    *,
-    holdout_fraction: float = 0.34,
-    seed: int = 42,
-) -> Tuple[List[TaskRecord], Dict[str, Any]]:
-    source = os.path.abspath(os.path.expanduser(path))
-    with open(source, encoding="utf-8") as f:
-        payload = json.load(f)
-    if isinstance(payload, list):
-        meta: Dict[str, Any] = {"format": "skillopt_sleep.tasks.v1", "tasks_file": source}
-        raw_tasks = payload
-    elif isinstance(payload, dict):
-        meta = {k: v for k, v in payload.items() if k != "tasks"}
-        meta["tasks_file"] = source
-        raw_tasks = payload.get("tasks", [])
-    else:
-        raise ValueError("tasks file must contain a JSON object with tasks or a JSON task array")
-    if not isinstance(raw_tasks, list):
-        raise ValueError("tasks file field 'tasks' must be an array")
-
-    tasks: List[TaskRecord] = []
-    for item in raw_tasks:
-        if not isinstance(item, dict):
-            raise ValueError("each task entry must be a JSON object")
-        tasks.append(TaskRecord.from_dict(item))
-    return _normalize_tasks(tasks, holdout_fraction=holdout_fraction, seed=seed), meta