self-evolve-framework 1.3.0 → 1.4.0

package/template/skills/skillopt-sleep/scripts/python/harvest_codex.py

@@ -0,0 +1,253 @@
+"""SkillOpt-Sleep Codex Desktop session harvesting.
+
+Reads Codex Desktop archived session JSONL files and normalizes them into
+``SessionDigest`` records without copying developer/system instructions, tool
+arguments, or raw tool outputs.
+"""
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Dict, Iterable, List, Optional
+
+from skillopt_sleep.harvest import (
+    _detect_feedback,
+    _is_meta_prompt,
+    _iter_jsonl,
+    _project_matches,
+)
+from skillopt_sleep.types import SessionDigest
+
+_SECRET_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
+    (re.compile(r"sk-[A-Za-z0-9_-]{10,}"), "[REDACTED_OPENAI_KEY]"),
+    (re.compile(r"(?i)(Authorization:\s*Bearer\s+)[^\s\"']+"), r"\1[REDACTED]"),
+    (re.compile(r"(?i)(Authorization:\s*Basic\s+)[^\s\"']+"), r"\1[REDACTED]"),
+    (
+        re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b(\s*[:=]\s*)[^\s\"']+"),
+        r"\1\2[REDACTED]",
+    ),
+    (
+        re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b(\s+)[^\s\"']+"),
+        r"\1\2[REDACTED]",
+    ),
+    (
+        re.compile(
+            r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----",
+            re.DOTALL,
+        ),
+        "[REDACTED_PRIVATE_KEY]",
+    ),
+)
+
+
+def _payload(rec: Dict[str, Any]) -> Dict[str, Any]:
+    payload = rec.get("payload")
+    return payload if isinstance(payload, dict) else {}
+
+
+def _timestamp(rec: Dict[str, Any], payload: Dict[str, Any]) -> str:
+    for value in (
+        payload.get("timestamp"),
+        rec.get("timestamp"),
+        payload.get("started_at"),
+        payload.get("completed_at"),
+    ):
+        if isinstance(value, str) and value:
+            return value
+    return ""
+
+
+def _text_from_any(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: List[str] = []
+        for item in content:
+            if isinstance(item, str):
+                parts.append(item)
+            elif isinstance(item, dict):
+                if item.get("type") == "text" and item.get("text"):
+                    parts.append(str(item["text"]))
+                elif item.get("text"):
+                    parts.append(str(item["text"]))
+        return "\n".join(parts)
+    if isinstance(content, dict):
+        if content.get("text"):
+            return str(content["text"])
+        if content.get("content"):
+            return _text_from_any(content["content"])
+    return ""
+
+
+def _strip_codex_meta(text: str) -> str:
+    stripped = text.strip()
+    if not stripped:
+        return ""
+    if stripped.startswith("<codex_internal_context"):
+        return ""
+    if stripped.startswith("<environment_context"):
+        return ""
+    if stripped.startswith("# AGENTS.md instructions") or "--- project-doc ---" in stripped:
+        for marker in ("</environment_context>", "</INSTRUCTIONS>"):
+            idx = stripped.rfind(marker)
+            if idx == -1:
+                continue
+            tail = stripped[idx + len(marker):].strip()
+            if tail and not tail.startswith("<"):
+                return tail
+        return ""
+    return stripped
+
+
+def _sanitize_text(text: str) -> str:
+    sanitized = _strip_codex_meta(text).replace("\x00", "").strip()
+    if not sanitized or _is_meta_prompt(sanitized):
+        return ""
+    for pattern, replacement in _SECRET_PATTERNS:
+        sanitized = pattern.sub(replacement, sanitized)
+    return sanitized
+
+
+def _sanitize_tool_name(name: str) -> str:
+    return re.sub(r"[^A-Za-z0-9_.:-]+", "_", name)[:80]
+
+
+def _tool_name(payload: Dict[str, Any]) -> str:
+    payload_type = payload.get("type")
+    name = payload.get("name")
+    if isinstance(name, str) and name:
+        return _sanitize_tool_name(name)
+    if payload_type == "exec_command_end":
+        return "exec_command"
+    if payload_type == "patch_apply_end":
+        return "apply_patch"
+    if payload_type == "web_search_call":
+        return "web_search"
+    if payload_type == "tool_search_call":
+        return "tool_search"
+    if isinstance(payload_type, str) and payload_type.endswith("_tool_call"):
+        return _sanitize_tool_name(payload_type)
+    return ""
+
+
+def _dedup(xs: Iterable[str]) -> List[str]:
+    seen = set()
+    out: List[str] = []
+    for x in xs:
+        if x not in seen:
+            seen.add(x)
+            out.append(x)
+    return out
+
+
+def digest_codex_archived_session(path: str, project: str = "") -> Optional[SessionDigest]:
+    """Build a ``SessionDigest`` from one Codex Desktop archived session."""
+    session_id = os.path.splitext(os.path.basename(path))[0]
+    started = ""
+    ended = ""
+    session_project = ""
+    user_prompts: List[str] = []
+    assistant_finals: List[str] = []
+    tools: List[str] = []
+    feedback: List[str] = []
+    n_user = 0
+    n_asst = 0
+
+    for rec in _iter_jsonl(path):
+        payload = _payload(rec)
+        payload_type = payload.get("type")
+        ts = _timestamp(rec, payload)
+        if ts:
+            if not started:
+                started = ts
+            ended = ts
+        cwd = payload.get("cwd")
+        if isinstance(cwd, str) and cwd:
+            if not session_project:
+                session_project = cwd
+            if project and _project_matches(cwd, "invoked", project):
+                session_project = cwd
+
+        role = payload.get("role")
+        text = ""
+        output_role = ""
+        if payload_type == "user_message":
+            text = _text_from_any(payload.get("message"))
+            output_role = "user"
+        elif payload_type == "agent_message":
+            text = _text_from_any(payload.get("message"))
+            output_role = "assistant"
+        elif payload_type == "message" and role in {"user", "assistant"}:
+            text = _text_from_any(payload.get("content"))
+            output_role = str(role)
+        else:
+            tool = _tool_name(payload)
+            if tool:
+                tools.append(tool)
+            continue
+
+        sanitized = _sanitize_text(text)
+        if not sanitized:
+            continue
+        if output_role == "user":
+            n_user += 1
+            user_prompts.append(sanitized)
+            feedback.extend(_detect_feedback(sanitized))
+        elif output_role == "assistant":
+            n_asst += 1
+            assistant_finals.append(sanitized)
+
+    if project and not _project_matches(session_project or "", "invoked", project):
+        return None
+    if n_user == 0 and n_asst == 0:
+        return None
+
+    return SessionDigest(
+        session_id=session_id,
+        project=session_project,
+        started_at=started,
+        ended_at=ended,
+        user_prompts=user_prompts,
+        assistant_finals=assistant_finals[-5:],
+        tools_used=_dedup(tools),
+        files_touched=[],
+        feedback_signals=feedback,
+        n_user_turns=n_user,
+        n_assistant_turns=n_asst,
+        raw_path=path,
+    )
+
+
+def harvest_codex(
+    archived_sessions_dir: str,
+    *,
+    scope: Any = "all",
+    invoked_project: str = "",
+    since_iso: Optional[str] = None,
+    limit: int = 0,
+) -> List[SessionDigest]:
+    """Walk ``~/.codex/archived_sessions`` and return matching digests."""
+    digests: List[SessionDigest] = []
+    if not os.path.isdir(archived_sessions_dir):
+        return digests
+
+    paths = [
+        os.path.join(archived_sessions_dir, fn)
+        for fn in os.listdir(archived_sessions_dir)
+        if fn.endswith(".jsonl")
+    ]
+    paths.sort(key=lambda p: os.path.getmtime(p), reverse=True)
+
+    project_hint = invoked_project if scope == "invoked" else ""
+    for path in paths:
+        digest = digest_codex_archived_session(path, project=project_hint)
+        if digest is None:
+            continue
+        if not _project_matches(digest.project or "", scope, invoked_project):
+            continue
+        if since_iso and digest.ended_at and digest.ended_at < since_iso:
+            continue
+        digests.append(digest)
+        if limit and len(digests) >= limit:
+            break
+    return digests

package/template/skills/skillopt-sleep/scripts/python/harvest_sources.py

@@ -0,0 +1,41 @@
+"""Source selection for SkillOpt-Sleep transcript harvesting."""
+from __future__ import annotations
+
+from typing import Optional
+
+from skillopt_sleep.harvest import harvest
+from skillopt_sleep.harvest_codex import harvest_codex
+from skillopt_sleep.types import SessionDigest
+
+
+def harvest_for_config(cfg, *, since_iso: Optional[str] = None, limit: int = 0) -> list[SessionDigest]:
+    source = cfg.get("transcript_source", "claude")
+    scope = cfg.get("projects", "invoked")
+    invoked_project = cfg.get("invoked_project", "")
+
+    if source == "codex":
+        return harvest_codex(
+            cfg.codex_archived_sessions_dir,
+            scope=scope,
+            invoked_project=invoked_project,
+            since_iso=since_iso,
+            limit=limit,
+        )
+    if source == "auto":
+        codex_digests = harvest_codex(
+            cfg.codex_archived_sessions_dir,
+            scope=scope,
+            invoked_project=invoked_project,
+            since_iso=since_iso,
+            limit=limit,
+        )
+        if codex_digests:
+            return codex_digests
+
+    return harvest(
+        cfg.transcripts_dir,
+        scope=scope,
+        invoked_project=invoked_project,
+        since_iso=since_iso,
+        limit=limit,
+    )

package/template/skills/skillopt-sleep/scripts/python/judges.py

@@ -0,0 +1,84 @@
+"""SkillOpt-Sleep — rule-based judges (gbrain-evals compatible).
+
+Implements the programmatic check operators used by gbrain-evals'
+skillopt-v1 benchmark so we can score skill outputs locally, with NO judge
+API call:
+
+  * section_present <name>   — a markdown heading containing <name> exists
+  * regex <pattern>          — the pattern matches the response
+  * max_chars <n>            — response length <= n
+  * min_chars <n>            — response length >= n
+  * contains <text>          — substring present (case-insensitive)
+  * tool_called <name>       — a tool with <name> was invoked (needs a tool loop;
+                               in single-shot replay we approximate via an
+                               explicit "TOOL_CALL: <name>" marker the agent emits)
+
+A task whose judge is {"kind": "rule", "checks": [...]} passes (hard=1.0) iff
+ALL checks pass; soft = fraction of checks passed. This mirrors gbrain's
+all-checks-must-pass rule scoring and gives the gate a smooth signal.
+"""
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List, Tuple
+
+
+def _section_present(response: str, name: str) -> bool:
+    # a markdown heading line (#, ##, ...) or bold line that contains `name`
+    pat = re.compile(
+        r"(?im)^\s{0,3}(#{1,6}\s*.*%s|\*\*.*%s.*\*\*\s*:?)\s*$" % (re.escape(name), re.escape(name))
+    )
+    if pat.search(response or ""):
+        return True
+    # also accept "Name:" style label at line start
+    label = re.compile(r"(?im)^\s*%s\s*:" % re.escape(name))
+    return bool(label.search(response or ""))
+
+
+def _check(op: str, arg: Any, response: str, tools_called: List[str]) -> bool:
+    r = response or ""
+    if op == "section_present":
+        return _section_present(r, str(arg))
+    if op == "regex":
+        try:
+            return bool(re.search(str(arg), r))
+        except re.error:
+            return False
+    if op == "max_chars":
+        return len(r) <= int(arg)
+    if op == "min_chars":
+        return len(r) >= int(arg)
+    if op == "contains":
+        return str(arg).lower() in r.lower()
+    if op == "tool_called":
+        name = str(arg).lower()
+        if any(name == t.lower() for t in tools_called):
+            return True
+        # single-shot approximation: the agent emits an explicit marker
+        return bool(re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(name), r))
+    # unknown op: do not block
+    return True
+
+
+def score_rule_judge(
+    judge: Dict[str, Any],
+    response: str,
+    tools_called: List[str] | None = None,
+) -> Tuple[float, float, str]:
+    """Return (hard, soft, rationale) for a gbrain-style rule judge."""
+    checks = (judge or {}).get("checks", []) or []
+    if not checks:
+        return 0.0, 0.0, "no checks"
+    tools_called = tools_called or []
+    passed = 0
+    failed_desc: List[str] = []
+    for c in checks:
+        ok = _check(c.get("op", ""), c.get("arg"), response, tools_called)
+        if ok:
+            passed += 1
+        else:
+            failed_desc.append(f"{c.get('op')}={c.get('arg')}")
+    soft = passed / len(checks)
+    hard = 1.0 if passed == len(checks) else 0.0
+    rationale = "all checks passed" if hard else "failed: " + ", ".join(failed_desc)
+    return hard, soft, rationale

package/template/skills/skillopt-sleep/scripts/python/llm_miner.py

@@ -0,0 +1,134 @@
+"""SkillOpt-Sleep — LLM-backed task miner.
+
+The heuristic miner (mine.py) produces TaskRecords without a checkable
+reference, so real harvested transcripts can't show measurable lift. This
+module uses an optimizer backend to turn session digests into TaskRecords
+WITH a checkable rubric judge — the missing piece for real-data improvement.
+
+For each recurring intent it extracts:
+  * a clean, generalized `intent` (the reusable task, stripped of one-off specifics)
+  * a `rubric` (what a good answer must satisfy) -> stored as a rule judge of
+    `contains`/`regex`/`section_present` checks the local judge can score, OR a
+    free-text rubric scored by the backend's judge() when no programmatic check fits
+  * a preference signal (was the user satisfied?) to weight failures
+
+It is deliberately conservative: it only emits a task when it can name a
+concrete, checkable success criterion, so the gate has real signal. Tasks it
+can't make checkable are dropped (logged), not faked.
+"""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Callable, Dict, List
+
+from skillopt_sleep.backend import Backend, _extract_json
+from skillopt_sleep.types import SessionDigest, TaskRecord
+
+
+_MINER_PROMPT = """You are mining a user's past AI-assistant sessions to find RECURRING tasks
+worth optimizing a skill for. From the session below, extract 0-3 reusable tasks.
+
+A good task is something the user asks for repeatedly or had to correct, where a
+GENERAL rule would help next time (formatting, structure, tool-use, conventions).
+Skip one-off or purely exploratory requests.
+
+For each task return:
+  - "intent": the reusable request, generalized (no one-off specifics)
+  - "checks": a list of programmatic success checks a grader can run on a future
+     answer. Each check is one of:
+        {"op":"section_present","arg":"<heading text>"}
+        {"op":"regex","arg":"<python regex the answer must match>"}
+        {"op":"contains","arg":"<substring the answer must contain>"}
+        {"op":"max_chars","arg":<int>}
+     Only include checks you are confident a GOOD answer must satisfy.
+  - "rubric": a one-sentence description of what a good answer looks like
+  - "satisfied": true/false — did the user seem satisfied with the assistant's answer?
+
+Return ONLY a JSON array (possibly empty). No prose.
+
+# Session
+project: __PROJECT__
+user prompts:
+__PROMPTS__
+assistant final (last):
+__FINAL__
+feedback signals: __FEEDBACK__
+"""
+
+
+def _digest_to_prompt(d: SessionDigest) -> str:
+    prompts = "\n".join(f"  - {p[:240]}" for p in d.user_prompts[:6]) or "  (none)"
+    final = (d.assistant_finals[-1][:400] if d.assistant_finals else "(none)")
+    return (
+        _MINER_PROMPT
+        .replace("__PROJECT__", d.project or "(unknown)")
+        .replace("__PROMPTS__", prompts)
+        .replace("__FINAL__", final)
+        .replace("__FEEDBACK__", ", ".join(d.feedback_signals[:6]) or "(none)")
+    )
+
+
+def _mk_task(d: SessionDigest, obj: Dict[str, Any], idx: int) -> TaskRecord | None:
+    intent = str(obj.get("intent", "")).strip()
+    if len(intent) < 8:
+        return None
+    checks = obj.get("checks") or []
+    rubric = str(obj.get("rubric", "")).strip()
+    satisfied = bool(obj.get("satisfied", False))
+
+    # keep only well-formed checks
+    clean_checks = []
+    for c in checks:
+        if isinstance(c, dict) and c.get("op") in {
+            "section_present", "regex", "contains", "max_chars", "min_chars",
+        }:
+            clean_checks.append({"op": c["op"], "arg": c.get("arg")})
+
+    import hashlib
+    tid = "llm_" + hashlib.sha256((d.project + intent).encode()).hexdigest()[:12]
+
+    if clean_checks:
+        return TaskRecord(
+            id=tid, project=d.project, intent=intent,
+            reference_kind="rule", judge={"kind": "rule", "checks": clean_checks},
+            outcome="success" if satisfied else "fail",
+            tags=["mined:llm"], source_sessions=[d.session_id],
+        )
+    if rubric:
+        return TaskRecord(
+            id=tid, project=d.project, intent=intent,
+            reference_kind="rubric", reference=rubric,
+            outcome="success" if satisfied else "fail",
+            tags=["mined:llm"], source_sessions=[d.session_id],
+        )
+    return None  # not checkable -> drop
+
+
+def make_llm_miner(
+    backend: Backend,
+    *,
+    max_sessions: int = 20,
+    max_tasks: int = 40,
+) -> Callable[[List[SessionDigest]], List[TaskRecord]]:
+    """Return an llm_miner(digests) -> list[TaskRecord] bound to a backend."""
+
+    def _miner(digests: List[SessionDigest]) -> List[TaskRecord]:
+        out: List[TaskRecord] = []
+        for d in digests[:max_sessions]:
+            if not d.user_prompts:
+                continue
+            raw = backend._call(_digest_to_prompt(d), max_tokens=800)  # type: ignore[attr-defined]
+            arr = _extract_json(raw, "array")
+            if not isinstance(arr, list):
+                continue
+            for i, obj in enumerate(arr[:3]):
+                if isinstance(obj, dict):
+                    t = _mk_task(d, obj, i)
+                    if t is not None:
+                        out.append(t)
+                if len(out) >= max_tasks:
+                    return out
+        return out
+
+    return _miner

package/template/skills/skillopt-sleep/scripts/python/memory.py

@@ -0,0 +1,129 @@
+"""SkillOpt-Sleep — skill/memory document manipulation.
+
+Applies bounded EditRecords to a skill (SKILL.md body) or memory (CLAUDE.md)
+document, and provides Dream-style consolidation helpers (dedup near-identical
+lines, drop contradictions). All edits live inside a protected, clearly-marked
+region so the sleep cycle never clobbers the user's hand-written content.
+"""
+from __future__ import annotations
+
+import re
+from typing import List, Tuple
+
+from skillopt_sleep.types import EditRecord
+
+LEARNED_START = "<!-- SKILLOPT-SLEEP:LEARNED START -->"
+LEARNED_END = "<!-- SKILLOPT-SLEEP:LEARNED END -->"
+_BANNER = (
+    "_This block is maintained by SkillOpt-Sleep. Edits here are proposed "
+    "offline, validated against your past tasks, and adopted only after you "
+    "approve them. Hand-edits outside this block are never touched._"
+)
+
+
+def extract_learned(doc: str) -> str:
+    s = doc.find(LEARNED_START)
+    e = doc.find(LEARNED_END)
+    if s == -1 or e == -1:
+        return ""
+    return doc[s + len(LEARNED_START):e].strip()
+
+
+def _strip_learned(doc: str) -> str:
+    while True:
+        s = doc.find(LEARNED_START)
+        if s == -1:
+            break
+        e = doc.find(LEARNED_END, s)
+        if e == -1:
+            doc = doc[:s]
+            break
+        doc = doc[:s] + doc[e + len(LEARNED_END):]
+    while "\n\n\n" in doc:
+        doc = doc.replace("\n\n\n", "\n\n")
+    return doc.rstrip()
+
+
+def set_learned(doc: str, learned_lines: List[str]) -> str:
+    """Replace the protected learned region with the given bullet lines."""
+    base = _strip_learned(doc)
+    body = "\n".join(f"- {ln.strip().lstrip('- ').strip()}" for ln in learned_lines if ln.strip())
+    block = (
+        f"\n\n{LEARNED_START}\n"
+        f"## Learned preferences & procedures\n\n{_BANNER}\n\n{body}\n"
+        f"{LEARNED_END}\n"
+    )
+    return (base + block).lstrip("\n")
+
+
+def current_learned_lines(doc: str) -> List[str]:
+    inner = extract_learned(doc)
+    lines: List[str] = []
+    for ln in inner.splitlines():
+        ln = ln.strip()
+        if ln.startswith("- "):
+            lines.append(ln[2:].strip())
+    return lines
+
+
+def _norm(s: str) -> str:
+    return re.sub(r"\s+", " ", (s or "").lower()).strip()
+
+
+def apply_edits(doc: str, edits: List[EditRecord]) -> Tuple[str, List[EditRecord]]:
+    """Apply add/delete/replace edits to the protected learned region.
+
+    Returns (new_doc, applied_edits). Dedups: an `add` whose content already
+    exists (normalized) is skipped. `delete`/`replace` match on normalized
+    anchor substring.
+    """
+    lines = current_learned_lines(doc)
+    norm_set = {_norm(line) for line in lines}
+    applied: List[EditRecord] = []
+
+    for e in edits:
+        op = (e.op or "add").lower()
+        if op == "add":
+            if _norm(e.content) in norm_set or not e.content.strip():
+                continue
+            lines.append(e.content.strip())
+            norm_set.add(_norm(e.content))
+            applied.append(e)
+        elif op == "delete":
+            anchor = _norm(e.anchor or e.content)
+            keep = [line for line in lines if anchor not in _norm(line)]
+            if len(keep) != len(lines):
+                lines = keep
+                norm_set = {_norm(line) for line in lines}
+                applied.append(e)
+        elif op == "replace":
+            anchor = _norm(e.anchor)
+            new_lines = []
+            changed = False
+            for line in lines:
+                if anchor and anchor in _norm(line):
+                    new_lines.append(e.content.strip())
+                    changed = True
+                else:
+                    new_lines.append(line)
+            if changed:
+                lines = new_lines
+                norm_set = {_norm(line) for line in lines}
+                applied.append(e)
+
+    return set_learned(doc, lines), applied
+
+
+def ensure_skill_scaffold(doc: str, *, name: str, description: str) -> str:
+    """Ensure a SKILL.md has YAML frontmatter so local agents load it."""
+    if doc.lstrip().startswith("---"):
+        return doc
+    fm = (
+        "---\n"
+        f"name: {name}\n"
+        f"description: {description}\n"
+        "---\n\n"
+        f"# {name}\n\n"
+        "Preferences and procedures learned from your past local agent sessions.\n"
+    )
+    return fm + doc