self-evolve-framework 1.4.0 → 1.6.0

package/template/skills/skillopt-sleep/scripts/framework/skillopt/optimizer/appendix.py

@@ -0,0 +1,156 @@
+"""Skill-Aware Reflection — protected appendix field (EmbodiSkill S_app).
+
+EmbodiSkill (paper 2605.10332v1) splits a skill into ``S = (S_body, S_app)``:
+the body holds the main prescriptive rules; the appendix only *emphasizes*
+existing valid rules that the executor failed to follow (EXECUTION_LAPSE), and
+**never introduces new rules**.
+
+This module owns the appendix region of the skill document. It mirrors the
+protected-field pattern of :mod:`skillopt.optimizer.slow_update`, with two
+differences:
+
+1. **Append semantics** (not replace): execution-lapse reminders accumulate
+   across steps within a run, so new notes are merged into the existing
+   appendix rather than overwriting it.
+2. **Lightweight dedup**: near-duplicate reminders are collapsed (inspired by
+   GMemory's ``_dedupe_preserve_order``) so the appendix stays compact.
+
+The appendix lives **inside** the skill markdown, between dedicated markers, so
+it is persisted by the normal ``_save_skill`` path and is resume-safe. Step-level
+analyst edits cannot modify it (enforced by the shared protected-region check in
+:mod:`skillopt.optimizer.skill`).
+
+Public API
+----------
+- :func:`has_appendix_field`        — check if markers are present
+- :func:`inject_empty_appendix_field` — add empty placeholder (skill init)
+- :func:`extract_appendix_notes`    — read current notes as a list
+- :func:`append_to_appendix_field`  — merge new notes (dedup) into the region
+"""
+from __future__ import annotations
+
+import re
+
+# ── Protected field markers ─────────────────────────────────────────────────
+
+APPENDIX_START = "<!-- APPENDIX_START -->"
+APPENDIX_END = "<!-- APPENDIX_END -->"
+
+# Heading shown inside the rendered appendix block (human-readable only).
+APPENDIX_HEADING = "## Execution Notes Appendix"
+
+# Each note is rendered as a markdown bullet so the target model reads it as
+# ordinary guidance.
+_NOTE_BULLET_PREFIX = "- "
+
+
+# ── Dedup helpers ───────────────────────────────────────────────────────────
+
+
+def _canonicalize(text: str) -> str:
+    """Normalize a note for duplicate detection (whitespace/punct/case-insensitive)."""
+    normalized = re.sub(r"\s+", " ", str(text or "").strip())
+    normalized = normalized.rstrip(" .;:,_-")
+    return normalized.casefold()
+
+
+def _dedupe_preserve_order(notes: list[str]) -> list[str]:
+    """Drop blanks and near-duplicates, preserving first-seen order."""
+    seen: set[str] = set()
+    deduped: list[str] = []
+    for note in notes:
+        text = re.sub(r"\s+", " ", str(note).strip())
+        if not text:
+            continue
+        key = _canonicalize(text)
+        if not key or key in seen:
+            continue
+        seen.add(key)
+        deduped.append(text)
+    return deduped
+
+
+# ── Field manipulation ──────────────────────────────────────────────────────
+
+
+def has_appendix_field(skill: str) -> bool:
+    return APPENDIX_START in skill and APPENDIX_END in skill
+
+
+def _render_block(notes: list[str]) -> str:
+    """Render the full marker-delimited appendix block for *notes*."""
+    lines = [APPENDIX_START, APPENDIX_HEADING]
+    for note in notes:
+        lines.append(f"{_NOTE_BULLET_PREFIX}{note}")
+    lines.append(APPENDIX_END)
+    return "\n".join(lines)
+
+
+def inject_empty_appendix_field(skill: str) -> str:
+    """Add an empty appendix placeholder at the end of *skill* (idempotent).
+
+    Mirrors ``inject_empty_slow_update_field``: called once at skill init so the
+    protected region exists before any note is written.
+    """
+    if has_appendix_field(skill):
+        return skill
+    block = f"\n\n{APPENDIX_START}\n{APPENDIX_HEADING}\n{APPENDIX_END}\n"
+    return skill.rstrip() + block
+
+
+def extract_appendix_notes(skill: str) -> list[str]:
+    """Return the current appendix notes as a list of strings (no markers/heading)."""
+    start = skill.find(APPENDIX_START)
+    end = skill.find(APPENDIX_END)
+    if start == -1 or end == -1:
+        return []
+    inner = skill[start + len(APPENDIX_START):end].strip()
+    notes: list[str] = []
+    for raw_line in inner.splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        if line == APPENDIX_HEADING or line.lstrip("#").strip() == APPENDIX_HEADING.lstrip("#").strip():
+            continue
+        if line.startswith(_NOTE_BULLET_PREFIX):
+            line = line[len(_NOTE_BULLET_PREFIX):].strip()
+        elif line.startswith("-") or line.startswith("*"):
+            line = line[1:].strip()
+        if line:
+            notes.append(line)
+    return notes
+
+
+def _strip_all_appendix_fields(skill: str) -> str:
+    """Remove every appendix marker pair (and content between) from *skill*."""
+    while True:
+        start = skill.find(APPENDIX_START)
+        if start == -1:
+            break
+        end = skill.find(APPENDIX_END, start)
+        if end == -1:
+            skill = skill[:start] + skill[start + len(APPENDIX_START):]
+            break
+        skill = skill[:end + len(APPENDIX_END)].rsplit(APPENDIX_START, 1)[0] + skill[end + len(APPENDIX_END):]
+    skill = skill.replace(APPENDIX_END, "")
+    while "\n\n\n" in skill:
+        skill = skill.replace("\n\n\n", "\n\n")
+    return skill.rstrip()
+
+
+def append_to_appendix_field(skill: str, new_notes: list[str]) -> str:
+    """Merge *new_notes* into the appendix region (dedup), returning updated skill.
+
+    - If no appendix region exists yet, one is created.
+    - Existing notes are preserved; new ones are appended after dedup against the
+      combined set, so order is stable and duplicates are dropped.
+    - Empty / whitespace-only notes are ignored. If the merged set is empty, an
+      empty placeholder region is still ensured.
+    """
+    incoming = _dedupe_preserve_order(list(new_notes or []))
+    existing = extract_appendix_notes(skill)
+    merged = _dedupe_preserve_order(existing + incoming)
+
+    base = _strip_all_appendix_fields(skill)
+    block = _render_block(merged)
+    return f"{base}\n\n{block}\n"

package/template/skills/skillopt-sleep/scripts/framework/skillopt/optimizer/clip.py

@@ -0,0 +1,109 @@
+"""ReflACT gradient clipping — LLM-driven edit ranking and selection.
+
+Analogous to gradient clipping in neural network training: ranks candidate
+edits by importance and selects the top-L to apply, controlling the
+effective step size. Previously core/select.py.
+"""
+from __future__ import annotations
+
+from skillopt.model import chat_optimizer
+from skillopt.optimizer.meta_skill import format_meta_skill_context
+from skillopt.optimizer.update_modes import (
+    describe_item,
+    get_payload_items,
+    is_rewrite_mode,
+    normalize_update_mode,
+    payload_key,
+    payload_label,
+)
+from skillopt.prompts import load_prompt
+from skillopt.utils import extract_json
+
+
+# ── Public API ────────────────────────────────────────────────────────────────
+
+def rank_and_select(
+    skill_content: str,
+    patch: dict,
+    max_edits: int,
+    meta_skill_context: str = "",
+    update_mode: str = "patch",
+) -> dict:
+    """Use a optimizer LLM to rank edits by importance, then keep top-L.
+
+    If the edit pool is within budget, returns the patch unchanged.
+    Otherwise, calls the optimizer to rank and select the most impactful edits.
+
+    Parameters
+    ----------
+    skill_content : str
+        Current skill document.
+    patch : dict
+        Merged :class:`~skillopt.types.Patch` dict with ``edits`` list.
+    max_edits : int
+        Maximum number of edits to keep (the "edit budget").
+
+    Returns
+    -------
+    dict
+        :class:`~skillopt.types.Patch` dict with selected edits and
+        optional ``ranking_details``.
+    """
+    update_mode = normalize_update_mode(update_mode)
+    edits = get_payload_items(patch, update_mode)
+    if len(edits) <= max_edits:
+        return patch
+
+    # Build the edit pool description for the optimizer
+    edits_desc = []
+    for i, edit in enumerate(edits):
+        edits_desc.append(f"[{i}] {describe_item(edit, update_mode)}")
+
+    user = (
+        f"## Current Skill\n{skill_content}\n\n"
+        f"## {payload_label(update_mode, title=True)} Pool ({len(edits)} {payload_label(update_mode)}, budget={max_edits})\n"
+        + "\n".join(edits_desc)
+        + f"\n\nSelect the {max_edits} most important {payload_label(update_mode)}. "
+        f"Return their 0-based indices in priority order."
+    )
+    optimizer_ctx = format_meta_skill_context(meta_skill_context)
+    if optimizer_ctx:
+        user = f"{optimizer_ctx}\n\n{user}"
+    prompt_name = "ranking_rewrite" if is_rewrite_mode(update_mode) else "ranking"
+
+    try:
+        response, _ = chat_optimizer(
+            system=load_prompt(prompt_name), user=user,
+            max_completion_tokens=16384, retries=3, stage="ranking",
+        )
+        result = extract_json(response)
+        if result and "selected_indices" in result:
+            indices = result["selected_indices"]
+            selected = []
+            seen: set[int] = set()
+            for idx in indices:
+                if (
+                    isinstance(idx, int)
+                    and 0 <= idx < len(edits)
+                    and idx not in seen
+                ):
+                    selected.append(edits[idx])
+                    seen.add(idx)
+                if len(selected) >= max_edits:
+                    break
+            if selected:
+                return {
+                    "reasoning": patch.get("reasoning", "")
+                    + f" [optimizer-ranked: selected {len(selected)}/{len(edits)} {payload_label(update_mode)}]",
+                    payload_key(update_mode): selected,
+                    "ranking_details": result,
+                }
+    except Exception:  # noqa: BLE001
+        pass
+
+    # Fallback: simple truncation
+    return {
+        "reasoning": patch.get("reasoning", "")
+        + f" [fallback truncated {len(edits)}->{max_edits} {payload_label(update_mode)}]",
+        payload_key(update_mode): edits[:max_edits],
+    }

package/template/skills/skillopt-sleep/scripts/framework/skillopt/optimizer/lr_autonomous.py

@@ -0,0 +1,108 @@
+"""Optimizer-driven autonomous update-size decisions."""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from skillopt.model import chat_optimizer
+from skillopt.optimizer.meta_skill import format_meta_skill_context
+from skillopt.optimizer.update_modes import describe_item, get_payload_items, payload_label
+from skillopt.prompts import load_prompt
+from skillopt.utils import extract_json
+
+
+def _coerce_nonnegative_int(value: Any) -> int | None:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return max(0, value)
+    if isinstance(value, float) and value.is_integer():
+        return max(0, int(value))
+    text = str(value or "").strip()
+    if not text:
+        return None
+    match = re.search(r"-?\d+", text)
+    if not match:
+        return None
+    return max(0, int(match.group(0)))
+
+
+def decide_autonomous_learning_rate(
+    *,
+    skill_content: str,
+    merged_patch: dict,
+    update_mode: str,
+    rollout_hard: float,
+    rollout_soft: float,
+    rollout_n: int,
+    step_buffer_context: str = "",
+    meta_skill_context: str = "",
+) -> dict:
+    """Ask the optimizer to choose the number of update items for this step.
+
+    The prompt intentionally avoids default budgets, candidate budget lists, or
+    scheduler history. The only hard post-processing is validity: the returned
+    integer is clamped to the available item count.
+    """
+    items = get_payload_items(merged_patch, update_mode)
+    available = len(items)
+    item_lines = [
+        f"[{idx}] {describe_item(item, update_mode)}"
+        for idx, item in enumerate(items)
+    ]
+    user = (
+        f"## Current Skill\n{skill_content}\n\n"
+        f"## Current Step Evidence\n"
+        f"rollout_n={rollout_n}\n"
+        f"rollout_hard={rollout_hard:.6f}\n"
+        f"rollout_soft={rollout_soft:.6f}\n"
+        f"proposed_update_items={available}\n"
+        f"update_item_type={payload_label(update_mode)}\n\n"
+        f"## Proposed Update Items\n"
+        + "\n".join(item_lines)
+        + "\n\nDecide how many proposed update items should be applied now."
+    )
+    if step_buffer_context.strip():
+        user += f"\n\n## Previous Steps in This Epoch\n{step_buffer_context}"
+    optimizer_ctx = format_meta_skill_context(meta_skill_context)
+    if optimizer_ctx:
+        user = f"{optimizer_ctx}\n\n{user}"
+
+    response = ""
+    parsed: dict | None = None
+    decision: int | None = None
+    try:
+        response, _ = chat_optimizer(
+            system=load_prompt("lr_autonomous"),
+            user=user,
+            max_completion_tokens=16384,
+            retries=3,
+            stage="lr_autonomous",
+        )
+        parsed = extract_json(response)
+        if parsed:
+            decision = _coerce_nonnegative_int(parsed.get("learning_rate"))
+    except Exception as exc:  # noqa: BLE001
+        parsed = {"error": str(exc)}
+
+    fallback = False
+    if decision is None:
+        decision = 0
+        fallback = True
+
+    chosen = min(decision, available)
+    record = {
+        "learning_rate": chosen,
+        "raw_learning_rate": decision,
+        "available_update_items": available,
+        "clamped": chosen != decision,
+        "fallback": fallback,
+        "reasoning": (parsed or {}).get("reasoning", ""),
+        "confidence": (parsed or {}).get("confidence", ""),
+        "risk_notes": (parsed or {}).get("risk_notes", []),
+        "raw_response": response,
+    }
+    if parsed and "error" in parsed:
+        record["error"] = parsed["error"]
+    return record

package/template/skills/skillopt-sleep/scripts/framework/skillopt/optimizer/meta_skill.py

@@ -0,0 +1,79 @@
+"""Optimizer-side meta skill memory for cross-epoch optimization guidance.
+
+This module maintains a compact optimizer-facing memory distilled from
+adjacent-epoch skill comparisons. Unlike ``slow_update``, it does not
+modify the target skill document. Instead, it produces guidance meant to
+improve future optimizer behavior when proposing, merging, and ranking edits.
+"""
+from __future__ import annotations
+
+import traceback
+
+from skillopt.model import chat_optimizer
+from skillopt.optimizer.slow_update import format_comparison_text
+from skillopt.prompts import load_prompt
+from skillopt.utils import extract_json
+
+
+def format_meta_skill_context(meta_skill_content: str) -> str:
+    """Render optimizer memory into a prompt-ready context block."""
+    content = (meta_skill_content or "").strip()
+    if not content:
+        return ""
+    return (
+        "## Optimizer Meta Skill\n"
+        "This is optimizer-side memory distilled from prior epoch transitions in "
+        "this environment. Use it to improve how you propose, merge, and rank "
+        "skill edits. Prefer it when the current evidence is ambiguous, but do "
+        "not force it if the current trajectories clearly contradict it.\n\n"
+        f"{content}"
+    )
+
+
+def run_meta_skill(
+    prev_skill: str,
+    curr_skill: str,
+    comparison_pairs: list[dict],
+    *,
+    prev_meta_skill_content: str = "",
+    system_prompt: str | None = None,
+) -> dict | None:
+    """Produce updated optimizer-side meta skill from adjacent epochs."""
+    actual_system = system_prompt if system_prompt is not None else load_prompt("meta_skill")
+
+    prev_meta_section = (
+        prev_meta_skill_content.strip()
+        if prev_meta_skill_content and prev_meta_skill_content.strip()
+        else "(No previous optimizer meta skill — this is the first update.)"
+    )
+
+    comparison_text = format_comparison_text(comparison_pairs)
+    user = (
+        f"## Previous Epoch Last-Step Skill\n{prev_skill}\n\n"
+        f"## Current Epoch Last-Step Skill\n{curr_skill}\n\n"
+        f"## Previous Optimizer Meta Skill\n"
+        f"The following optimizer memory was available during the current epoch. "
+        f"Reflect on whether it improved or harmed the quality of edits.\n\n"
+        f"{prev_meta_section}\n\n"
+        f"## Longitudinal Comparison (same tasks, two last-step skills)\n"
+        f"{comparison_text}"
+    )
+
+    try:
+        response, _ = chat_optimizer(
+            system=actual_system,
+            user=user,
+            max_completion_tokens=16384,
+            retries=3,
+            stage="meta_skill",
+        )
+        result = extract_json(response)
+        if result and result.get("meta_skill_content"):
+            return {
+                "reasoning": str(result.get("reasoning", "")).strip(),
+                "meta_skill_content": str(result["meta_skill_content"]).strip(),
+            }
+    except Exception:  # noqa: BLE001
+        traceback.print_exc()
+
+    return None

package/template/skills/skillopt-sleep/scripts/framework/skillopt/optimizer/rewrite.py

@@ -0,0 +1,59 @@
+"""Optimizer-driven full skill rewrite from selected revise_suggestions."""
+from __future__ import annotations
+
+import json
+
+from skillopt.model import chat_optimizer
+from skillopt.prompts import load_prompt
+from skillopt.optimizer.update_modes import get_payload_items
+from skillopt.utils import extract_json
+
+
+def rewrite_skill_from_suggestions(
+    skill_content: str,
+    patch: dict,
+    *,
+    system_prompt: str | None = None,
+    step_buffer_context: str = "",
+    env: str | None = None,
+    reasoning_effort: str | None = "high",
+    max_completion_tokens: int = 64000,
+) -> dict | None:
+    suggestions = get_payload_items(patch, "rewrite_from_suggestions")
+    if not suggestions:
+        return None
+
+    user = (
+        f"## Current Skill\n{skill_content}\n\n"
+        f"## Selected Revise Suggestions ({len(suggestions)} total)\n"
+        f"{json.dumps(suggestions, ensure_ascii=False, indent=2)}\n\n"
+    )
+    if step_buffer_context.strip():
+        user += f"## Previous Steps in This Epoch\n{step_buffer_context}\n\n"
+    user += (
+        "Rewrite the full skill document so it integrates the selected suggestions. "
+        "Return the complete new skill in `new_skill`."
+    )
+
+    actual_system = system_prompt if system_prompt is not None else load_prompt(
+        "rewrite_skill", env=env,
+    )
+
+    try:
+        response, _ = chat_optimizer(
+            system=actual_system,
+            user=user,
+            max_completion_tokens=max_completion_tokens,
+            retries=3,
+            stage="rewrite",
+            reasoning_effort=reasoning_effort,
+        )
+        result = extract_json(response)
+        if result and str(result.get("new_skill", "")).strip():
+            result["new_skill"] = str(result["new_skill"]).rstrip() + "\n"
+            if "change_summary" not in result or not isinstance(result["change_summary"], list):
+                result["change_summary"] = []
+            return result
+    except Exception:  # noqa: BLE001
+        return None
+    return None

package/template/skills/skillopt-sleep/scripts/framework/skillopt/optimizer/scheduler.py

@@ -0,0 +1,127 @@
+"""Learning-rate (edit budget) schedulers for ReflACT.
+
+The "learning rate" in ReflACT is the maximum number of skill edits allowed
+per optimization step.  A scheduler controls how this budget changes over
+the course of training.
+
+Supported modes
+---------------
+- ``constant``   : Fixed budget throughout training.
+- ``linear``     : Linear decay from ``max_lr`` to ``min_lr``.
+- ``cosine``     : Cosine annealing from ``max_lr`` to ``min_lr``.
+- ``autonomous`` : No limit — the model decides how many edits to make.
+
+Usage::
+
+    scheduler = build_scheduler(cfg)
+    for step in range(1, total_steps + 1):
+        lr = scheduler.step()       # returns edit budget for this step
+        # ... use lr as max_edits ...
+"""
+from __future__ import annotations
+
+import math
+from abc import ABC, abstractmethod
+
+
+class LRScheduler(ABC):
+    """Base class for edit-budget schedulers."""
+
+    def __init__(self, max_lr: int, min_lr: int, total_steps: int) -> None:
+        self.max_lr = max_lr
+        self.min_lr = min_lr
+        self.total_steps = total_steps
+        self._current_step = 0
+
+    @abstractmethod
+    def _compute_lr(self, step: int) -> int:
+        """Return the edit budget for the given 1-indexed step."""
+
+    def step(self) -> int:
+        """Advance one step and return the edit budget."""
+        self._current_step += 1
+        return self._compute_lr(self._current_step)
+
+    def get_lr(self, step: int) -> int:
+        """Return the edit budget for an arbitrary step (1-indexed)."""
+        return self._compute_lr(step)
+
+    def state_dict(self) -> dict:
+        return {"current_step": self._current_step}
+
+    def load_state_dict(self, state: dict) -> None:
+        self._current_step = state.get("current_step", 0)
+
+
+class ConstantScheduler(LRScheduler):
+    """Fixed edit budget throughout training."""
+
+    def _compute_lr(self, step: int) -> int:
+        return self.max_lr
+
+
+class LinearScheduler(LRScheduler):
+    """Linear decay from ``max_lr`` to ``min_lr`` over ``total_steps``."""
+
+    def _compute_lr(self, step: int) -> int:
+        if self.total_steps <= 1:
+            return self.max_lr
+        t = min(step, self.total_steps) / self.total_steps
+        lr = self.max_lr + (self.min_lr - self.max_lr) * t
+        return max(self.min_lr, round(lr))
+
+
+class CosineScheduler(LRScheduler):
+    """Cosine annealing from ``max_lr`` to ``min_lr`` over ``total_steps``."""
+
+    def _compute_lr(self, step: int) -> int:
+        if self.total_steps <= 1:
+            return self.max_lr
+        t = min(step, self.total_steps) / self.total_steps
+        lr = self.min_lr + 0.5 * (self.max_lr - self.min_lr) * (1 + math.cos(math.pi * t))
+        return max(self.min_lr, round(lr))
+
+
+class AutonomousScheduler(LRScheduler):
+    """No edit limit — the model decides freely."""
+
+    NO_LIMIT = 999
+
+    def _compute_lr(self, step: int) -> int:
+        return self.NO_LIMIT
+
+
+# ── Factory ──────────────────────────────────────────────────────────────
+
+_REGISTRY: dict[str, type[LRScheduler]] = {
+    "constant": ConstantScheduler,
+    "linear": LinearScheduler,
+    "cosine": CosineScheduler,
+    "autonomous": AutonomousScheduler,
+}
+
+
+def build_scheduler(
+    mode: str = "constant",
+    max_lr: int = 8,
+    min_lr: int = 2,
+    total_steps: int = 8,
+) -> LRScheduler:
+    """Build a scheduler from config parameters.
+
+    Parameters
+    ----------
+    mode : str
+        One of ``constant``, ``linear``, ``cosine``, ``autonomous``.
+    max_lr : int
+        Initial / maximum edit budget.
+    min_lr : int
+        Minimum edit budget (for decay modes).
+    total_steps : int
+        Total number of optimization steps in training.
+    """
+    if mode not in _REGISTRY:
+        raise ValueError(
+            f"Unknown scheduler mode '{mode}'. Available: {list(_REGISTRY.keys())}"
+        )
+    return _REGISTRY[mode](max_lr=max_lr, min_lr=min_lr, total_steps=total_steps)

package/template/skills/skillopt-sleep/scripts/framework/skillopt/optimizer/select.py

@@ -0,0 +1,4 @@
+"""Backward-compat stub — moved to skillopt.optimizer.clip."""
+from skillopt.optimizer.clip import rank_and_select  # noqa: F401
+
+__all__ = ["rank_and_select"]