self-evolve-framework 1.5.0 → 1.6.0

package/template/skills/skillopt-sleep/scripts/framework/skillopt/types.py

@@ -0,0 +1,306 @@
+"""Standardized I/O types for the ReflACT pipeline.
+
+Shared dataclass definitions for the 6-stage per-step pipeline
+and the 2 epoch-level stages.  All types support round-trip
+conversion to/from plain dicts for incremental adoption.
+
+Re-exports
+----------
+GateResult, GateAction — from skillopt.evaluation.gate
+BatchSpec              — from skillopt.datasets.base
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field, fields as dc_fields
+from typing import Any, Literal
+
+from skillopt.evaluation.gate import GateAction, GateResult  # noqa: F401
+from skillopt.datasets.base import BatchSpec  # noqa: F401
+
+
+# ── Atomic types ─────────────────────────────────────────────────────────
+
+EditOp = Literal["append", "insert_after", "replace", "delete"]
+
+
+@dataclass
+class Edit:
+    """A single edit operation on a skill document.
+
+    Used across Reflect → Aggregate → Select → Update → MetaReflect.
+    """
+
+    op: EditOp
+    content: str = ""
+    target: str = ""
+    support_count: int | None = None
+    source_type: Literal["failure", "success"] | None = None
+    merge_level: int | None = None
+    update_origin: str = ""
+    update_target: str = ""
+
+    @classmethod
+    def from_dict(cls, d: dict) -> Edit:
+        return cls(
+            op=d.get("op", "append"),
+            content=d.get("content", ""),
+            target=d.get("target", ""),
+            support_count=d.get("support_count"),
+            source_type=d.get("source_type"),
+            merge_level=d.get("merge_level"),
+            update_origin=d.get("update_origin", ""),
+            update_target=d.get("update_target", ""),
+        )
+
+    def to_dict(self) -> dict:
+        d: dict[str, Any] = {"op": self.op, "content": self.content}
+        if self.target:
+            d["target"] = self.target
+        if self.support_count is not None:
+            d["support_count"] = self.support_count
+        if self.source_type is not None:
+            d["source_type"] = self.source_type
+        if self.merge_level is not None:
+            d["merge_level"] = self.merge_level
+        if self.update_origin:
+            d["update_origin"] = self.update_origin
+        if self.update_target:
+            d["update_target"] = self.update_target
+        return d
+
+
+@dataclass
+class Patch:
+    """A set of edits with reasoning.
+
+    Output of Aggregate (③), Select (④); input to Update (⑤).
+    """
+
+    edits: list[Edit] = field(default_factory=list)
+    reasoning: str = ""
+    ranking_details: dict[str, Any] | None = None
+
+    @classmethod
+    def from_dict(cls, d: dict) -> Patch:
+        edits_raw = d.get("edits", [])
+        return cls(
+            edits=[Edit.from_dict(e) if isinstance(e, dict) else e for e in edits_raw],
+            reasoning=d.get("reasoning", ""),
+            ranking_details=d.get("ranking_details"),
+        )
+
+    def to_dict(self) -> dict:
+        d: dict[str, Any] = {
+            "reasoning": self.reasoning,
+            "edits": [e.to_dict() if isinstance(e, Edit) else e for e in self.edits],
+        }
+        if self.ranking_details is not None:
+            d["ranking_details"] = self.ranking_details
+        return d
+
+
+# ── Stage ① ROLLOUT ──────────────────────────────────────────────────────
+
+@dataclass
+class RolloutResult:
+    """Result of a single episode/task rollout.
+
+    Universal fields are required; env-specific fields live in ``extras``.
+    """
+
+    id: str
+    hard: int
+    soft: float
+    n_turns: int = 0
+    fail_reason: str = ""
+    task_type: str = ""
+    task_description: str = ""
+    predicted_answer: str = ""
+    question: str = ""
+    reference_text: str = ""
+    target_system_prompt: str = ""
+    target_user_prompt: str = ""
+    spreadsheet_preview: str = ""
+    extras: dict[str, Any] = field(default_factory=dict)
+
+    _KNOWN_FIELDS: frozenset[str] | None = field(
+        default=None, init=False, repr=False, compare=False,  # type: ignore[assignment]
+    )
+
+    @classmethod
+    def _get_known_fields(cls) -> frozenset[str]:
+        if cls._KNOWN_FIELDS is None:
+            cls._KNOWN_FIELDS = frozenset(
+                f.name for f in dc_fields(cls)
+                if f.name != "_KNOWN_FIELDS"
+            )
+        return cls._KNOWN_FIELDS
+
+    @classmethod
+    def from_dict(cls, d: dict) -> RolloutResult:
+        known = cls._get_known_fields()
+        extras = {k: v for k, v in d.items() if k not in known}
+        return cls(
+            id=str(d.get("id", "")),
+            hard=int(d.get("hard", 0)),
+            soft=float(d.get("soft", 0.0)),
+            n_turns=int(d.get("n_turns", 0)),
+            fail_reason=str(d.get("fail_reason", "")),
+            task_type=str(d.get("task_type", "")),
+            task_description=str(d.get("task_description", "")),
+            predicted_answer=str(d.get("predicted_answer", "")),
+            question=str(d.get("question", "")),
+            reference_text=str(d.get("reference_text", "")),
+            target_system_prompt=str(d.get("target_system_prompt", "")),
+            target_user_prompt=str(d.get("target_user_prompt", "")),
+            spreadsheet_preview=str(d.get("spreadsheet_preview", "")),
+            extras=extras,
+        )
+
+    def to_dict(self) -> dict:
+        d: dict[str, Any] = {
+            "id": self.id,
+            "hard": self.hard,
+            "soft": self.soft,
+        }
+        for attr in (
+            "n_turns", "fail_reason", "task_type", "task_description",
+            "predicted_answer", "question", "reference_text",
+            "target_system_prompt", "target_user_prompt",
+            "spreadsheet_preview",
+        ):
+            val = getattr(self, attr)
+            if val:
+                d[attr] = val
+        d.update(self.extras)
+        return d
+
+
+# ── Stage ② REFLECT ──────────────────────────────────────────────────────
+
+@dataclass
+class FailureSummaryEntry:
+    """One entry in the failure summary produced by error analysts."""
+
+    failure_type: str
+    count: int = 0
+    description: str = ""
+
+    @classmethod
+    def from_dict(cls, d: dict) -> FailureSummaryEntry:
+        return cls(
+            failure_type=d.get("failure_type", ""),
+            count=int(d.get("count", 0)),
+            description=d.get("description", ""),
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "failure_type": self.failure_type,
+            "count": self.count,
+            "description": self.description,
+        }
+
+
+@dataclass
+class RawPatch:
+    """Analyst output from the Reflect stage — a patch with provenance.
+
+    Wraps the dict produced by ``run_error_analyst_minibatch``
+    and ``run_success_analyst_minibatch``.
+    """
+
+    patch: Patch
+    source_type: Literal["failure", "success"] = "failure"
+    batch_size: int = 0
+    failure_summary: list[FailureSummaryEntry] = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, d: dict | None) -> RawPatch | None:
+        if d is None:
+            return None
+        inner = d.get("patch", d)
+        if not isinstance(inner, dict):
+            return None
+        patch = Patch.from_dict(inner)
+        return cls(
+            patch=patch,
+            source_type=d.get("source_type", "failure"),
+            batch_size=int(d.get("batch_size", 0)),
+            failure_summary=[
+                FailureSummaryEntry.from_dict(fs)
+                for fs in d.get("failure_summary", [])
+            ],
+        )
+
+    def to_dict(self) -> dict:
+        d: dict[str, Any] = {
+            "patch": self.patch.to_dict(),
+            "source_type": self.source_type,
+            "batch_size": self.batch_size,
+        }
+        if self.failure_summary:
+            d["failure_summary"] = [fs.to_dict() for fs in self.failure_summary]
+        return d
+
+
+# ── Epoch-level: SLOW_UPDATE ─────────────────────────────────────────────
+
+@dataclass
+class SlowUpdateResult:
+    """Output of the epoch-level slow update stage (EMA / regularization)."""
+
+    reasoning: str = ""
+    slow_update_content: str = ""
+    action: str = ""
+    time_s: float | None = None
+    prev_hard: float | None = None
+    curr_hard: float | None = None
+    selection_hard: float | None = None
+    selection_soft: float | None = None
+    candidate_hash: str = ""
+    update_origin: str = ""
+    update_target: str = ""
+
+    @classmethod
+    def from_dict(cls, d: dict | None) -> SlowUpdateResult | None:
+        if d is None:
+            return None
+        return cls(
+            reasoning=d.get("reasoning", ""),
+            slow_update_content=d.get("slow_update_content", ""),
+            action=d.get("action", ""),
+            time_s=d.get("time_s"),
+            prev_hard=d.get("prev_hard"),
+            curr_hard=d.get("curr_hard"),
+            selection_hard=d.get("selection_hard"),
+            selection_soft=d.get("selection_soft"),
+            candidate_hash=d.get("candidate_hash", ""),
+            update_origin=d.get("update_origin", ""),
+            update_target=d.get("update_target", ""),
+        )
+
+    def to_dict(self) -> dict:
+        d: dict[str, Any] = {
+            "reasoning": self.reasoning,
+            "slow_update_content": self.slow_update_content,
+        }
+        if self.action:
+            d["action"] = self.action
+        if self.time_s is not None:
+            d["time_s"] = self.time_s
+        if self.prev_hard is not None:
+            d["prev_hard"] = self.prev_hard
+        if self.curr_hard is not None:
+            d["curr_hard"] = self.curr_hard
+        if self.selection_hard is not None:
+            d["selection_hard"] = self.selection_hard
+        if self.selection_soft is not None:
+            d["selection_soft"] = self.selection_soft
+        if self.candidate_hash:
+            d["candidate_hash"] = self.candidate_hash
+        if self.update_origin:
+            d["update_origin"] = self.update_origin
+        if self.update_target:
+            d["update_target"] = self.update_target
+        return d

package/template/skills/skillopt-sleep/scripts/framework/skillopt/utils/__init__.py

@@ -0,0 +1,4 @@
+"""ReflACT utilities — JSON extraction, scoring, hashing."""
+
+from skillopt.utils.json_utils import extract_json, extract_json_array  # noqa: F401
+from skillopt.utils.scoring import compute_score, skill_hash  # noqa: F401

package/template/skills/skillopt-sleep/scripts/framework/skillopt/utils/json_utils.py

@@ -0,0 +1,172 @@
+"""JSON extraction helpers for LLM responses."""
+from __future__ import annotations
+
+import json
+import re
+import warnings
+
+
+def _top_level_brace_objects(text: str) -> list[str]:
+    """Return every balanced *top-level* ``{...}`` span in ``text``.
+
+    Fully string/escape aware: braces inside quoted strings are ignored both
+    when scanning for an object start AND while tracking depth inside one, so a
+    ``{`` that appears in prose (e.g. ``'set it to {x}'``) is never mistaken for
+    the start of a JSON object. Used to detect ambiguity: when a response carries
+    more than one top-level object we must not let a repair pass silently pick
+    one — it may pick the wrong (discarded) edit, strictly worse than None.
+    """
+    spans: list[str] = []
+    i, n = 0, len(text)
+    outer_in_str = False
+    outer_esc = False
+    while i < n:
+        ch = text[i]
+        # Skip over braces that live *inside* a quoted string before any object
+        # has started — otherwise a `{` in prose like '"set it to {x}"' is wrongly
+        # treated as an object start, and the repair pass below turns non-JSON
+        # prose into a bogus dict (strictly worse than returning None).
+        if outer_in_str:
+            if outer_esc:
+                outer_esc = False
+            elif ch == "\\":
+                outer_esc = True
+            elif ch == '"':
+                outer_in_str = False
+            i += 1
+            continue
+        if ch == '"':
+            outer_in_str = True
+            i += 1
+            continue
+        if ch != "{":
+            i += 1
+            continue
+        depth = 0
+        in_str = False
+        esc = False
+        start = i
+        while i < n:
+            ch = text[i]
+            if in_str:
+                if esc:
+                    esc = False
+                elif ch == "\\":
+                    esc = True
+                elif ch == '"':
+                    in_str = False
+            elif ch == '"':
+                in_str = True
+            elif ch == "{":
+                depth += 1
+            elif ch == "}":
+                depth -= 1
+                if depth == 0:
+                    spans.append(text[start:i + 1])
+                    i += 1
+                    break
+            i += 1
+        else:
+            break  # unterminated final object
+    return spans
+
+
+def _looks_json_like(span: str) -> bool:
+    """Heuristic: does ``span`` look like an intended JSON object (vs. prose)?
+
+    A genuine JSON object's first non-space character after ``{`` is either ``"``
+    (a string key) or ``}`` (an empty object). Prose pseudo-objects that the
+    repair pass would otherwise fabricate into bogus dicts — ``{op: delete}``,
+    ``{x: 1}`` quoted in single quotes or backticks, etc. — start with a bare
+    word and are rejected. This complements the string-aware scan, which only
+    skips *double*-quoted prose; single-quoted / backticked / unquoted prose
+    braces are caught here instead. Legitimate repair targets (trailing commas,
+    unescaped quotes inside string values) all begin with ``"`` and pass.
+    """
+    inner = span.strip()
+    if not (inner.startswith("{") and inner.endswith("}")):
+        return False
+    after_brace = inner[1:].lstrip()
+    return after_brace[:1] in ('"', '}')
+
+
+def extract_json(text: str) -> dict | None:
+    """Extract a JSON object from LLM response text.
+
+    Tries ```json fences first, then bare {...} patterns.
+    """
+    m = re.search(r"```json\s*(.*?)```", text, re.DOTALL)
+    if m:
+        try:
+            return json.loads(m.group(1))
+        except json.JSONDecodeError:
+            pass
+    m = re.search(r"\{.*\}", text, re.DOTALL)
+    if m:
+        try:
+            return json.loads(m.group(0))
+        except json.JSONDecodeError:
+            pass
+    # Tolerant fallback for non-OpenAI backends (Claude/Qwen, …) whose free-form
+    # JSON strict json.loads rejects — unescaped ASCII quotes inside CJK string
+    # values, trailing commas, etc. Repair so the analyst's edits aren't silently
+    # dropped, but ONLY a single unambiguous object: never feed the greedy `{.*}`
+    # span or the raw text, or json_repair would quietly return one of several
+    # objects (empirically the wrong/last one) — strictly worse than None, which
+    # the caller can detect and retry/skip.
+    #
+    # Pick the candidate FIRST, before importing json_repair, so the optional
+    # dependency only matters (and only warns) when there is genuinely a single
+    # malformed object we could have repaired. Ordinary no-JSON / prose replies
+    # have no candidate and return None silently.
+    candidate = None
+    fenced = re.search(r"```json\s*(.*?)```", text, re.DOTALL)
+    if fenced and len(_top_level_brace_objects(fenced.group(1))) == 1:
+        candidate = fenced.group(1)
+    else:
+        objs = _top_level_brace_objects(text)
+        if len(objs) == 1:
+            candidate = objs[0]
+        # 0 or >1 top-level objects → too ambiguous to repair safely → None
+    if not candidate:
+        return None
+    # Final guard: only repair spans that actually look like an intended JSON
+    # object. Prose pseudo-objects in single quotes / backticks / bare text
+    # (e.g. `{op: delete}`) reach here because the scan only skips double-quoted
+    # prose; repairing them would fabricate a wrong dict (worse than None).
+    if not _looks_json_like(candidate):
+        return None
+    try:
+        from json_repair import repair_json
+    except ModuleNotFoundError:
+        warnings.warn(
+            "json_repair not installed; malformed-JSON recovery disabled — "
+            "a non-OpenAI analyst edit may be silently dropped. pip install json_repair",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+        return None
+    try:
+        repaired = repair_json(candidate, return_objects=True)
+        if isinstance(repaired, dict) and repaired:
+            return repaired
+    except Exception:  # noqa: BLE001 — repair is best-effort
+        pass
+    return None
+
+
+def extract_json_array(text: str) -> list | None:
+    """Extract a JSON array from LLM response text."""
+    m = re.search(r"```json\s*(.*?)```", text, re.DOTALL)
+    if m:
+        try:
+            return json.loads(m.group(1))
+        except json.JSONDecodeError:
+            pass
+    m = re.search(r"\[.*\]", text, re.DOTALL)
+    if m:
+        try:
+            return json.loads(m.group(0))
+        except json.JSONDecodeError:
+            pass
+    return None

package/template/skills/skillopt-sleep/scripts/framework/skillopt/utils/scoring.py

@@ -0,0 +1,28 @@
+"""Scoring and hashing utilities."""
+from __future__ import annotations
+
+import hashlib
+
+
+def compute_score(results: list) -> tuple[float, float]:
+    """Compute hard and soft accuracy from a list of episode results.
+
+    Accepts both plain dicts and :class:    instances.  hard may be continuous (0.0-1.0) when using smoothed reward.
+    """
+    if not results:
+        return 0.0, 0.0
+
+    def _hard(r: object) -> float:
+        return float(r.hard if hasattr(r, "hard") else r.get("hard", 0))
+
+    def _soft(r: object) -> float:
+        return float(r.soft if hasattr(r, "soft") else r.get("soft", 0.0))
+
+    hard = sum(_hard(r) for r in results) / len(results)
+    soft = sum(_soft(r) for r in results) / len(results)
+    return hard, soft
+
+
+def skill_hash(content: str) -> str:
+    """Return a short deterministic hash of skill content (for caching)."""
+    return hashlib.sha256(content.encode()).hexdigest()[:16]