self-evolve-framework 1.5.0 → 1.6.0

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/__init__.py

@@ -0,0 +1 @@
+"""ReflACT environment adapters."""

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/_template/README.md

@@ -0,0 +1,43 @@
+# Benchmark Template
+
+This directory provides scaffold files for adding a new benchmark to SkillOpt.
+
+## Files
+
+- `env_template.py` — Environment adapter template (subclasses
+  `EnvAdapter`; implements the 4 abstract methods so the file is
+  instantiable out of the box — `reflect` is inherited).
+- `loader_template.py` — Data loader template (subclasses
+  `SplitDataLoader`; implements `load_split_items` for `.json`/`.jsonl`).
+- `config_template.yaml` — Config file template.
+
+## Usage
+
+1. **Copy the directory:**
+   ```bash
+   cp -r skillopt/envs/_template skillopt/envs/your_benchmark
+   ```
+2. **Rename the files** (drop the `_template` suffix):
+   ```bash
+   cd skillopt/envs/your_benchmark
+   mv env_template.py    adapter.py
+   mv loader_template.py dataloader.py
+   ```
+   …and inside each file rename the classes
+   (`TemplateBenchmarkEnv → YourBenchmarkAdapter`,
+   `TemplateBenchmarkLoader → YourBenchmarkLoader`)
+   and fix the cross-import in `adapter.py`.
+3. **Implement the TODO blocks** inside `adapter.py:rollout` and the
+   `_normalize_item` helper in `dataloader.py`. (`reflect` is inherited from
+   `EnvAdapter`; override it only for custom reflection logic.)
+4. **Register** the adapter — add a `try / except ImportError` block in
+   `scripts/train.py`'s `_register_builtins()` mapping the registry key
+   to your `YourBenchmarkAdapter` class. There is no
+   `BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`; the live
+   registry is `_ENV_REGISTRY` in `scripts/train.py`.
+5. **Create the config** at `configs/your_benchmark/default.yaml`
+   (start from `config_template.yaml`). `_base_` is a **string path**,
+   not a list.
+
+See the [Add a New Benchmark guide](../../../docs/guide/new-benchmark.md)
+for the full step-by-step with a worked `docfaithful` example.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/_template/config_template.yaml

@@ -0,0 +1,55 @@
+# ──────────────────────────────────────────────────
+# SkillOpt Config Template — <Your Benchmark Name>
+# ──────────────────────────────────────────────────
+# Copy this file to configs/<your_benchmark>/default.yaml
+# and customize the values below.
+
+# Inherit global defaults.
+# NOTE: `_base_` is a string path, not a list.
+_base_: ../_base_/default.yaml
+
+# ── Environment ──────────────────────────────────
+env:
+  name: your_benchmark            # Must match the key registered in scripts/train.py
+  # Optional: a seed skill document. Create this file yourself before the
+  # first run, or omit the key to start from an empty skill.
+  # skill_init: skillopt/envs/your_benchmark/skills/initial.md
+  data_path: data/your_benchmark  # Path to your data (for split_mode: ratio)
+  split_dir: ""                   # Set this and use split_mode: split_dir for pre-split data
+  split_mode: ratio               # "ratio" or "split_dir"
+  split_ratio: "2:1:7"            # train:val:test (used when split_mode: ratio)
+  workers: 4                      # Parallel rollout workers
+  max_completion_tokens: 4096     # Cap per target-model call
+  limit: 0                        # 0 = no limit; small int = debug sample
+
+# ── Training ─────────────────────────────────────
+train:
+  num_epochs: 4
+  batch_size: 40
+  accumulation: 1
+  seed: 42
+
+# ── Gradient (Reflection) ───────────────────────
+gradient:
+  analyst_workers: 16             # Parallel reflection workers
+  minibatch_size: 8
+  merge_batch_size: 8
+
+# ── Optimizer ────────────────────────────────────
+optimizer:
+  learning_rate: 4                # Max edits per step (edit budget)
+  lr_scheduler: cosine            # cosine | linear | constant | autonomous
+  use_slow_update: true           # Epoch-boundary momentum
+  use_meta_skill: true            # Cross-epoch optimizer memory
+
+# ── Evaluation ───────────────────────────────────
+evaluation:
+  use_gate: true                  # Validation gating
+  eval_test: true                 # Run test eval after training
+
+# ── Model ────────────────────────────────────────
+# Override only what differs from the inherited defaults.
+model:
+  optimizer_backend: openai_chat  # openai_chat | claude_chat | qwen_chat | minimax_chat
+  target_backend: openai_chat     # … plus codex_exec / claude_code_exec for target only
+  reasoning_effort: medium

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/_template/env_template.py

@@ -0,0 +1,151 @@
+"""
+Benchmark Environment Template
+===============================
+Copy this file and implement the TODO sections to add a new benchmark.
+
+The EnvAdapter is responsible for:
+  1. Building per-batch environment managers (train and eval splits).
+  2. Running rollouts under the current skill document.
+  3. Reflecting on those rollouts into raw patch dicts.
+  4. Reporting the distinct task types in your data (for stratified
+     sampling).
+
+For a fully worked example see ``skillopt/envs/officeqa/``.
+"""
+from __future__ import annotations
+
+from skillopt.datasets.base import BatchSpec
+from skillopt.envs.base import EnvAdapter
+from skillopt.envs._template.loader_template import TemplateBenchmarkLoader
+
+
+class TemplateBenchmarkEnv(EnvAdapter):
+    """
+    Environment adapter for <Your Benchmark Name>.
+
+    Rename this class. Each abstract method below is required by
+    :class:`skillopt.envs.base.EnvAdapter`. The template implementations
+    are minimal so this file is importable and instantiable; replace the
+    TODOs with real logic.
+    """
+
+    def __init__(
+        self,
+        split_dir: str = "",
+        data_path: str = "",
+        split_mode: str = "split_dir",
+        split_ratio: str = "2:1:7",
+        split_seed: int = 42,
+        split_output_dir: str = "",
+        workers: int = 4,
+        analyst_workers: int = 4,
+        failure_only: bool = False,
+        minibatch_size: int = 8,
+        edit_budget: int = 4,
+        seed: int = 42,
+        limit: int = 0,
+        max_completion_tokens: int = 4096,
+    ) -> None:
+        self.workers = workers
+        self.analyst_workers = analyst_workers
+        self.failure_only = failure_only
+        self.minibatch_size = minibatch_size
+        self.edit_budget = edit_budget
+        self.max_completion_tokens = int(max_completion_tokens)
+        self.dataloader = TemplateBenchmarkLoader(
+            split_dir=split_dir,
+            data_path=data_path,
+            split_mode=split_mode,
+            split_ratio=split_ratio,
+            split_seed=split_seed,
+            split_output_dir=split_output_dir,
+            seed=seed,
+            limit=limit,
+        )
+
+    # ── Lifecycle hooks ────────────────────────────────────────────────
+
+    def setup(self, cfg: dict) -> None:
+        super().setup(cfg)
+        self.dataloader.setup(cfg)
+
+    def get_dataloader(self):
+        return self.dataloader
+
+    # ── Batch → env manager ────────────────────────────────────────────
+
+    def build_env_from_batch(self, batch: BatchSpec, **kwargs):
+        # Dataset-backed envs typically just pass items straight through.
+        return list(batch.payload or [])
+
+    def build_train_env(self, batch_size: int, seed: int, **kwargs):
+        batch = self.dataloader.build_train_batch(
+            batch_size=batch_size, seed=seed, **kwargs
+        )
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
+        batch = self.dataloader.build_eval_batch(
+            env_num=env_num, split=split, seed=seed, **kwargs
+        )
+        return self.build_env_from_batch(batch, **kwargs)
+
+    # ── Rollout: run episodes under current skill ──────────────────────
+
+    def rollout(
+        self,
+        env_manager,
+        skill_content: str,
+        out_dir: str,
+        **kwargs,
+    ) -> list[dict]:
+        """
+        Run a batch of episodes under the current skill.
+
+        TODO: replace this loop with your real rollout. For each item:
+          1. Build the prompt using `skill_content` as the system message.
+          2. Call your target model.
+          3. Score the prediction.
+          4. Return a dict with at minimum: ``id`` (str), ``hard`` (0|1),
+             ``soft`` (float in [0, 1]). Add any env-specific extras you
+             need for reflect() — they will be preserved on
+             ``RolloutResult.extras``.
+        """
+        items: list[dict] = env_manager
+        results: list[dict] = []
+        for item in items:
+            # ── REPLACE THIS BLOCK WITH YOUR REAL ROLLOUT ──
+            results.append(
+                {
+                    "id": str(item.get("id", "")),
+                    "hard": 0,
+                    "soft": 0.0,
+                    "predicted_answer": "",
+                    "question": item.get("question", ""),
+                    "fail_reason": "template rollout — not implemented",
+                }
+            )
+        return results
+
+    # ── Reflect (inherited) ─────────────────────────────────────────────
+    #
+    # ``reflect`` is inherited from ``EnvAdapter``: the default delegates to
+    # ``skillopt.gradient.reflect.run_minibatch_reflect`` using your
+    # ``analyst_error_*`` / ``analyst_success_*`` prompts. You do NOT need to
+    # implement it — override only if your benchmark needs custom reflection.
+
+    # ── Stratification hint ────────────────────────────────────────────
+
+    def get_task_types(self) -> list[str]:
+        """Distinct task-type strings used for stratified sampling."""
+        seen: list[str] = []
+        all_items = (
+            self.dataloader.train_items
+            + self.dataloader.val_items
+            + self.dataloader.test_items
+        )
+        for item in all_items:
+            tt = str(item.get("task_type") or "template")
+            if tt not in seen:
+                seen.append(tt)
+        return seen or ["template"]

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/_template/loader_template.py

@@ -0,0 +1,87 @@
+"""
+Benchmark Data Loader Template
+================================
+Copy this file and implement ``load_split_items`` to load your benchmark
+data. The loader is a :class:`skillopt.datasets.base.SplitDataLoader`
+subclass — the base class handles both ``split_mode="split_dir"`` (read
+an existing train/val/test layout) and ``split_mode="ratio"`` (build the
+splits from a single raw file deterministically).
+
+For a fully worked example see
+``skillopt/envs/officeqa/dataloader.py``.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from skillopt.datasets.base import SplitDataLoader
+
+
+def _normalize_item(raw: dict) -> dict:
+    """
+    Normalise one raw entry into the dict shape SkillOpt expects.
+
+    The only **hard** requirement is ``"id"`` (str). Add whatever extra
+    fields your :class:`TemplateBenchmarkEnv.rollout` needs.
+    """
+    return {
+        "id": str(raw.get("uid") or raw.get("id") or ""),
+        "question": str(raw.get("question") or raw.get("prompt") or ""),
+        "ground_truth": str(raw.get("ground_truth") or raw.get("answer") or ""),
+        "task_type": str(raw.get("category") or raw.get("task_type") or "template"),
+        # ── add benchmark-specific keys here ──
+    }
+
+
+class TemplateBenchmarkLoader(SplitDataLoader):
+    """
+    Data loader for <Your Benchmark Name>.
+
+    Subclass note: you usually only need to implement
+    :meth:`load_split_items`. The base class drives ``setup(cfg)``,
+    materialises ratio-mode splits, exposes ``train_items``,
+    ``val_items``, ``test_items``, and builds ``BatchSpec`` objects on
+    demand.
+
+    If you want to support ``split_mode="ratio"`` (auto-split a single
+    file into train/val/test), also implement
+    :meth:`load_raw_items(data_path)` returning the full list of items.
+    """
+
+    def load_split_items(self, split_path: str) -> list[dict]:
+        """Load all items for one split directory.
+
+        ``split_path`` is e.g. ``data/your_benchmark/train/``. Return a
+        list of dicts, each shaped like :func:`_normalize_item`'s output.
+        """
+        path = Path(split_path)
+
+        json_files = sorted(path.glob("*.json"))
+        if json_files:
+            with json_files[0].open(encoding="utf-8") as f:
+                payload = json.load(f)
+            if not isinstance(payload, list):
+                raise ValueError(
+                    f"Expected JSON array at top level of {json_files[0]}"
+                )
+            return [_normalize_item(row) for row in payload]
+
+        jsonl_files = sorted(path.glob("*.jsonl"))
+        if jsonl_files:
+            items: list[dict] = []
+            with jsonl_files[0].open(encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    items.append(_normalize_item(json.loads(line)))
+            return items
+
+        raise FileNotFoundError(
+            f"No .json or .jsonl file found in {split_path}"
+        )
+
+    # Optional — only needed if you intend to use ``split_mode='ratio'``.
+    # def load_raw_items(self, data_path: str) -> list[dict]:
+    #     ...

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/alfworld/__init__.py

@@ -0,0 +1,5 @@
+"""ALFWorld environment adapter for ReflACT."""
+
+from skillopt.envs.alfworld.adapter import ALFWorldAdapter
+
+__all__ = ["ALFWorldAdapter"]