self-evolve-framework 1.6.0 → 1.6.1

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/__init__.py

@@ -1 +0,0 @@
-"""ReflACT environment adapters."""

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/_template/README.md

@@ -1,43 +0,0 @@
-# Benchmark Template
-
-This directory provides scaffold files for adding a new benchmark to SkillOpt.
-
-## Files
-
-- `env_template.py` — Environment adapter template (subclasses
-  `EnvAdapter`; implements the 4 abstract methods so the file is
-  instantiable out of the box — `reflect` is inherited).
-- `loader_template.py` — Data loader template (subclasses
-  `SplitDataLoader`; implements `load_split_items` for `.json`/`.jsonl`).
-- `config_template.yaml` — Config file template.
-
-## Usage
-
-1. **Copy the directory:**
-   ```bash
-   cp -r skillopt/envs/_template skillopt/envs/your_benchmark
-   ```
-2. **Rename the files** (drop the `_template` suffix):
-   ```bash
-   cd skillopt/envs/your_benchmark
-   mv env_template.py    adapter.py
-   mv loader_template.py dataloader.py
-   ```
-   …and inside each file rename the classes
-   (`TemplateBenchmarkEnv → YourBenchmarkAdapter`,
-   `TemplateBenchmarkLoader → YourBenchmarkLoader`)
-   and fix the cross-import in `adapter.py`.
-3. **Implement the TODO blocks** inside `adapter.py:rollout` and the
-   `_normalize_item` helper in `dataloader.py`. (`reflect` is inherited from
-   `EnvAdapter`; override it only for custom reflection logic.)
-4. **Register** the adapter — add a `try / except ImportError` block in
-   `scripts/train.py`'s `_register_builtins()` mapping the registry key
-   to your `YourBenchmarkAdapter` class. There is no
-   `BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`; the live
-   registry is `_ENV_REGISTRY` in `scripts/train.py`.
-5. **Create the config** at `configs/your_benchmark/default.yaml`
-   (start from `config_template.yaml`). `_base_` is a **string path**,
-   not a list.
-
-See the [Add a New Benchmark guide](../../../docs/guide/new-benchmark.md)
-for the full step-by-step with a worked `docfaithful` example.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/_template/config_template.yaml

@@ -1,55 +0,0 @@
-# ──────────────────────────────────────────────────
-# SkillOpt Config Template — <Your Benchmark Name>
-# ──────────────────────────────────────────────────
-# Copy this file to configs/<your_benchmark>/default.yaml
-# and customize the values below.
-
-# Inherit global defaults.
-# NOTE: `_base_` is a string path, not a list.
-_base_: ../_base_/default.yaml
-
-# ── Environment ──────────────────────────────────
-env:
-  name: your_benchmark            # Must match the key registered in scripts/train.py
-  # Optional: a seed skill document. Create this file yourself before the
-  # first run, or omit the key to start from an empty skill.
-  # skill_init: skillopt/envs/your_benchmark/skills/initial.md
-  data_path: data/your_benchmark  # Path to your data (for split_mode: ratio)
-  split_dir: ""                   # Set this and use split_mode: split_dir for pre-split data
-  split_mode: ratio               # "ratio" or "split_dir"
-  split_ratio: "2:1:7"            # train:val:test (used when split_mode: ratio)
-  workers: 4                      # Parallel rollout workers
-  max_completion_tokens: 4096     # Cap per target-model call
-  limit: 0                        # 0 = no limit; small int = debug sample
-
-# ── Training ─────────────────────────────────────
-train:
-  num_epochs: 4
-  batch_size: 40
-  accumulation: 1
-  seed: 42
-
-# ── Gradient (Reflection) ───────────────────────
-gradient:
-  analyst_workers: 16             # Parallel reflection workers
-  minibatch_size: 8
-  merge_batch_size: 8
-
-# ── Optimizer ────────────────────────────────────
-optimizer:
-  learning_rate: 4                # Max edits per step (edit budget)
-  lr_scheduler: cosine            # cosine | linear | constant | autonomous
-  use_slow_update: true           # Epoch-boundary momentum
-  use_meta_skill: true            # Cross-epoch optimizer memory
-
-# ── Evaluation ───────────────────────────────────
-evaluation:
-  use_gate: true                  # Validation gating
-  eval_test: true                 # Run test eval after training
-
-# ── Model ────────────────────────────────────────
-# Override only what differs from the inherited defaults.
-model:
-  optimizer_backend: openai_chat  # openai_chat | claude_chat | qwen_chat | minimax_chat
-  target_backend: openai_chat     # … plus codex_exec / claude_code_exec for target only
-  reasoning_effort: medium

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/_template/env_template.py

@@ -1,151 +0,0 @@
-"""
-Benchmark Environment Template
-===============================
-Copy this file and implement the TODO sections to add a new benchmark.
-
-The EnvAdapter is responsible for:
-  1. Building per-batch environment managers (train and eval splits).
-  2. Running rollouts under the current skill document.
-  3. Reflecting on those rollouts into raw patch dicts.
-  4. Reporting the distinct task types in your data (for stratified
-     sampling).
-
-For a fully worked example see ``skillopt/envs/officeqa/``.
-"""
-from __future__ import annotations
-
-from skillopt.datasets.base import BatchSpec
-from skillopt.envs.base import EnvAdapter
-from skillopt.envs._template.loader_template import TemplateBenchmarkLoader
-
-
-class TemplateBenchmarkEnv(EnvAdapter):
-    """
-    Environment adapter for <Your Benchmark Name>.
-
-    Rename this class. Each abstract method below is required by
-    :class:`skillopt.envs.base.EnvAdapter`. The template implementations
-    are minimal so this file is importable and instantiable; replace the
-    TODOs with real logic.
-    """
-
-    def __init__(
-        self,
-        split_dir: str = "",
-        data_path: str = "",
-        split_mode: str = "split_dir",
-        split_ratio: str = "2:1:7",
-        split_seed: int = 42,
-        split_output_dir: str = "",
-        workers: int = 4,
-        analyst_workers: int = 4,
-        failure_only: bool = False,
-        minibatch_size: int = 8,
-        edit_budget: int = 4,
-        seed: int = 42,
-        limit: int = 0,
-        max_completion_tokens: int = 4096,
-    ) -> None:
-        self.workers = workers
-        self.analyst_workers = analyst_workers
-        self.failure_only = failure_only
-        self.minibatch_size = minibatch_size
-        self.edit_budget = edit_budget
-        self.max_completion_tokens = int(max_completion_tokens)
-        self.dataloader = TemplateBenchmarkLoader(
-            split_dir=split_dir,
-            data_path=data_path,
-            split_mode=split_mode,
-            split_ratio=split_ratio,
-            split_seed=split_seed,
-            split_output_dir=split_output_dir,
-            seed=seed,
-            limit=limit,
-        )
-
-    # ── Lifecycle hooks ────────────────────────────────────────────────
-
-    def setup(self, cfg: dict) -> None:
-        super().setup(cfg)
-        self.dataloader.setup(cfg)
-
-    def get_dataloader(self):
-        return self.dataloader
-
-    # ── Batch → env manager ────────────────────────────────────────────
-
-    def build_env_from_batch(self, batch: BatchSpec, **kwargs):
-        # Dataset-backed envs typically just pass items straight through.
-        return list(batch.payload or [])
-
-    def build_train_env(self, batch_size: int, seed: int, **kwargs):
-        batch = self.dataloader.build_train_batch(
-            batch_size=batch_size, seed=seed, **kwargs
-        )
-        return self.build_env_from_batch(batch, **kwargs)
-
-    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
-        batch = self.dataloader.build_eval_batch(
-            env_num=env_num, split=split, seed=seed, **kwargs
-        )
-        return self.build_env_from_batch(batch, **kwargs)
-
-    # ── Rollout: run episodes under current skill ──────────────────────
-
-    def rollout(
-        self,
-        env_manager,
-        skill_content: str,
-        out_dir: str,
-        **kwargs,
-    ) -> list[dict]:
-        """
-        Run a batch of episodes under the current skill.
-
-        TODO: replace this loop with your real rollout. For each item:
-          1. Build the prompt using `skill_content` as the system message.
-          2. Call your target model.
-          3. Score the prediction.
-          4. Return a dict with at minimum: ``id`` (str), ``hard`` (0|1),
-             ``soft`` (float in [0, 1]). Add any env-specific extras you
-             need for reflect() — they will be preserved on
-             ``RolloutResult.extras``.
-        """
-        items: list[dict] = env_manager
-        results: list[dict] = []
-        for item in items:
-            # ── REPLACE THIS BLOCK WITH YOUR REAL ROLLOUT ──
-            results.append(
-                {
-                    "id": str(item.get("id", "")),
-                    "hard": 0,
-                    "soft": 0.0,
-                    "predicted_answer": "",
-                    "question": item.get("question", ""),
-                    "fail_reason": "template rollout — not implemented",
-                }
-            )
-        return results
-
-    # ── Reflect (inherited) ─────────────────────────────────────────────
-    #
-    # ``reflect`` is inherited from ``EnvAdapter``: the default delegates to
-    # ``skillopt.gradient.reflect.run_minibatch_reflect`` using your
-    # ``analyst_error_*`` / ``analyst_success_*`` prompts. You do NOT need to
-    # implement it — override only if your benchmark needs custom reflection.
-
-    # ── Stratification hint ────────────────────────────────────────────
-
-    def get_task_types(self) -> list[str]:
-        """Distinct task-type strings used for stratified sampling."""
-        seen: list[str] = []
-        all_items = (
-            self.dataloader.train_items
-            + self.dataloader.val_items
-            + self.dataloader.test_items
-        )
-        for item in all_items:
-            tt = str(item.get("task_type") or "template")
-            if tt not in seen:
-                seen.append(tt)
-        return seen or ["template"]

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/_template/loader_template.py

@@ -1,87 +0,0 @@
-"""
-Benchmark Data Loader Template
-================================
-Copy this file and implement ``load_split_items`` to load your benchmark
-data. The loader is a :class:`skillopt.datasets.base.SplitDataLoader`
-subclass — the base class handles both ``split_mode="split_dir"`` (read
-an existing train/val/test layout) and ``split_mode="ratio"`` (build the
-splits from a single raw file deterministically).
-
-For a fully worked example see
-``skillopt/envs/officeqa/dataloader.py``.
-"""
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-from skillopt.datasets.base import SplitDataLoader
-
-
-def _normalize_item(raw: dict) -> dict:
-    """
-    Normalise one raw entry into the dict shape SkillOpt expects.
-
-    The only **hard** requirement is ``"id"`` (str). Add whatever extra
-    fields your :class:`TemplateBenchmarkEnv.rollout` needs.
-    """
-    return {
-        "id": str(raw.get("uid") or raw.get("id") or ""),
-        "question": str(raw.get("question") or raw.get("prompt") or ""),
-        "ground_truth": str(raw.get("ground_truth") or raw.get("answer") or ""),
-        "task_type": str(raw.get("category") or raw.get("task_type") or "template"),
-        # ── add benchmark-specific keys here ──
-    }
-
-
-class TemplateBenchmarkLoader(SplitDataLoader):
-    """
-    Data loader for <Your Benchmark Name>.
-
-    Subclass note: you usually only need to implement
-    :meth:`load_split_items`. The base class drives ``setup(cfg)``,
-    materialises ratio-mode splits, exposes ``train_items``,
-    ``val_items``, ``test_items``, and builds ``BatchSpec`` objects on
-    demand.
-
-    If you want to support ``split_mode="ratio"`` (auto-split a single
-    file into train/val/test), also implement
-    :meth:`load_raw_items(data_path)` returning the full list of items.
-    """
-
-    def load_split_items(self, split_path: str) -> list[dict]:
-        """Load all items for one split directory.
-
-        ``split_path`` is e.g. ``data/your_benchmark/train/``. Return a
-        list of dicts, each shaped like :func:`_normalize_item`'s output.
-        """
-        path = Path(split_path)
-
-        json_files = sorted(path.glob("*.json"))
-        if json_files:
-            with json_files[0].open(encoding="utf-8") as f:
-                payload = json.load(f)
-            if not isinstance(payload, list):
-                raise ValueError(
-                    f"Expected JSON array at top level of {json_files[0]}"
-                )
-            return [_normalize_item(row) for row in payload]
-
-        jsonl_files = sorted(path.glob("*.jsonl"))
-        if jsonl_files:
-            items: list[dict] = []
-            with jsonl_files[0].open(encoding="utf-8") as f:
-                for line in f:
-                    line = line.strip()
-                    if not line:
-                        continue
-                    items.append(_normalize_item(json.loads(line)))
-            return items
-
-        raise FileNotFoundError(
-            f"No .json or .jsonl file found in {split_path}"
-        )
-
-    # Optional — only needed if you intend to use ``split_mode='ratio'``.
-    # def load_raw_items(self, data_path: str) -> list[dict]:
-    #     ...

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/alfworld/__init__.py

@@ -1,5 +0,0 @@
-"""ALFWorld environment adapter for ReflACT."""
-
-from skillopt.envs.alfworld.adapter import ALFWorldAdapter
-
-__all__ = ["ALFWorldAdapter"]