self-evolve-framework 1.5.0 → 1.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "self-evolve-framework",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "CodeBuddy 自我进化飞轮 — 仅适用于 CodeBuddy IDE。Post-Edit 验证闭环 + 规则自动推荐 + 离线进化分析",
   "type": "module",
   "bin": {

package/template/skills/skillopt-sleep/SKILL.md

@@ -80,3 +80,45 @@ python -m .codebuddy.skills.skillopt-sleep.scripts.python \
 ## 内核
 
 执行引擎来自 [microsoft/SkillOpt](https://github.com/microsoft/SkillOpt) — 由微软研究院开源的 skill 文档训练框架。
+
+## 文件结构
+
+```
+.codebuddy/skills/skillopt-sleep/
+├── SKILL.md                  ← 本指令文件
+├── scripts/
+│   ├── python/               ← 睡眠引擎（SkillOpt-Sleep）
+│   │   ├── cycle.py          → 六阶段循环（Harvest/Mine/Replay/Consolidate/Gate/Adopt）
+│   │   ├── harvest.py        → 会话历史收集
+│   │   ├── mine.py           → 模式挖掘
+│   │   ├── replay.py         → 回放验证
+│   │   ├── consolidate.py    → 合并优化
+│   │   ├── gate.py           → Held-out 验证门控
+│   │   ├── staging.py        → 暂存管理
+│   │   └── experiments/      → 基准实验
+│   ├── framework/            ← 核心训练框架（SkillOpt）
+│   │   ├── engine/trainer.py → Skill 训练器
+│   │   ├── envs/             → 基准环境（alfworld/docvqa/searchqa 等）
+│   │   ├── optimizer/        → 有界编辑优化器
+│   │   ├── prompts/          → Analyst/Merge/Rewrite 提示模板
+│   │   ├── model/            → LLM 后端
+│   │   ├── gradient/         → 梯度计算
+│   │   ├── evaluation/       → 评估
+│   │   └── scheduler/        → 调度
+│   └── shell/                ← 辅助脚本
+├── configs/
+│   ├── _base_/default.yaml   ← 默认配置
+│   ├── alfworld/             → AlfWorld 基准配置
+│   ├── docvqa/               → DocVQA 基准配置
+│   ├── searchqa/             → SearchQA 基准配置
+│   └── features/             → 特性开关（soft_gate）
+```
+
+## 进阶用法（训练新 skill）
+
+```bash
+cd .codebuddy/skills/skillopt-sleep/scripts/framework
+python -m skillopt.engine.trainer --config ../../configs/_base_/default.yaml
+```
+
+详细的训练配置见 `configs/` 中各基准的 YAML 文件。

package/template/skills/skillopt-sleep/configs/_base_/default.yaml

@@ -0,0 +1,103 @@
+# SkillOpt default configuration — base for all environments.
+# Environment configs should inherit via: _base_: default.yaml
+
+model:
+  backend: azure_openai
+  optimizer: gpt-5.5
+  target: gpt-5.5
+  optimizer_backend: openai_chat
+  target_backend: openai_chat
+  reasoning_effort: medium
+  rewrite_reasoning_effort: ""
+  rewrite_max_completion_tokens: 64000
+  codex_exec_path: codex
+  codex_exec_sandbox: workspace-write
+  codex_exec_profile: ""
+  codex_exec_full_auto: false
+  codex_exec_reasoning_effort: none
+  codex_exec_use_sdk: auto
+  codex_exec_network_access: false
+  codex_exec_web_search: false
+  codex_exec_approval_policy: never
+  claude_code_exec_path: claude
+  claude_code_exec_profile: ""
+  claude_code_exec_use_sdk: auto
+  claude_code_exec_effort: medium
+  claude_code_exec_max_thinking_tokens: 16384
+  codex_trace_to_optimizer: true
+  azure_openai_endpoint: ""           # e.g. "https://your-resource.openai.azure.com/"
+  azure_openai_api_version: "2024-12-01-preview"
+  azure_openai_api_key: ""       # Fill locally if you do not export AZURE_OPENAI_API_KEY
+  azure_openai_auth_mode: ""           # empty → fall back to AZURE_OPENAI_AUTH_MODE env (default "azure_cli")
+  azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
+  azure_openai_managed_identity_client_id: ""
+  optimizer_azure_openai_endpoint: ""   # e.g. "https://your-resource.openai.azure.com/"
+  optimizer_azure_openai_api_version: "2024-12-01-preview"
+  optimizer_azure_openai_api_key: ""
+  optimizer_azure_openai_auth_mode: ""           # empty → fall back to OPTIMIZER_AZURE_OPENAI_AUTH_MODE env, then shared
+  optimizer_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
+  optimizer_azure_openai_managed_identity_client_id: ""
+  target_azure_openai_endpoint: ""   # e.g. "https://your-resource.openai.azure.com/"
+  target_azure_openai_api_version: "2024-12-01-preview"
+  target_azure_openai_api_key: ""
+  target_azure_openai_auth_mode: ""           # empty → fall back to TARGET_AZURE_OPENAI_AUTH_MODE env, then shared
+  target_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
+  target_azure_openai_managed_identity_client_id: ""
+
+  # MiniMax backend settings (minimax_chat target)
+  minimax_base_url: ""          # https://api.minimax.io/v1 if blank
+  minimax_api_key: ""
+  minimax_model: "MiniMax-M2.7"
+  minimax_temperature: "0.7"
+  minimax_max_tokens: "8000"
+  minimax_enable_thinking: "false"
+  optimizer_minimax_base_url: ""    # per-role override
+  target_minimax_base_url: ""       # per-role override
+  optimizer_minimax_api_key: ""
+  target_minimax_api_key: ""
+
+train:
+  num_epochs: 4
+  train_size: 0          # 0 = derive from dataset split when available
+  batch_size: 40
+  accumulation: 1
+  seed: 42
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+  analyst_workers: 16
+  max_analyst_rounds: 3
+  failure_only: false
+
+optimizer:
+  learning_rate: 4          # max edits per step (edit_budget)
+  min_learning_rate: 2      # min edits for decay schedulers
+  lr_scheduler: cosine      # constant / linear / cosine / autonomous
+  lr_control_mode: fixed    # fixed / autonomous / none
+  skill_update_mode: patch  # patch / rewrite_from_suggestions / full_rewrite_minibatch
+  use_slow_update: true
+  slow_update_samples: 20
+  slow_update_gate_with_selection: false
+  longitudinal_pair_policy: mixed  # mixed / changed / unchanged
+  use_meta_skill: true
+  use_skill_aware_reflection: false  # EmbodiSkill: split failures into SKILL_DEFECT (edit body) vs EXECUTION_LAPSE (protected appendix)
+  skill_aware_appendix_source: both  # both = success+failure emit appendix notes; failure_only = only EXECUTION_LAPSE (paper-faithful)
+  skill_aware_consolidate_threshold: 0  # 0 = off; >0 = LLM-consolidate the appendix when its note count exceeds N
+
+evaluation:
+  use_gate: true
+  sel_env_num: 0
+  test_env_num: 0
+  eval_test: true
+
+env:
+  name: ""
+  skill_init: ""
+  split_mode: ratio       # ratio = build deterministic split from data_path; split_dir = use pre-split train/val/test
+  split_seed: 42
+  split_dir: ""
+  data_path: ""
+  split_output_dir: ""
+  exec_timeout: 120      # per target model/code-agent call timeout in seconds
+  out_root: ""

package/template/skills/skillopt-sleep/configs/alfworld/default.yaml

@@ -0,0 +1,29 @@
+_base_: ../_base_/default.yaml
+
+train:
+  train_size: 0
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+evaluation:
+  sel_env_num: 0
+  test_env_num: 0
+
+env:
+  name: alfworld
+  skill_init: skillopt/envs/alfworld/skills/initial.md
+  split_mode: split_dir
+  split_dir: data/alfworld_path_split
+  data_path: ""
+  split_output_dir: ""
+  max_steps: 50
+  max_completion_tokens: 16384
+  workers: 8
+  max_api_workers: 8
+  limit: 0

package/template/skills/skillopt-sleep/configs/docvqa/default.yaml

@@ -0,0 +1,28 @@
+_base_: ../_base_/default.yaml
+
+model:
+  reasoning_effort: medium
+
+train:
+  batch_size: 40
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+env:
+  name: docvqa
+  skill_init: skillopt/envs/docvqa/skills/initial.md
+  split_mode: split_dir
+  split_dir: data/docvqa/splits
+  data_path: ""
+  split_output_dir: ""
+  max_turns: 1
+  max_completion_tokens: 16384
+  workers: 16
+  image_detail: auto
+  limit: 0

package/template/skills/skillopt-sleep/configs/features/soft_gate.yaml

@@ -0,0 +1,47 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# Feature: soft / mixed validation-gate metric (community-contributed, PR #25)
+# ─────────────────────────────────────────────────────────────────────────────
+#
+# This is NOT a default SkillOpt setting and was NOT used to produce the
+# numbers reported in the paper. It is provided as a reference for users
+# who encounter a specific scenario where the default `hard` gate is too
+# coarse to drive training.
+#
+# When to consider this:
+#   - You are running on a custom environment.
+#   - Your held-out *selection* split has very few items (e.g. ≤ ~10).
+#   - Your reward function is continuous / partial-credit (e.g. F1, BLEU,
+#     soft match) rather than purely binary 0/1.
+#
+# Symptom this addresses:
+#   With a small selection split + continuous rewards, candidate skills
+#   often improve per-item soft scores (e.g. 0.06 → 0.26 on one item) but
+#   never flip the discrete hard outcome. The default `hard` gate then
+#   rejects every candidate and training stalls. Switching the gate to
+#   `soft` or `mixed` lets these partial improvements be accepted.
+#
+# When NOT to use this:
+#   - When reproducing the paper. The paper-reported numbers were obtained
+#     under the default `hard` gate.
+#   - When your selection split is large (dozens+ items) and / or your
+#     reward is already binary — `hard` is the more conservative choice
+#     and matches the design described in the paper.
+#
+# To use: inherit your env config from this file, e.g.
+#   _base_: ../features/soft_gate.yaml
+# or copy the `evaluation:` block below into your config.
+# ─────────────────────────────────────────────────────────────────────────────
+
+_base_: ../_base_/default.yaml
+
+evaluation:
+  # Three options:
+  #   'hard'  — default; exact-match accuracy. Use this to reproduce the paper.
+  #   'soft'  — per-item soft / partial-credit score (recommended for the
+  #             small-split + continuous-reward scenario described above).
+  #   'mixed' — weighted average: (1 - w) * hard + w * soft, with `w` set by
+  #             `gate_mixed_weight` below.
+  gate_metric: soft
+
+  # Only used when gate_metric == 'mixed'. Ignored otherwise.
+  gate_mixed_weight: 0.5

package/template/skills/skillopt-sleep/configs/livemathematicianbench/default.yaml

@@ -0,0 +1,22 @@
+_base_: ../_base_/default.yaml
+
+train:
+  train_size: 0
+  batch_size: 40
+  accumulation: 1
+
+env:
+  name: livemathematicianbench
+  skill_init: skillopt/envs/livemathematicianbench/skills/initial.md
+  split_mode: split_dir
+  split_dir: data/livemathematicianbench_split
+  data_path: ""
+  split_output_dir: ""
+  max_turns: 1
+  max_completion_tokens: 16384
+  exec_timeout: 300
+  workers: 64
+  limit: 0
+  shuffle_choices: true
+  use_theorem: false
+  use_sketch: false

package/template/skills/skillopt-sleep/configs/officeqa/default.yaml

@@ -0,0 +1,34 @@
+_base_: ../_base_/default.yaml
+
+model:
+  reasoning_effort: medium
+
+train:
+  batch_size: 40
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+env:
+  name: officeqa
+  skill_init: skillopt/envs/officeqa/skills/initial.md
+  split_mode: split_dir
+  split_dir: data/officeqa_split
+  data_dirs:
+    - data/officeqa_docs_official
+  workers: 4
+  max_tool_turns: 24
+  max_completion_tokens: 16384
+  search_mode: offline
+  max_queries_per_turn: 4
+  search_api_url: http://apisix.westus2.cloudapp.azure.com/search_tool/search
+  search_auth_env: OFFICEQA_CUSTOM_SEARCH_AUTH
+  search_provider: duckduckgo
+  search_max_num_results: 4
+  search_timeout_seconds: 20
+  limit: 0

package/template/skills/skillopt-sleep/configs/searchqa/default.yaml

@@ -0,0 +1,32 @@
+_base_: ../_base_/default.yaml
+
+model:
+  reasoning_effort: medium
+
+train:
+  train_size: 400
+  batch_size: 40
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+evaluation:
+  sel_env_num: 0
+  test_env_num: 0
+
+env:
+  name: searchqa
+  skill_init: skillopt/envs/searchqa/skills/initial.md
+  split_mode: split_dir
+  split_dir: data/searchqa_split
+  data_path: ""
+  split_output_dir: ""
+  max_turns: 1
+  max_completion_tokens: 16384
+  workers: 24
+  limit: 0

package/template/skills/skillopt-sleep/configs/spreadsheetbench/default.yaml

@@ -0,0 +1,34 @@
+_base_: ../_base_/default.yaml
+
+model:
+  reasoning_effort: medium
+
+train:
+  train_size: 80
+  batch_size: 40
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+evaluation:
+  sel_env_num: 0
+  test_env_num: 0
+
+env:
+  name: spreadsheetbench
+  skill_init: skillopt/envs/spreadsheetbench/skills/initial.md
+  split_mode: split_dir
+  split_dir: data/spreadsheetbench_split
+  data_path: ""
+  split_output_dir: ""
+  data_root: data/spreadsheetbench_verified_400
+  mode: multi
+  max_turns: 30
+  max_completion_tokens: 16384
+  exec_timeout: 600
+  workers: 24

package/template/skills/skillopt-sleep/scripts/framework/skillopt/__init__.py

@@ -0,0 +1,28 @@
+"""ReflACT: Reflective Agent Tuning.
+
+A general-purpose framework for iteratively optimizing LLM agent skills
+through structured reflection and self-improvement.
+
+Pipeline stages:
+  1. Rollout   — execute episodes with current skill
+  2. Reflect   — analyze trajectories, generate patches
+  3. Aggregate — hierarchical merge of patches
+  4. Select    — rank and select top edits
+  5. Update    — apply edits to skill document
+  6. Evaluate  — validate candidate skill, accept/reject
+"""
+
+__version__ = "0.1.0"
+
+from skillopt.types import (  # noqa: F401
+    BatchSpec,
+    Edit,
+    EditOp,
+    FailureSummaryEntry,
+    GateAction,
+    GateResult,
+    Patch,
+    RawPatch,
+    RolloutResult,
+    SlowUpdateResult,
+)