self-evolve-framework 1.6.0 → 1.6.1

package/template/skills/impeccable/scripts/pin.mjs

@@ -1,214 +0,0 @@
-#!/usr/bin/env node
-/**
- * Pin/unpin sub-commands as standalone skill shortcuts.
- *
- * Usage:
- *   node <scripts_path>/pin.mjs pin <command>
- *   node <scripts_path>/pin.mjs unpin <command>
- *
- * `pin audit` creates a lightweight /audit skill that redirects to /impeccable audit.
- * `unpin audit` removes that shortcut.
- *
- * The script discovers harness directories (.claude/skills, .cursor/skills, etc.)
- * in the project root and creates/removes the pin in all of them.
- */
-
-import { existsSync, readFileSync, writeFileSync, mkdirSync, rmSync, readdirSync } from 'node:fs';
-import { join, resolve, dirname } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-// All known harness directories
-const HARNESS_DIRS = [
-  '.claude', '.cursor', '.gemini', '.codex', '.agents',
-  '.trae', '.trae-cn', '.pi', '.opencode', '.kiro', '.rovodev',
-];
-
-// Valid sub-command names
-const VALID_COMMANDS = [
-  'craft', 'init', 'extract', 'document', 'shape',
-  'critique', 'audit',
-  'polish', 'bolder', 'quieter', 'distill', 'harden', 'onboard', 'live',
-  'animate', 'colorize', 'typeset', 'layout', 'delight', 'overdrive',
-  'clarify', 'adapt', 'optimize',
-];
-
-// Marker to identify pinned skills (so unpin doesn't delete user skills)
-const PIN_MARKER = '<!-- impeccable-pinned-skill -->';
-
-/**
- * Walk up from startDir to find a project root.
- */
-function findProjectRoot(startDir = process.cwd()) {
-  let dir = resolve(startDir);
-  while (dir !== '/') {
-    if (
-      existsSync(join(dir, 'package.json')) ||
-      existsSync(join(dir, '.git')) ||
-      existsSync(join(dir, 'skills-lock.json'))
-    ) {
-      return dir;
-    }
-    const parent = resolve(dir, '..');
-    if (parent === dir) break;
-    dir = parent;
-  }
-  return resolve(startDir);
-}
-
-/**
- * Find harness skill directories that have an impeccable skill installed.
- */
-function findHarnessDirs(projectRoot) {
-  const dirs = [];
-  for (const harness of HARNESS_DIRS) {
-    const skillsDir = join(projectRoot, harness, 'skills');
-    // Only pin in harness dirs that already have impeccable installed
-    const impeccableDir = join(skillsDir, 'impeccable');
-    if (existsSync(impeccableDir) || existsSync(join(skillsDir, 'i-impeccable'))) {
-      dirs.push(skillsDir);
-    }
-  }
-  return dirs;
-}
-
-/**
- * Load command metadata (descriptions for pinned skills).
- */
-function loadCommandMetadata() {
-  const metadataPath = join(__dirname, 'command-metadata.json');
-  if (existsSync(metadataPath)) {
-    return JSON.parse(readFileSync(metadataPath, 'utf-8'));
-  }
-  return {};
-}
-
-/**
- * Generate a pinned skill's SKILL.md content.
- */
-function generatePinnedSkill(command, metadata) {
-  const desc = metadata[command]?.description || `Shortcut for /impeccable ${command}.`;
-  const hint = metadata[command]?.argumentHint || '[target]';
-
-  return `---
-name: ${command}
-description: "${desc}"
-argument-hint: "${hint}"
-user-invocable: true
----
-
-${PIN_MARKER}
-
-This is a pinned shortcut for \`{{command_prefix}}impeccable ${command}\`.
-
-Invoke {{command_prefix}}impeccable ${command}, passing along any arguments provided here, and follow its instructions.
-`;
-}
-
-/**
- * Pin a command: create shortcut skill in all harness dirs.
- */
-function pin(command, projectRoot) {
-  const metadata = loadCommandMetadata();
-  const harnessDirs = findHarnessDirs(projectRoot);
-
-  if (harnessDirs.length === 0) {
-    console.log('No harness directories with impeccable installed found.');
-    return false;
-  }
-
-  const content = generatePinnedSkill(command, metadata);
-  let created = 0;
-
-  for (const skillsDir of harnessDirs) {
-    // Check if skill already exists (and isn't a pin)
-    const skillDir = join(skillsDir, command);
-    if (existsSync(skillDir)) {
-      const existingMd = join(skillDir, 'SKILL.md');
-      if (existsSync(existingMd)) {
-        const existing = readFileSync(existingMd, 'utf-8');
-        if (!existing.includes(PIN_MARKER)) {
-          console.log(`  SKIP: ${skillDir} (non-pinned skill already exists)`);
-          continue;
-        }
-      }
-    }
-
-    mkdirSync(skillDir, { recursive: true });
-    writeFileSync(join(skillDir, 'SKILL.md'), content, 'utf-8');
-    console.log(`  + ${skillDir}`);
-    created++;
-  }
-
-  if (created > 0) {
-    console.log(`\nPinned '${command}' as a standalone shortcut in ${created} location(s).`);
-    console.log(`You can now use /${command} directly.`);
-  }
-
-  return created > 0;
-}
-
-/**
- * Unpin a command: remove shortcut skill from all harness dirs.
- */
-function unpin(command, projectRoot) {
-  const harnessDirs = findHarnessDirs(projectRoot);
-  let removed = 0;
-
-  for (const skillsDir of harnessDirs) {
-    const skillDir = join(skillsDir, command);
-    if (!existsSync(skillDir)) continue;
-
-    const skillMd = join(skillDir, 'SKILL.md');
-    if (!existsSync(skillMd)) continue;
-
-    // Safety: only remove if it's a pinned skill
-    const content = readFileSync(skillMd, 'utf-8');
-    if (!content.includes(PIN_MARKER)) {
-      console.log(`  SKIP: ${skillDir} (not a pinned skill)`);
-      continue;
-    }
-
-    rmSync(skillDir, { recursive: true, force: true });
-    console.log(`  - ${skillDir}`);
-    removed++;
-  }
-
-  if (removed > 0) {
-    console.log(`\nUnpinned '${command}' from ${removed} location(s).`);
-    console.log(`Use /impeccable ${command} to access it.`);
-  } else {
-    console.log(`No pinned '${command}' shortcut found.`);
-  }
-
-  return removed > 0;
-}
-
-// --- CLI ---
-const [,, action, command] = process.argv;
-
-if (!action || !command) {
-  console.log('Usage: node pin.mjs <pin|unpin> <command>');
-  console.log(`\nAvailable commands: ${VALID_COMMANDS.join(', ')}`);
-  process.exit(1);
-}
-
-if (action !== 'pin' && action !== 'unpin') {
-  console.error(`Unknown action: ${action}. Use 'pin' or 'unpin'.`);
-  process.exit(1);
-}
-
-if (!VALID_COMMANDS.includes(command)) {
-  console.error(`Unknown command: ${command}`);
-  console.error(`Available commands: ${VALID_COMMANDS.join(', ')}`);
-  process.exit(1);
-}
-
-const root = findProjectRoot();
-
-if (action === 'pin') {
-  pin(command, root);
-} else {
-  unpin(command, root);
-}

package/template/skills/skillopt-sleep/configs/_base_/default.yaml

@@ -1,103 +0,0 @@
-# SkillOpt default configuration — base for all environments.
-# Environment configs should inherit via: _base_: default.yaml
-
-model:
-  backend: azure_openai
-  optimizer: gpt-5.5
-  target: gpt-5.5
-  optimizer_backend: openai_chat
-  target_backend: openai_chat
-  reasoning_effort: medium
-  rewrite_reasoning_effort: ""
-  rewrite_max_completion_tokens: 64000
-  codex_exec_path: codex
-  codex_exec_sandbox: workspace-write
-  codex_exec_profile: ""
-  codex_exec_full_auto: false
-  codex_exec_reasoning_effort: none
-  codex_exec_use_sdk: auto
-  codex_exec_network_access: false
-  codex_exec_web_search: false
-  codex_exec_approval_policy: never
-  claude_code_exec_path: claude
-  claude_code_exec_profile: ""
-  claude_code_exec_use_sdk: auto
-  claude_code_exec_effort: medium
-  claude_code_exec_max_thinking_tokens: 16384
-  codex_trace_to_optimizer: true
-  azure_openai_endpoint: ""           # e.g. "https://your-resource.openai.azure.com/"
-  azure_openai_api_version: "2024-12-01-preview"
-  azure_openai_api_key: ""       # Fill locally if you do not export AZURE_OPENAI_API_KEY
-  azure_openai_auth_mode: ""           # empty → fall back to AZURE_OPENAI_AUTH_MODE env (default "azure_cli")
-  azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
-  azure_openai_managed_identity_client_id: ""
-  optimizer_azure_openai_endpoint: ""   # e.g. "https://your-resource.openai.azure.com/"
-  optimizer_azure_openai_api_version: "2024-12-01-preview"
-  optimizer_azure_openai_api_key: ""
-  optimizer_azure_openai_auth_mode: ""           # empty → fall back to OPTIMIZER_AZURE_OPENAI_AUTH_MODE env, then shared
-  optimizer_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
-  optimizer_azure_openai_managed_identity_client_id: ""
-  target_azure_openai_endpoint: ""   # e.g. "https://your-resource.openai.azure.com/"
-  target_azure_openai_api_version: "2024-12-01-preview"
-  target_azure_openai_api_key: ""
-  target_azure_openai_auth_mode: ""           # empty → fall back to TARGET_AZURE_OPENAI_AUTH_MODE env, then shared
-  target_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
-  target_azure_openai_managed_identity_client_id: ""
-
-  # MiniMax backend settings (minimax_chat target)
-  minimax_base_url: ""          # https://api.minimax.io/v1 if blank
-  minimax_api_key: ""
-  minimax_model: "MiniMax-M2.7"
-  minimax_temperature: "0.7"
-  minimax_max_tokens: "8000"
-  minimax_enable_thinking: "false"
-  optimizer_minimax_base_url: ""    # per-role override
-  target_minimax_base_url: ""       # per-role override
-  optimizer_minimax_api_key: ""
-  target_minimax_api_key: ""
-
-train:
-  num_epochs: 4
-  train_size: 0          # 0 = derive from dataset split when available
-  batch_size: 40
-  accumulation: 1
-  seed: 42
-
-gradient:
-  minibatch_size: 8
-  merge_batch_size: 8
-  analyst_workers: 16
-  max_analyst_rounds: 3
-  failure_only: false
-
-optimizer:
-  learning_rate: 4          # max edits per step (edit_budget)
-  min_learning_rate: 2      # min edits for decay schedulers
-  lr_scheduler: cosine      # constant / linear / cosine / autonomous
-  lr_control_mode: fixed    # fixed / autonomous / none
-  skill_update_mode: patch  # patch / rewrite_from_suggestions / full_rewrite_minibatch
-  use_slow_update: true
-  slow_update_samples: 20
-  slow_update_gate_with_selection: false
-  longitudinal_pair_policy: mixed  # mixed / changed / unchanged
-  use_meta_skill: true
-  use_skill_aware_reflection: false  # EmbodiSkill: split failures into SKILL_DEFECT (edit body) vs EXECUTION_LAPSE (protected appendix)
-  skill_aware_appendix_source: both  # both = success+failure emit appendix notes; failure_only = only EXECUTION_LAPSE (paper-faithful)
-  skill_aware_consolidate_threshold: 0  # 0 = off; >0 = LLM-consolidate the appendix when its note count exceeds N
-
-evaluation:
-  use_gate: true
-  sel_env_num: 0
-  test_env_num: 0
-  eval_test: true
-
-env:
-  name: ""
-  skill_init: ""
-  split_mode: ratio       # ratio = build deterministic split from data_path; split_dir = use pre-split train/val/test
-  split_seed: 42
-  split_dir: ""
-  data_path: ""
-  split_output_dir: ""
-  exec_timeout: 120      # per target model/code-agent call timeout in seconds
-  out_root: ""

package/template/skills/skillopt-sleep/configs/alfworld/default.yaml

@@ -1,29 +0,0 @@
-_base_: ../_base_/default.yaml
-
-train:
-  train_size: 0
-  accumulation: 1
-
-gradient:
-  minibatch_size: 8
-  merge_batch_size: 8
-
-optimizer:
-  learning_rate: 4
-
-evaluation:
-  sel_env_num: 0
-  test_env_num: 0
-
-env:
-  name: alfworld
-  skill_init: skillopt/envs/alfworld/skills/initial.md
-  split_mode: split_dir
-  split_dir: data/alfworld_path_split
-  data_path: ""
-  split_output_dir: ""
-  max_steps: 50
-  max_completion_tokens: 16384
-  workers: 8
-  max_api_workers: 8
-  limit: 0

package/template/skills/skillopt-sleep/configs/docvqa/default.yaml

@@ -1,28 +0,0 @@
-_base_: ../_base_/default.yaml
-
-model:
-  reasoning_effort: medium
-
-train:
-  batch_size: 40
-  accumulation: 1
-
-gradient:
-  minibatch_size: 8
-  merge_batch_size: 8
-
-optimizer:
-  learning_rate: 4
-
-env:
-  name: docvqa
-  skill_init: skillopt/envs/docvqa/skills/initial.md
-  split_mode: split_dir
-  split_dir: data/docvqa/splits
-  data_path: ""
-  split_output_dir: ""
-  max_turns: 1
-  max_completion_tokens: 16384
-  workers: 16
-  image_detail: auto
-  limit: 0

package/template/skills/skillopt-sleep/configs/features/soft_gate.yaml

@@ -1,47 +0,0 @@
-# ─────────────────────────────────────────────────────────────────────────────
-# Feature: soft / mixed validation-gate metric (community-contributed, PR #25)
-# ─────────────────────────────────────────────────────────────────────────────
-#
-# This is NOT a default SkillOpt setting and was NOT used to produce the
-# numbers reported in the paper. It is provided as a reference for users
-# who encounter a specific scenario where the default `hard` gate is too
-# coarse to drive training.
-#
-# When to consider this:
-#   - You are running on a custom environment.
-#   - Your held-out *selection* split has very few items (e.g. ≤ ~10).
-#   - Your reward function is continuous / partial-credit (e.g. F1, BLEU,
-#     soft match) rather than purely binary 0/1.
-#
-# Symptom this addresses:
-#   With a small selection split + continuous rewards, candidate skills
-#   often improve per-item soft scores (e.g. 0.06 → 0.26 on one item) but
-#   never flip the discrete hard outcome. The default `hard` gate then
-#   rejects every candidate and training stalls. Switching the gate to
-#   `soft` or `mixed` lets these partial improvements be accepted.
-#
-# When NOT to use this:
-#   - When reproducing the paper. The paper-reported numbers were obtained
-#     under the default `hard` gate.
-#   - When your selection split is large (dozens+ items) and / or your
-#     reward is already binary — `hard` is the more conservative choice
-#     and matches the design described in the paper.
-#
-# To use: inherit your env config from this file, e.g.
-#   _base_: ../features/soft_gate.yaml
-# or copy the `evaluation:` block below into your config.
-# ─────────────────────────────────────────────────────────────────────────────
-
-_base_: ../_base_/default.yaml
-
-evaluation:
-  # Three options:
-  #   'hard'  — default; exact-match accuracy. Use this to reproduce the paper.
-  #   'soft'  — per-item soft / partial-credit score (recommended for the
-  #             small-split + continuous-reward scenario described above).
-  #   'mixed' — weighted average: (1 - w) * hard + w * soft, with `w` set by
-  #             `gate_mixed_weight` below.
-  gate_metric: soft
-
-  # Only used when gate_metric == 'mixed'. Ignored otherwise.
-  gate_mixed_weight: 0.5

package/template/skills/skillopt-sleep/configs/livemathematicianbench/default.yaml

@@ -1,22 +0,0 @@
-_base_: ../_base_/default.yaml
-
-train:
-  train_size: 0
-  batch_size: 40
-  accumulation: 1
-
-env:
-  name: livemathematicianbench
-  skill_init: skillopt/envs/livemathematicianbench/skills/initial.md
-  split_mode: split_dir
-  split_dir: data/livemathematicianbench_split
-  data_path: ""
-  split_output_dir: ""
-  max_turns: 1
-  max_completion_tokens: 16384
-  exec_timeout: 300
-  workers: 64
-  limit: 0
-  shuffle_choices: true
-  use_theorem: false
-  use_sketch: false

package/template/skills/skillopt-sleep/configs/officeqa/default.yaml

@@ -1,34 +0,0 @@
-_base_: ../_base_/default.yaml
-
-model:
-  reasoning_effort: medium
-
-train:
-  batch_size: 40
-  accumulation: 1
-
-gradient:
-  minibatch_size: 8
-  merge_batch_size: 8
-
-optimizer:
-  learning_rate: 4
-
-env:
-  name: officeqa
-  skill_init: skillopt/envs/officeqa/skills/initial.md
-  split_mode: split_dir
-  split_dir: data/officeqa_split
-  data_dirs:
-    - data/officeqa_docs_official
-  workers: 4
-  max_tool_turns: 24
-  max_completion_tokens: 16384
-  search_mode: offline
-  max_queries_per_turn: 4
-  search_api_url: http://apisix.westus2.cloudapp.azure.com/search_tool/search
-  search_auth_env: OFFICEQA_CUSTOM_SEARCH_AUTH
-  search_provider: duckduckgo
-  search_max_num_results: 4
-  search_timeout_seconds: 20
-  limit: 0

package/template/skills/skillopt-sleep/configs/searchqa/default.yaml

@@ -1,32 +0,0 @@
-_base_: ../_base_/default.yaml
-
-model:
-  reasoning_effort: medium
-
-train:
-  train_size: 400
-  batch_size: 40
-  accumulation: 1
-
-gradient:
-  minibatch_size: 8
-  merge_batch_size: 8
-
-optimizer:
-  learning_rate: 4
-
-evaluation:
-  sel_env_num: 0
-  test_env_num: 0
-
-env:
-  name: searchqa
-  skill_init: skillopt/envs/searchqa/skills/initial.md
-  split_mode: split_dir
-  split_dir: data/searchqa_split
-  data_path: ""
-  split_output_dir: ""
-  max_turns: 1
-  max_completion_tokens: 16384
-  workers: 24
-  limit: 0

package/template/skills/skillopt-sleep/configs/spreadsheetbench/default.yaml

@@ -1,34 +0,0 @@
-_base_: ../_base_/default.yaml
-
-model:
-  reasoning_effort: medium
-
-train:
-  train_size: 80
-  batch_size: 40
-  accumulation: 1
-
-gradient:
-  minibatch_size: 8
-  merge_batch_size: 8
-
-optimizer:
-  learning_rate: 4
-
-evaluation:
-  sel_env_num: 0
-  test_env_num: 0
-
-env:
-  name: spreadsheetbench
-  skill_init: skillopt/envs/spreadsheetbench/skills/initial.md
-  split_mode: split_dir
-  split_dir: data/spreadsheetbench_split
-  data_path: ""
-  split_output_dir: ""
-  data_root: data/spreadsheetbench_verified_400
-  mode: multi
-  max_turns: 30
-  max_completion_tokens: 16384
-  exec_timeout: 600
-  workers: 24

package/template/skills/skillopt-sleep/scripts/framework/skillopt/__init__.py

@@ -1,28 +0,0 @@
-"""ReflACT: Reflective Agent Tuning.
-
-A general-purpose framework for iteratively optimizing LLM agent skills
-through structured reflection and self-improvement.
-
-Pipeline stages:
-  1. Rollout   — execute episodes with current skill
-  2. Reflect   — analyze trajectories, generate patches
-  3. Aggregate — hierarchical merge of patches
-  4. Select    — rank and select top edits
-  5. Update    — apply edits to skill document
-  6. Evaluate  — validate candidate skill, accept/reject
-"""
-
-__version__ = "0.1.0"
-
-from skillopt.types import (  # noqa: F401
-    BatchSpec,
-    Edit,
-    EditOp,
-    FailureSummaryEntry,
-    GateAction,
-    GateResult,
-    Patch,
-    RawPatch,
-    RolloutResult,
-    SlowUpdateResult,
-)