self-evolve-framework 1.6.0 → 1.6.1

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/alfworld/rollout.py

@@ -1,366 +0,0 @@
-"""ALFWorld rollout module for ReflACT.
-
-Provides:
-  - build_alfworld_env(): build ALFWorld environment (wraps vendored SkillRL env)
-  - run_alfworld_batch(): run a batch of ALFWorld episodes in parallel
-  - TASKS: list of ALFWorld task types
-"""
-from __future__ import annotations
-
-import concurrent.futures
-import json
-import os
-import re
-
-from skillopt.model import chat_target
-
-# ── Constants ─────────────────────────────────────────────────────────────────
-
-TASKS = [
-    "pick_and_place",
-    "pick_two_obj_and_place",
-    "look_at_obj_in_light",
-    "pick_heat_then_place_in_recep",
-    "pick_cool_then_place_in_recep",
-    "pick_clean_then_place_in_recep",
-]
-
-# ── Helpers ───────────────────────────────────────────────────────────────────
-
-
-def _get_task_type(gamefile: str) -> str:
-    for task in TASKS:
-        if task in gamefile:
-            return task
-    return "other"
-
-
-def _extract_action(model_response: str) -> str | None:
-    match = re.search(r"<action>(.*?)</action>", model_response, re.DOTALL)
-    return match.group(1).strip() if match else None
-
-
-def _extract_think(model_response: str) -> str | None:
-    match = re.search(r"<think>(.*?)</think>", model_response, re.DOTALL)
-    return match.group(1).strip() if match else None
-
-
-def _build_skill_prompt(skill_content: str) -> str:
-    """Build the skill section to inject into the agent's system prompt."""
-    if not skill_content or not skill_content.strip():
-        return ""
-    return (
-        "\n\n## Skill Knowledge\n"
-        "Below is a skill document with learned strategies. "
-        "Use these guidelines to inform your decisions:\n\n"
-        f"{skill_content}\n"
-    )
-
-
-def _append_diagnostic_instruction(prompt: str, diagnostic_instruction: str) -> str:
-    if not diagnostic_instruction or not diagnostic_instruction.strip():
-        return prompt
-    return f"{prompt}\n\n## Training Readout\n{diagnostic_instruction.strip()}\n"
-
-
-def _resolve_alfworld_gamefile(gamefile: str) -> str:
-    path = os.path.expanduser(os.path.expandvars(str(gamefile)))
-    if os.path.isabs(path):
-        return path
-
-    data_root = os.environ.get("ALFWORLD_DATA", "").strip()
-    if not data_root:
-        return path
-
-    root = os.path.expanduser(os.path.expandvars(data_root))
-    return os.path.abspath(os.path.join(root, path))
-
-
-def _resolve_alfworld_gamefiles(gamefiles: list[str] | None) -> list[str] | None:
-    if gamefiles is None:
-        return None
-    return [_resolve_alfworld_gamefile(gamefile) for gamefile in gamefiles]
-
-
-# ── Environment builder ──────────────────────────────────────────────────────
-
-
-def build_alfworld_env(
-    env_num: int,
-    eval_dataset: str = "eval_out_of_distribution",
-    seed: int = 42,
-    is_train: bool = False,
-    specific_gamefiles: list[str] | None = None,
-):
-    """Build ALFWorld environment manager.
-
-    Args:
-        env_num: number of parallel environments
-        eval_dataset: 'eval_in_distribution' or 'eval_out_of_distribution' or train
-        seed: random seed
-        is_train: whether to use training set
-
-    Returns:
-        env_manager: AlfWorldEnvironmentManager instance
-    """
-    from functools import partial
-
-    from omegaconf import OmegaConf
-
-    from skillopt.envs.alfworld.vendor.alfworld_envs import build_alfworld_envs
-    from skillopt.envs.alfworld.vendor.alfworld_projection import alfworld_projection
-    from skillopt.envs.alfworld.vendor.env_manager import AlfWorldEnvironmentManager
-
-    HERE = os.path.dirname(os.path.abspath(__file__))
-
-    alf_config_path = os.path.join(HERE, "vendor", "config_tw.yaml")
-    env_kwargs = {"eval_dataset": eval_dataset}
-    resolved_gamefiles = _resolve_alfworld_gamefiles(specific_gamefiles)
-
-    envs = build_alfworld_envs(
-        alf_config_path,
-        seed=seed,
-        env_num=env_num,
-        group_n=1,
-        is_train=is_train,
-        env_kwargs=env_kwargs,
-        resources_per_worker=None,
-        gamefiles=resolved_gamefiles,
-    )
-
-    config = OmegaConf.create(
-        {
-            "env": {
-                "history_length": 2,
-                "env_name": "alfworld/AlfredTWEnv",
-            }
-        }
-    )
-
-    projection_f = partial(alfworld_projection)
-    env_manager = AlfWorldEnvironmentManager(envs, projection_f, config)
-    return env_manager
-
-
-# ── Batch rollout ─────────────────────────────────────────────────────────────
-
-
-def run_alfworld_batch(
-    env_manager,
-    skill_content: str,
-    max_steps: int = 50,
-    out_root: str = "",
-    max_api_workers: int = 8,
-    temperature: float = 0.4,
-    max_completion_tokens: int = 16384,
-    diagnostic_mode: bool = False,
-    diagnostic_instruction: str = "",
-    result_ids: list[str] | None = None,
-) -> list[dict]:
-    """Run a batch of ALFWorld episodes.
-
-    Returns a list of result dicts compatible with SkillOpt pipeline:
-    [
-        {
-            "id": "<env_idx>_<gamefile_hash>",
-            "hard": 0 or 1,
-            "soft": 0.0 or 1.0,
-            "n_turns": <int>,
-            "fail_reason": "<str>",
-            "agent_ok": True,
-            "task_type": "<str>",
-            "gamefile": "<str>",
-            "task_description": "<str>",
-        },
-        ...
-    ]
-
-    Also saves conversation.json per environment in out_root/predictions/<task_id>/
-    """
-    skill_prompt = _build_skill_prompt(skill_content)
-
-    obs, infos = env_manager.reset({})
-    env_num = len(obs["text"])
-    env_dones = [False] * env_num
-    overall_success = [False] * env_num
-
-    # Build per-env metadata
-    env_meta: list[dict] = []
-    for i in range(env_num):
-        gamefile = infos[i].get("extra.gamefile", "") if isinstance(infos[i], dict) else ""
-        task_type = _get_task_type(gamefile)
-        # Extract task description from initial observation
-        task_desc = ""
-        anchor_text = obs["anchor"][i] if "anchor" in obs else ""
-        task_start = anchor_text.find("Your task is to: ")
-        if task_start != -1:
-            task_desc = anchor_text[task_start + len("Your task is to: "):].strip()
-
-        env_meta.append({
-            "gamefile": gamefile,
-            "task_type": task_type,
-            "task_description": task_desc,
-        })
-
-    # Per-env conversation records
-    conversations: list[list[dict]] = [[] for _ in range(env_num)]
-
-    for step_idx in range(max_steps):
-        if all(env_dones):
-            break
-
-        active_indices = [i for i in range(env_num) if not env_dones[i]]
-
-        # Build prompts with skill injection
-        prompts: dict[int, str] = {}
-        for i in active_indices:
-            prompt = obs["text"][i]
-            if skill_prompt:
-                # Inject skill before the action instruction
-                prompt = skill_prompt + "\n" + prompt
-            if diagnostic_mode and diagnostic_instruction.strip():
-                prompt = _append_diagnostic_instruction(prompt, diagnostic_instruction)
-            prompts[i] = prompt
-
-        # Call API in parallel
-        actions = ["None"] * env_num
-
-        def call_api(idx):
-            try:
-                response, _ = chat_target(
-                    system="You are an expert agent operating in the ALFRED Embodied Environment.",
-                    user=prompts[idx],
-                    max_completion_tokens=max_completion_tokens,
-                    retries=5,
-                    stage="rollout",
-                    timeout=None,
-                )
-                response = (response or "").strip()
-                if not response:
-                    return idx, "<think>empty model response</think><action>look</action>"
-                if _extract_action(response) is None:
-                    return idx, "<think>missing action tag</think><action>look</action>"
-                return idx, response
-            except Exception:
-                return idx, "<think>error</think><action>look</action>"
-
-        executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_api_workers)
-        try:
-            futures = {executor.submit(call_api, i): i for i in active_indices}
-            pending_futs = set(futures)
-            while pending_futs:
-                done, _ = concurrent.futures.wait(
-                    pending_futs,
-                    timeout=5,
-                    return_when=concurrent.futures.FIRST_COMPLETED,
-                )
-                for future in done:
-                    pending_futs.remove(future)
-                    try:
-                        idx, response = future.result()
-                    except Exception:  # noqa: BLE001
-                        idx = futures[future]
-                        response = "<think>error</think><action>look</action>"
-                    actions[idx] = response
-        finally:
-            executor.shutdown(wait=False, cancel_futures=True)
-
-        # Save model responses before stepping
-        model_responses = {i: actions[i] for i in active_indices}
-
-        # Step environment
-        obs, rewards, dones, infos = env_manager.step(actions)
-
-        # Record trajectory
-        for i in active_indices:
-            step_record = {
-                "step": step_idx,
-                "action": _extract_action(model_responses[i]),
-                "reasoning": _extract_think(model_responses[i]),
-                "model_response": model_responses[i],
-                "env_feedback": obs["anchor"][i] if "anchor" in obs else "",
-                "reward": float(rewards[i]),
-                "done": bool(dones[i]),
-            }
-            conversations[i].append(step_record)
-
-        # Update done status
-        for i in range(env_num):
-            if env_dones[i]:
-                continue
-            if dones[i]:
-                env_dones[i] = True
-                won = bool(infos[i].get("won", False))
-                overall_success[i] = won
-
-    # Build results and save conversations
-    results: list[dict] = []
-    pred_dir = os.path.join(out_root, "predictions") if out_root else ""
-
-    for i in range(env_num):
-        gamefile = env_meta[i]["gamefile"]
-        task_type = env_meta[i]["task_type"]
-        task_desc = env_meta[i]["task_description"]
-        n_turns = len(conversations[i])
-        won = overall_success[i]
-
-        # Generate stable task ID from env index and gamefile
-        task_id = str(result_ids[i]) if result_ids and i < len(result_ids) else f"env_{i:03d}"
-
-        fail_reason = ""
-        if not won:
-            if not env_dones[i]:
-                fail_reason = f"Timeout after {max_steps} steps"
-            else:
-                fail_reason = "Episode ended without completing the task"
-
-        result = {
-            "id": task_id,
-            "hard": 1 if won else 0,
-            "soft": 1.0 if won else 0.0,
-            "n_turns": n_turns,
-            "fail_reason": fail_reason,
-            "agent_ok": True,  # ALFWorld agent always runs OK (no crash)
-            "task_type": task_type,
-            "gamefile": gamefile,
-            "task_description": task_desc,
-            "instruction_type": task_type,  # for compatibility with v2 pipeline
-        }
-        results.append(result)
-
-        # Save conversation
-        if pred_dir:
-            conv_dir = os.path.join(pred_dir, task_id)
-            os.makedirs(conv_dir, exist_ok=True)
-            with open(os.path.join(conv_dir, "conversation.json"), "w") as f:
-                json.dump(conversations[i], f, ensure_ascii=False, indent=2)
-
-    return results
-
-
-# ── Item loading (for compatibility with split_three_way) ────────────────────
-
-
-def load_alfworld_items(
-    eval_dataset: str,
-    env_num: int,
-    seed: int = 42,
-    is_train: bool = False,
-) -> list[dict]:
-    """Create pseudo-item dicts for ALFWorld environments.
-
-    Since ALFWorld doesn't have a static JSON dataset like SpreadsheetBench,
-    we create lightweight item dicts that carry enough metadata for the pipeline.
-    The actual environment is built dynamically.
-
-    Returns:
-        List of dicts with "id" keys, one per environment slot.
-    """
-    items = []
-    for i in range(env_num):
-        items.append({
-            "id": f"env_{i:03d}",
-            "eval_dataset": eval_dataset,
-            "env_index": i,
-        })
-    return items

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/alfworld/skills/initial.md

@@ -1,45 +0,0 @@
-# ALFWorld Embodied Agent Skill
-
-## Overview
-This skill guides agents operating in the ALFWorld text-based embodied environment.
-The agent must complete household tasks by navigating rooms, interacting with objects,
-and using appliances. Actions must be chosen from the admissible action list provided
-at each step.
-
-**Output format**: Always output `<think>...</think>` for reasoning, then `<action>...</action>` for the chosen action.
-
----
-
-## Task Types
-
-| Type | Goal | Key Steps |
-|------|------|-----------|
-| Pick & Place | Put object X in/on receptacle Y | Find X -> take X -> go to Y -> put X in/on Y |
-| Pick Two & Place | Put two instances of X in/on Y | Find X1 -> take -> place -> find X2 -> take -> place |
-| Examine in Light | Examine object X under desklamp | Find X -> take X -> find desklamp -> use desklamp |
-| Clean & Place | Clean object X and put in/on Y | Find X -> take X -> go to sink -> clean X -> go to Y -> put X |
-| Heat & Place | Heat object X and put in/on Y | Find X -> take X -> go to microwave -> heat X -> go to Y -> put X |
-| Cool & Place | Cool object X and put in/on Y | Find X -> take X -> go to fridge -> cool X -> go to Y -> put X |
-
----
-
-## General Principles
-
-1. **Decompose the task**: Parse the goal into ordered sub-goals (locate, acquire, transform, deliver). Complete each before moving to the next.
-2. **Systematic exploration**: Search each surface and container exactly once before revisiting. Open closed containers (drawers, cabinets, fridge) before judging them empty.
-3. **Grab immediately**: When a required object is visible and reachable, take it right away before moving elsewhere.
-4. **Transform before placing**: If the task requires cleaning, heating, or cooling, perform the state change at the appropriate appliance before heading to the final destination.
-5. **Direct delivery**: Once holding the transformed (or untransformed) goal object, navigate straight to the target receptacle and place it.
-6. **Track progress**: Maintain an internal count of how many objects still need to be found and placed. Only stop searching when the count reaches zero.
-7. **Avoid loops**: Never repeat the same action more than twice in a row. If stuck, move to a different unexplored location.
-8. **Only choose admissible actions**: Always pick an action from the admissible action list. Do not invent actions.
-
----
-
-## Common Mistakes to Avoid
-
-- **Revisiting searched locations**: Keep track of which surfaces/containers have been checked; do not re-examine them.
-- **Ignoring visible objects**: If the target object appears in the observation, pick it up immediately.
-- **Skipping state changes**: Do not place an object at the destination without first cleaning/heating/cooling it when required.
-- **Premature termination**: Do not stop the episode until all goal conditions are verified as met.
-- **Action loops**: Repeatedly toggling or examining the same object wastes steps. Move on to new locations instead.

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/alfworld/vendor/__init__.py

@@ -1,9 +0,0 @@
-"""Vendored ALFWorld environment runtime.
-
-Minimal subset of SkillRL's agent_system package needed to run
-ALFWorld environments with ReflACT. Original source:
-https://github.com/NTU-LANTERN/SkillRL (Apache-2.0 License)
-"""
-from .alfworld_envs import AlfworldEnvs, build_alfworld_envs
-from .alfworld_projection import alfworld_projection
-from .env_manager import AlfWorldEnvironmentManager

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/alfworld/vendor/alfworld_envs.py

@@ -1,221 +0,0 @@
-# Vendored from SkillRL (Apache-2.0 License)
-# Original: agent_system/environments/env_package/alfworld/envs.py
-# Modified: imports use pip-installed alfworld package instead of vendored copy.
-
-import os
-import multiprocessing as mp
-import traceback
-import yaml
-import gymnasium as gym
-import numpy as np
-
-from alfworld.agents.environment import get_environment
-
-
-def load_config_file(path):
-    assert os.path.exists(path), f"Invalid config file: {path}"
-    with open(path) as reader:
-        config = yaml.safe_load(reader)
-    return config
-
-
-def compute_reward(info, multi_modal=False):
-    if multi_modal:
-        reward = 10.0 * float(info['won']) + float(info['goal_condition_success_rate'])
-    else:
-        reward = 10.0 * float(info['won'])
-    return reward
-
-
-class AlfworldWorker:
-    """Stateful worker that holds one ALFWorld sub-environment."""
-
-    def __init__(self, config, seed, base_env, gamefile=None):
-        if gamefile:
-            base_env.game_files = [gamefile]
-            if hasattr(base_env, "num_games"):
-                base_env.num_games = 1
-        self.env = base_env.init_env(batch_size=1)
-        self.env.seed(seed)
-
-    def step(self, action):
-        actions = [action]
-        obs, scores, dones, infos = self.env.step(actions)
-        infos['observation_text'] = obs
-        return obs, scores, dones, infos
-
-    def reset(self):
-        obs, infos = self.env.reset()
-        infos['observation_text'] = obs
-        return obs, infos
-
-
-def _worker_loop(cmd_q, result_q, config, seed, is_train, eval_dataset, gamefile):
-    """Run one ALFWorld environment in a child process."""
-    try:
-        env_type = config['env']['type']
-        base_env = get_environment(env_type)(
-            config,
-            train_eval='train' if is_train else eval_dataset,
-        )
-        worker = AlfworldWorker(config, seed, base_env, gamefile)
-        result_q.put((True, "ready"))
-    except BaseException:
-        result_q.put((False, traceback.format_exc()))
-        return
-
-    while True:
-        cmd, payload = cmd_q.get()
-        if cmd == "close":
-            result_q.put((True, None))
-            return
-        try:
-            if cmd == "reset":
-                result = worker.reset()
-            elif cmd == "step":
-                result = worker.step(payload)
-            else:
-                raise ValueError(f"Unknown ALFWorld worker command: {cmd}")
-            result_q.put((True, result))
-        except BaseException:
-            result_q.put((False, traceback.format_exc()))
-
-
-class _ProcessWorker:
-    """Small stdlib actor wrapper for one environment process."""
-
-    def __init__(self, ctx, config, seed, is_train, eval_dataset, gamefile=None):
-        self.cmd_q = ctx.Queue(maxsize=1)
-        self.result_q = ctx.Queue(maxsize=1)
-        self.process = ctx.Process(
-            target=_worker_loop,
-            args=(self.cmd_q, self.result_q, config, seed, is_train, eval_dataset, gamefile),
-        )
-        self.process.start()
-        ok, payload = self.result_q.get()
-        if not ok:
-            self.close(kill=True)
-            raise RuntimeError(f"Failed to start ALFWorld worker:\n{payload}")
-
-    def send(self, cmd, payload=None):
-        self.cmd_q.put((cmd, payload))
-
-    def recv(self):
-        ok, payload = self.result_q.get()
-        if not ok:
-            raise RuntimeError(f"ALFWorld worker failed:\n{payload}")
-        return payload
-
-    def close(self, kill=False):
-        if self.process.is_alive() and not kill:
-            try:
-                self.send("close")
-                self.recv()
-            except Exception:
-                kill = True
-        if kill and self.process.is_alive():
-            self.process.terminate()
-        self.process.join(timeout=5)
-        if self.process.is_alive():
-            self.process.kill()
-            self.process.join(timeout=1)
-        self.cmd_q.close()
-        self.result_q.close()
-
-
-class AlfworldEnvs(gym.Env):
-    """Vectorized ALFWorld environment using local process workers."""
-
-    def __init__(self, alf_config_path, seed, env_num, group_n,
-                 resources_per_worker, is_train=True, env_kwargs=None, gamefiles=None):
-        super().__init__()
-        if env_kwargs is None:
-            env_kwargs = {}
-
-        eval_dataset = env_kwargs.get('eval_dataset', 'eval_in_distribution')
-        config = load_config_file(alf_config_path)
-        env_type = config['env']['type']
-        self.multi_modal = (env_type == 'AlfredThorEnv')
-        self.num_processes = env_num * group_n
-        self.group_n = group_n
-        self.gamefiles = list(gamefiles or [])
-        if self.gamefiles and len(self.gamefiles) != self.num_processes:
-            raise ValueError(
-                f"Expected {self.num_processes} gamefiles, got {len(self.gamefiles)}"
-            )
-
-        start_method = os.environ.get("ALFWORLD_WORKER_START_METHOD") or None
-        ctx = mp.get_context(start_method) if start_method else mp.get_context()
-        self.workers = []
-        for i in range(self.num_processes):
-            worker_gamefile = self.gamefiles[i] if self.gamefiles else None
-            worker = _ProcessWorker(
-                ctx,
-                config,
-                seed + (i // self.group_n),
-                is_train,
-                eval_dataset,
-                worker_gamefile,
-            )
-            self.workers.append(worker)
-
-        self.prev_admissible_commands = [None for _ in range(self.num_processes)]
-
-    def step(self, actions):
-        assert len(actions) == self.num_processes
-
-        for i, worker in enumerate(self.workers):
-            worker.send("step", actions[i])
-        results = [worker.recv() for worker in self.workers]
-
-        text_obs_list = []
-        rewards_list = []
-        dones_list = []
-        info_list = []
-
-        for i, (obs, scores, dones, info) in enumerate(results):
-            for k in info.keys():
-                info[k] = info[k][0]
-            text_obs_list.append(obs[0])
-            dones_list.append(dones[0])
-            info_list.append(info)
-            self.prev_admissible_commands[i] = info['admissible_commands']
-            rewards_list.append(compute_reward(info, self.multi_modal))
-
-        image_obs_list = None
-        return text_obs_list, image_obs_list, rewards_list, dones_list, info_list
-
-    def reset(self):
-        for worker in self.workers:
-            worker.send("reset")
-        results = [worker.recv() for worker in self.workers]
-
-        text_obs_list = []
-        info_list = []
-
-        for i, (obs, info) in enumerate(results):
-            for k in info.keys():
-                info[k] = info[k][0]
-            text_obs_list.append(obs[0])
-            self.prev_admissible_commands[i] = info['admissible_commands']
-            info_list.append(info)
-
-        image_obs_list = None
-        return text_obs_list, image_obs_list, info_list
-
-    @property
-    def get_admissible_commands(self):
-        return self.prev_admissible_commands
-
-    def close(self):
-        for worker in self.workers:
-            worker.close()
-
-
-def build_alfworld_envs(alf_config_path, seed, env_num, group_n,
-                        resources_per_worker, is_train=True, env_kwargs=None, gamefiles=None):
-    """Build vectorized ALFWorld environments."""
-    return AlfworldEnvs(
-        alf_config_path, seed, env_num, group_n,
-        resources_per_worker, is_train, env_kwargs, gamefiles,
-    )