starforge-cli 0.1.6__py3-none-any.whl

starforge_cli/recipe_lock.py

@@ -0,0 +1,453 @@
+"""实验 recipe 锁文件（recipe.lock.json）的生成、检查与显式升级。
+
+对外只暴露三个稳定动作：inspect / upgrade / require_current。
+锁住的是当前 catalog 的 recipe bundle 与精确 runtime，不是创建它的 SDK 版本。
+"""
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Any, Callable, Iterable
+
+from starforge_core import __version__ as CORE_VERSION
+from starforge_core.contract import SpecError
+from starforge_core.recipes import Recipe, get_recipe
+
+LOCK_FILE = "recipe.lock.json"
+LOCK_VERSION = "forge/recipe-lock/v3"
+_LEGACY_LOCK_VERSION = "forge/recipe-lock/v2"
+_SUPPORTED_LOCK_VERSIONS = (LOCK_VERSION, _LEGACY_LOCK_VERSION)
+_EXPERIMENT_KINDS = ("experiments", "projects", "smoke")
+
+
+class LockState(str, Enum):
+    CURRENT = "current"
+    CORE_INCOMPATIBLE = "sdk_incompatible"
+    RECIPE_STALE = "recipe_stale"
+    RUNTIME_REMOVED = "runtime_removed"
+    MALFORMED = "malformed"
+
+
+@dataclass(frozen=True)
+class LockDiff:
+    field: str
+    locked: Any
+    current: Any
+
+
+@dataclass(frozen=True)
+class LockInspection:
+    state: LockState
+    path: Path
+    recipe_name: str
+    framework_version: str
+    payload: dict[str, Any] | None
+    expected: dict[str, Any] | None
+    diffs: tuple[LockDiff, ...]
+    message: str
+
+    @property
+    def is_current(self) -> bool:
+        return self.state is LockState.CURRENT
+
+
+class RecipeLockError(ValueError):
+    def __init__(self, inspection: LockInspection):
+        super().__init__(inspection.message)
+        self.inspection = inspection
+
+
+class RecipeLockManager:
+    """锁文件的唯一入口：解析、分类、升级、提交前校验。"""
+
+    def inspect(self, exp_dir: Path, recipe_name: str = "") -> LockInspection:
+        path = Path(exp_dir) / LOCK_FILE
+        if not path.is_file():
+            return _malformed(
+                path,
+                recipe_name,
+                f"实验缺少 {LOCK_FILE}；用 `sf new` 重建，或 `sf recipe upgrade` 生成",
+            )
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            return _malformed(path, recipe_name, f"{path} 非法: {exc}")
+        if not isinstance(payload, dict):
+            return _malformed(path, recipe_name, f"{path} 根节点必须是对象")
+
+        normalized = _normalize_lock(payload)
+        if normalized is None:
+            return _malformed(
+                path,
+                recipe_name,
+                f"{path} 不是支持的 recipe lock（{_LEGACY_LOCK_VERSION} / {LOCK_VERSION}）",
+            )
+        locked_name = recipe_name.strip() or normalized["recipe_name"]
+        if not locked_name:
+            return _malformed(path, "", f"{path} 缺少 recipe.name")
+        if recipe_name.strip() and normalized["recipe_name"] not in {"", locked_name}:
+            return _malformed(
+                path,
+                locked_name,
+                f"{path} 锁定的是 {normalized['recipe_name']!r}，与请求的 {locked_name!r} 不一致",
+            )
+        try:
+            recipe = get_recipe(locked_name)
+        except SpecError as exc:
+            return _malformed(path, locked_name, str(exc))
+
+        sdk_reason = recipe.core_incompatibility()
+        if sdk_reason:
+            return LockInspection(
+                state=LockState.CORE_INCOMPATIBLE,
+                path=path,
+                recipe_name=locked_name,
+                framework_version=normalized["framework_version"],
+                payload=payload,
+                expected=None,
+                diffs=(),
+                message=f"{path}: {sdk_reason}",
+            )
+
+        locked_runtime = normalized["framework_version"]
+        if locked_runtime and locked_runtime not in recipe.runtime.supported_versions:
+            expected = recipe_lock(locked_name, recipe.runtime.default_version)
+            return LockInspection(
+                state=LockState.RUNTIME_REMOVED,
+                path=path,
+                recipe_name=locked_name,
+                framework_version=locked_runtime,
+                payload=payload,
+                expected=expected,
+                diffs=_diffs(normalized, expected),
+                message=(
+                    f"{path} 锁定的 {recipe.framework}@{locked_runtime} 已从 catalog 移除；"
+                    f"可用: {', '.join(recipe.runtime.supported_versions)}。"
+                    f"执行 `sf recipe upgrade {exp_dir.name} --accept-runtime-change`"
+                    " 或指定 --framework-version"
+                ),
+            )
+
+        expected = recipe_lock(locked_name, locked_runtime)
+        diffs = _diffs(normalized, expected)
+        if diffs:
+            return LockInspection(
+                state=LockState.RECIPE_STALE,
+                path=path,
+                recipe_name=locked_name,
+                framework_version=locked_runtime or expected["framework"]["version"],
+                payload=payload,
+                expected=expected,
+                diffs=diffs,
+                message=(
+                    f"{path} 与当前 core recipe bundle 不一致。"
+                    f"执行 `sf recipe upgrade {exp_dir.name}` 后重新提交"
+                ),
+            )
+        return LockInspection(
+            state=LockState.CURRENT,
+            path=path,
+            recipe_name=locked_name,
+            framework_version=expected["framework"]["version"],
+            payload=payload,
+            expected=expected,
+            diffs=(),
+            message=f"{path} 已锁定当前 recipe bundle",
+        )
+
+    def require_current(self, exp_dir: Path, recipe_name: str = "") -> str:
+        inspection = self.inspect(exp_dir, recipe_name)
+        if not inspection.is_current:
+            raise RecipeLockError(inspection)
+        return inspection.framework_version
+
+    def write_current(
+        self,
+        exp_dir: Path,
+        recipe_name: str,
+        framework_version: str = "",
+    ) -> dict[str, Any]:
+        payload = recipe_lock(recipe_name, framework_version)
+        _atomic_write(Path(exp_dir) / LOCK_FILE, payload)
+        return payload
+
+    def upgrade(
+        self,
+        exp_dir: Path,
+        *,
+        recipe_name: str = "",
+        framework_version: str = "",
+        accept_runtime_change: bool = False,
+        dry_run: bool = False,
+        validate_config: Callable[[Path, Recipe], list[str]] | None = None,
+    ) -> LockInspection:
+        inspection = self.inspect(exp_dir, recipe_name)
+        if inspection.state is LockState.MALFORMED:
+            raise RecipeLockError(inspection)
+        if inspection.state is LockState.CORE_INCOMPATIBLE:
+            raise RecipeLockError(inspection)
+
+        selected = framework_version.strip()
+        if inspection.state is LockState.RUNTIME_REMOVED:
+            if not selected and not accept_runtime_change:
+                raise RecipeLockError(inspection)
+            selected = selected or get_recipe(inspection.recipe_name).runtime.default_version
+        elif not selected:
+            selected = inspection.framework_version
+
+        recipe = get_recipe(inspection.recipe_name)
+        try:
+            recipe.runtime.resolve(selected)
+        except SpecError as exc:
+            raise RecipeLockError(
+                LockInspection(
+                    state=LockState.RUNTIME_REMOVED,
+                    path=inspection.path,
+                    recipe_name=inspection.recipe_name,
+                    framework_version=selected,
+                    payload=inspection.payload,
+                    expected=recipe_lock(inspection.recipe_name, recipe.runtime.default_version),
+                    diffs=inspection.diffs,
+                    message=str(exc),
+                )
+            ) from exc
+
+        if validate_config is not None:
+            errors = validate_config(Path(exp_dir), recipe)
+            if errors:
+                raise RecipeLockError(
+                    LockInspection(
+                        state=LockState.RECIPE_STALE,
+                        path=inspection.path,
+                        recipe_name=inspection.recipe_name,
+                        framework_version=selected,
+                        payload=inspection.payload,
+                        expected=recipe_lock(inspection.recipe_name, selected),
+                        diffs=inspection.diffs,
+                        message=(
+                            f"{inspection.path} 升级后 config 不兼容: "
+                            + "；".join(errors)
+                        ),
+                    )
+                )
+
+        expected = recipe_lock(inspection.recipe_name, selected)
+        if not dry_run:
+            _atomic_write(inspection.path, expected)
+        return LockInspection(
+            state=LockState.CURRENT,
+            path=inspection.path,
+            recipe_name=inspection.recipe_name,
+            framework_version=selected,
+            payload=expected if not dry_run else inspection.payload,
+            expected=expected,
+            diffs=_diffs(_normalize_lock(inspection.payload or {}) or {}, expected),
+            message=(
+                f"{inspection.path} 将升级到当前 recipe bundle"
+                if dry_run
+                else f"{inspection.path} 已升级到当前 recipe bundle"
+            ),
+        )
+
+    def upgrade_all(
+        self,
+        repo_root: Path,
+        *,
+        framework_version: str = "",
+        accept_runtime_change: bool = False,
+        dry_run: bool = False,
+        validate_config: Callable[[Path, Recipe], list[str]] | None = None,
+    ) -> list[LockInspection]:
+        targets = list(iter_lock_dirs(repo_root))
+        planned: list[tuple[Path, LockInspection]] = []
+        failures: list[LockInspection] = []
+        for exp_dir in targets:
+            inspection = self.inspect(exp_dir)
+            if inspection.is_current:
+                planned.append((exp_dir, inspection))
+                continue
+            try:
+                planned.append((
+                    exp_dir,
+                    self.upgrade(
+                        exp_dir,
+                        framework_version=framework_version,
+                        accept_runtime_change=accept_runtime_change,
+                        dry_run=True,
+                        validate_config=validate_config,
+                    ),
+                ))
+            except RecipeLockError as exc:
+                failures.append(exc.inspection)
+        if failures:
+            raise RecipeLockBatchError(failures)
+
+        if dry_run:
+            return [item for _, item in planned]
+
+        backups: list[tuple[Path, bytes | None]] = []
+        written: list[LockInspection] = []
+        try:
+            for exp_dir, inspection in planned:
+                if inspection.expected is None or inspection.is_current and inspection.payload == inspection.expected:
+                    written.append(inspection)
+                    continue
+                path = exp_dir / LOCK_FILE
+                previous = path.read_bytes() if path.is_file() else None
+                _atomic_write(path, inspection.expected)
+                backups.append((path, previous))
+                written.append(
+                    LockInspection(
+                        state=LockState.CURRENT,
+                        path=path,
+                        recipe_name=inspection.recipe_name,
+                        framework_version=inspection.expected["framework"]["version"],
+                        payload=inspection.expected,
+                        expected=inspection.expected,
+                        diffs=inspection.diffs,
+                        message=f"{path} 已升级到当前 recipe bundle",
+                    )
+                )
+        except Exception:
+            _rollback(backups)
+            raise
+        return written
+
+
+class RecipeLockBatchError(ValueError):
+    def __init__(self, failures: list[LockInspection]):
+        lines = [item.message for item in failures]
+        super().__init__("批量升级中止，未写入任何锁文件:\n" + "\n".join(f"- {line}" for line in lines))
+        self.failures = failures
+
+
+def recipe_lock(recipe_name: str, framework_version: str = "") -> dict[str, Any]:
+    recipe = get_recipe(recipe_name)
+    selected = framework_version.strip() or recipe.runtime.default_version
+    variant = recipe.runtime.resolve(selected)
+    return {
+        "apiVersion": LOCK_VERSION,
+        "recipe": {
+            "name": recipe.id,
+            "version": recipe.version,
+            "digest": recipe.digest,
+            "manifest_digest": recipe.manifest_digest,
+            "template_digest": recipe.template_digest,
+        },
+        "framework": {
+            "kind": recipe.framework,
+            "version": selected,
+            "runtime_id": variant.runtime_id,
+        },
+        "created_by": {"core_version": CORE_VERSION},
+        "requires": {"core": recipe.core_requires},
+    }
+
+
+def write_recipe_lock(exp_dir: Path, recipe_name: str, framework_version: str = "") -> None:
+    RecipeLockManager().write_current(exp_dir, recipe_name, framework_version)
+
+
+def validate_recipe_lock(exp_dir: Path, recipe_name: str) -> str:
+    """提交/校验入口：锁必须已经是当前 bundle，绝不静默改写。"""
+    return RecipeLockManager().require_current(exp_dir, recipe_name)
+
+
+def iter_lock_dirs(repo_root: Path) -> Iterable[Path]:
+    root = Path(repo_root)
+    for kind in _EXPERIMENT_KINDS:
+        base = root / kind
+        if not base.is_dir():
+            continue
+        for path in sorted(base.iterdir()):
+            if path.is_dir() and (path / LOCK_FILE).is_file():
+                yield path
+
+
+def _normalize_lock(payload: dict[str, Any]) -> dict[str, Any] | None:
+    api = str(payload.get("apiVersion") or "").strip()
+    recipe = payload.get("recipe") if isinstance(payload.get("recipe"), dict) else {}
+    if api == LOCK_VERSION:
+        framework = payload.get("framework") if isinstance(payload.get("framework"), dict) else {}
+        requires = payload.get("requires") if isinstance(payload.get("requires"), dict) else {}
+        return {
+            "apiVersion": api,
+            "recipe_name": str(recipe.get("name") or "").strip(),
+            "recipe_version": str(recipe.get("version") or "").strip(),
+            "digest": str(recipe.get("digest") or "").strip(),
+            "manifest_digest": str(recipe.get("manifest_digest") or "").strip(),
+            "template_digest": str(recipe.get("template_digest") or "").strip(),
+            "framework_kind": str(framework.get("kind") or "").strip(),
+            "framework_version": str(framework.get("version") or "").strip(),
+            "runtime_id": str(framework.get("runtime_id") or "").strip(),
+            "core_requires": str(requires.get("core") or "").strip(),
+        }
+    if api == _LEGACY_LOCK_VERSION:
+        return {
+            "apiVersion": api,
+            "recipe_name": str(recipe.get("name") or "").strip(),
+            "recipe_version": str(recipe.get("version") or "").strip(),
+            "digest": str(recipe.get("digest") or "").strip(),
+            "manifest_digest": "",
+            "template_digest": "",
+            "framework_kind": str(recipe.get("framework") or "").strip(),
+            "framework_version": str(recipe.get("framework_version") or "").strip(),
+            "runtime_id": "",
+            "core_requires": "",
+        }
+    return None
+
+
+def _diffs(normalized: dict[str, Any], expected: dict[str, Any]) -> tuple[LockDiff, ...]:
+    current = _normalize_lock(expected) or {}
+    fields = (
+        ("apiVersion", "apiVersion"),
+        ("recipe.name", "recipe_name"),
+        ("recipe.version", "recipe_version"),
+        ("recipe.digest", "digest"),
+        ("recipe.manifest_digest", "manifest_digest"),
+        ("recipe.template_digest", "template_digest"),
+        ("framework.kind", "framework_kind"),
+        ("framework.version", "framework_version"),
+        ("framework.runtime_id", "runtime_id"),
+        ("requires.core", "core_requires"),
+    )
+    out: list[LockDiff] = []
+    for label, key in fields:
+        locked = normalized.get(key, "")
+        wanted = current.get(key, "")
+        if locked != wanted:
+            out.append(LockDiff(field=label, locked=locked, current=wanted))
+    return tuple(out)
+
+
+def _malformed(path: Path, recipe_name: str, message: str) -> LockInspection:
+    return LockInspection(
+        state=LockState.MALFORMED,
+        path=path,
+        recipe_name=recipe_name,
+        framework_version="",
+        payload=None,
+        expected=None,
+        diffs=(),
+        message=message,
+    )
+
+
+def _atomic_write(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data = json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
+    tmp = path.with_name(path.name + ".tmp")
+    tmp.write_text(data, encoding="utf-8")
+    os.replace(tmp, path)
+
+
+def _rollback(backups: list[tuple[Path, bytes | None]]) -> None:
+    for path, previous in reversed(backups):
+        if previous is None:
+            path.unlink(missing_ok=True)
+        else:
+            path.write_bytes(previous)

starforge_cli/scaffold/agent-run.py.tmpl

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# 多轮 Agent（GRPO）训练骨架（NeMo-RL 0.6.0）。
+# agent 本质就是 GRPO + max_rollout_turns>1 + 一个自定义环境；入口由所选 recipe 明确声明。
+#
+# 这是一个【可运行的最小骨架】：默认指向 NeMo-RL 官方 examples/run_grpo_sliding_puzzle.py 的范式，
+# 你要做的是把下面两个 TODO 换成自己的「环境 + 数据集」：
+#   · 完整实战参考（自定义多工具环境 + 随机出题）：experiments/agent-grpo_qwen3.5-9b_multitool_v1/run.py
+#   · 官方范式（滑块拼图）：<NEMO_RL_DIR>/examples/run_grpo_sliding_puzzle.py
+# 环境写法见 common/environments/（实现 EnvironmentInterface），config 的 env.<task>.cfg 由本文件读取。
+import argparse
+import itertools
+import os
+import pprint
+import sys
+from typing import Any, Iterator
+
+from omegaconf import OmegaConf
+from torch.utils.data import IterableDataset
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+REPO_ROOT = os.path.abspath(os.path.join(THIS_DIR, "..", ".."))
+if REPO_ROOT not in sys.path:
+    sys.path.insert(0, REPO_ROOT)
+
+from nemo_rl.algorithms.grpo import MasterConfig, grpo_train, setup
+from nemo_rl.algorithms.utils import get_tokenizer, set_seed
+from nemo_rl.data.interfaces import DatumSpec, LLMMessageLogType
+from nemo_rl.distributed.virtual_cluster import init_ray
+from nemo_rl.models.generation import configure_generation_config
+from nemo_rl.utils.config import (
+    load_config,
+    parse_hydra_overrides,
+    register_omegaconf_resolvers,
+)
+from nemo_rl.utils.logger import get_next_experiment_dir
+
+# TODO(1/2)：换成你的环境（实现 nemo_rl 的 EnvironmentInterface，放在 common/environments/）。
+# from common.environments.your_env import YourAgentEnv
+
+TASK_NAME = "your_task"           # 须与 config.yaml 的 env.<TASK_NAME>.cfg 对齐
+STOP_STRINGS = ["</tool>", "</answer>"]   # 按你的多轮协议设停止符
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="多轮 Agent GRPO 训练")
+    parser.add_argument("--config", type=str, default=None, help="YAML 配置路径")
+    args, overrides = parser.parse_known_args()
+    return args, overrides
+
+
+def generate_datum(tokenizer, env_cfg: dict[str, Any], idx: int) -> DatumSpec:
+    # TODO(2/2)：构造一条训练样本（题面 + 目标），metadata 会被你的环境读取用于判分/多轮控制。
+    question = "TODO: 在这里生成/取一条需要多轮工具调用的题目"
+    prompt_text = tokenizer.apply_chat_template(
+        [{"role": "user", "content": question}],
+        tokenize=False, add_generation_prompt=True, add_special_tokens=False,
+    ).strip()
+    token_ids = tokenizer(prompt_text, return_tensors="pt", add_special_tokens=False)["input_ids"][0]
+    message_log: LLMMessageLogType = [
+        {"role": "user", "content": prompt_text, "token_ids": token_ids}
+    ]
+    metadata = {
+        "num_turns": 0,
+        "max_turns": int(env_cfg.get("max_turns", 6)),
+        # ... 你的环境判分所需字段（如 target / kb / tolerance）放这里
+    }
+    return {
+        "message_log": message_log,
+        "length": len(token_ids),
+        "extra_env_info": metadata,
+        "loss_multiplier": 1.0,
+        "idx": idx,
+        "task_name": TASK_NAME,
+        "stop_strings": STOP_STRINGS,
+    }
+
+
+class IterableAgentDataset(IterableDataset):
+    def __init__(self, tokenizer, env_cfg, length):
+        super().__init__()
+        self.tokenizer, self.env_cfg, self.length = tokenizer, env_cfg, length
+
+    def __iter__(self) -> Iterator[DatumSpec]:
+        for i in itertools.count():
+            yield generate_datum(self.tokenizer, self.env_cfg, i)
+
+    def __len__(self):
+        return self.length
+
+
+def main():
+    register_omegaconf_resolvers()
+    args, overrides = parse_args()
+    if not args.config:
+        args.config = os.path.join(THIS_DIR, "config.yaml")
+
+    config = load_config(args.config)
+    print(f"已加载配置: {args.config}")
+    if overrides:
+        print(f"CLI overrides: {overrides}")
+        config = parse_hydra_overrides(config, overrides)
+    config = OmegaConf.to_container(config, resolve=True)
+    config: MasterConfig = MasterConfig(**config)
+    print("最终配置：")
+    pprint.pprint(config)
+
+    config.logger["log_dir"] = get_next_experiment_dir(config.logger["log_dir"])
+    print(f"日志目录: {config.logger['log_dir']}")
+
+    init_ray()
+    set_seed(config.grpo["seed"])
+
+    tokenizer = get_tokenizer(config.policy["tokenizer"])
+    config.policy["generation"] = configure_generation_config(
+        config.policy["generation"], tokenizer
+    )
+
+    env_cfg = config.env[TASK_NAME]["cfg"]
+    # TODO：实例化你的环境（CPU actor：num_gpus=0）。
+    # env = YourAgentEnv.options(num_gpus=0).remote(cfg=dict(env_cfg))
+    raise SystemExit(
+        "agent 骨架未完成：请实现 TASK_NAME 对应的环境与 generate_datum，再取消下方训练调用的注释。\n"
+        "参考 experiments/agent-grpo_qwen3.5-9b_multitool_v1/run.py。"
+    )
+    task_to_env = {TASK_NAME: env}  # noqa: F821  (实现 env 后取消上方 raise)
+
+    ds_length = (
+        config.grpo["num_prompts_per_step"]
+        * config.grpo["num_generations_per_prompt"]
+        * config.grpo["max_num_steps"]
+    )
+    dataset = IterableAgentDataset(tokenizer, env_cfg, ds_length)
+    val_dataset = IterableAgentDataset(tokenizer, env_cfg, config.grpo["max_val_samples"])
+
+    (policy, policy_generation, cluster, dataloader, val_dataloader,
+     loss_fn, logger, checkpointer, grpo_state, master_config) = setup(
+        config, tokenizer, dataset, val_dataset
+    )
+    grpo_train(
+        policy, policy_generation, dataloader, val_dataloader, tokenizer, loss_fn,
+        task_to_env, task_to_env, logger, checkpointer, grpo_state, master_config,
+    )
+
+
+if __name__ == "__main__":
+    main()

starforge_cli/scaffold/custom-framework/train.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# 显式 custom recipe 的训练脚本。
+#
+# 由 Job Capsule runner 内的 CustomAdapter 直接执行。框架来自版本化 recipe，不读取
+# FRAMEWORK 环境变量或 framework 文件，也不会由其他 adapter 失败后回退至此。
+#
+# ── 已经给你准备好的环境变量（契约，别改名）─────────────────────────────────────
+#   FORGE_OUT_DIR                 产物目录（已 mkdir）。checkpoint / 日志请写这里，
+#                               它已按 <用户>/<实验>/<run_id> 隔离好，不会和别人互相覆盖
+#   FORGE_EXP_DIR                 本实验目录（就是这个文件所在目录）
+#   FORGE_WORK_DIR                上传包根目录
+#   FORGE_FRAMEWORK / FORGE_RECIPE  固定为 custom / custom
+#   FORGE_CLUSTER_NUM_NODES       ★服务端权威拓扑：你实际能用几个节点
+#   FORGE_CLUSTER_GPUS_PER_NODE   ★服务端权威拓扑：每节点几张卡
+#   STARFORGE_ENDPOINT/RUN_ID/TOKEN   指标上报凭据（无则为本地直跑，上报自动 no-op）
+#   HF_HOME / HF_TOKEN / 各 *_DATA_DIR   由服务端按需注入
+#
+# ── 三件必须做的事 ─────────────────────────────────────────────────────────────
+#  ① checkpoint 写进 $FORGE_OUT_DIR —— 写别处的话作业结束就随临时目录一起没了
+#  ② 用 common/observability/report.py 上报指标 —— 否则 console 上只有日志、没有曲线：
+#        from common.observability.report import StarForgeCallback
+#        trainer = SFTTrainer(..., callbacks=[StarForgeCallback()])
+#     手写循环则用 report.init() / report.log({...}, step=i) / report.finish()
+#  ③ 遵守 FORGE_CLUSTER_* 拓扑 —— 配额按它记账，watchdog 会做集群级 Ray 用卡对账，
+#     实际占卡超出记账会被告警甚至停止作业
+#
+# ⚠️ 训练依赖必须烘焙进本作业引用的 digest runtime image。需要 TRL / flash-attn
+#    等依赖时发布新的 runtime artifact；新作业自动采用，无需重启长驻集群。
+set -euo pipefail
+
+: "${FORGE_WORK_DIR:?FORGE_WORK_DIR is required}"
+: "${FORGE_EXP_DIR:?FORGE_EXP_DIR is required}"
+: "${FORGE_OUT_DIR:?FORGE_OUT_DIR is required}"
+: "${FORGE_FRAMEWORK:?FORGE_FRAMEWORK is required}"
+: "${FORGE_RECIPE:?FORGE_RECIPE is required}"
+: "${FORGE_CLUSTER_NUM_NODES:?FORGE_CLUSTER_NUM_NODES is required}"
+: "${FORGE_CLUSTER_GPUS_PER_NODE:?FORGE_CLUSTER_GPUS_PER_NODE is required}"
+
+echo "[train] out_dir  : ${FORGE_OUT_DIR}"
+echo "[train] topology : ${FORGE_CLUSTER_NUM_NODES:-?} 节点 × ${FORGE_CLUSTER_GPUS_PER_NODE:-?} 卡"
+
+# ── TODO: 换成你自己的启动命令 ────────────────────────────────────────────────
+# 示例（HF accelerate，单节点多卡）：
+#
+#   exec accelerate launch \
+#       --num_processes "${FORGE_CLUSTER_GPUS_PER_NODE:-1}" \
+#       "${FORGE_EXP_DIR}/train.py" \
+#       --output_dir "${FORGE_OUT_DIR}" \
+#       --config "${FORGE_EXP_DIR}/config.yaml"
+#
+# 示例（直接跑一个 python 脚本）：
+#
+#   exec python "${FORGE_EXP_DIR}/train.py" --output-dir "${FORGE_OUT_DIR}"
+
+echo "[train] 还没填训练命令：请编辑 ${FORGE_EXP_DIR}/train.sh" >&2
+exit 1