starforge-cli 0.1.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. starforge_cli/__init__.py +3 -0
  2. starforge_cli/api_client.py +589 -0
  3. starforge_cli/auth.py +349 -0
  4. starforge_cli/catalog.py +124 -0
  5. starforge_cli/cli.py +74 -0
  6. starforge_cli/cli_ui.py +469 -0
  7. starforge_cli/client_device.py +104 -0
  8. starforge_cli/commands/__init__.py +1 -0
  9. starforge_cli/commands/admin.py +140 -0
  10. starforge_cli/commands/bench.py +94 -0
  11. starforge_cli/commands/common.py +178 -0
  12. starforge_cli/commands/dataset.py +150 -0
  13. starforge_cli/commands/exp.py +213 -0
  14. starforge_cli/commands/init.py +52 -0
  15. starforge_cli/commands/jobs.py +223 -0
  16. starforge_cli/commands/login.py +54 -0
  17. starforge_cli/commands/plugin.py +243 -0
  18. starforge_cli/commands/recipe.py +163 -0
  19. starforge_cli/commands/serve.py +79 -0
  20. starforge_cli/commands/submit.py +467 -0
  21. starforge_cli/commands/sweep.py +154 -0
  22. starforge_cli/config_resolve.py +17 -0
  23. starforge_cli/data_prep.py +60 -0
  24. starforge_cli/new_experiment.py +195 -0
  25. starforge_cli/packing.py +179 -0
  26. starforge_cli/plugins_lock.py +73 -0
  27. starforge_cli/project.py +130 -0
  28. starforge_cli/recipe_lock.py +453 -0
  29. starforge_cli/scaffold/agent-run.py.tmpl +146 -0
  30. starforge_cli/scaffold/custom-framework/train.sh +56 -0
  31. starforge_cli/scaffold/experiment-template/.gitkeep +0 -0
  32. starforge_cli/scaffold/experiment-template/README.md +36 -0
  33. starforge_cli/scaffold/experiment-template/config.yaml +44 -0
  34. starforge_cli/scaffold/project/common/README.md +12 -0
  35. starforge_cli/scaffold/project/common/__init__.py +0 -0
  36. starforge_cli/scaffold/project/configs/README.md +103 -0
  37. starforge_cli/scaffold/project/configs/base/README.md +24 -0
  38. starforge_cli/scaffold/project/configs/base/distillation_math.yaml +284 -0
  39. starforge_cli/scaffold/project/configs/base/grpo_lora.yaml +30 -0
  40. starforge_cli/scaffold/project/configs/base/grpo_math_1B.yaml +470 -0
  41. starforge_cli/scaffold/project/configs/base/grpo_megatron.yaml +43 -0
  42. starforge_cli/scaffold/project/configs/base/grpo_noncolocated.yaml +18 -0
  43. starforge_cli/scaffold/project/configs/base/grpo_sliding_puzzle.yaml +81 -0
  44. starforge_cli/scaffold/project/configs/base/ppo_math_1B.yaml +454 -0
  45. starforge_cli/scaffold/project/configs/base/rm.yaml +224 -0
  46. starforge_cli/scaffold/project/configs/base/sft.yaml +294 -0
  47. starforge_cli/scaffold/project/configs/models/README.md +16 -0
  48. starforge_cli/scaffold/project/configs/models/qwen3.5-4b.yaml +12 -0
  49. starforge_cli/scaffold/project/configs/models/qwen3.5-9b.yaml +10 -0
  50. starforge_cli/scaffold/project/gitignore +11 -0
  51. starforge_cli/spec_builder.py +372 -0
  52. starforge_cli-0.1.6.dist-info/METADATA +40 -0
  53. starforge_cli-0.1.6.dist-info/RECORD +55 -0
  54. starforge_cli-0.1.6.dist-info/WHEEL +4 -0
  55. starforge_cli-0.1.6.dist-info/entry_points.txt +2 -0
@@ -0,0 +1,60 @@
1
+ """数据预处理脚本注册表:按约定发现,不再维护硬编码名单。
2
+
3
+ 加数据集 = 往 common/data/ 放一个 prepare_<name>.py —— 与「加方法 = 往
4
+ catalog 加 recipe 目录」同一哲学。CLI(补全、错误提示、执行)都从这里取,
5
+ 后续插件系统(kind=data-prep)把已安装插件的脚本目录并入扫描即可,命令层
6
+ 零改动。
7
+
8
+ 脚本首行 docstring 会作为该数据集的一句话说明展示在 CLI 里,用 ast 读取而
9
+ 不 import —— 数据脚本普遍拉重依赖(datasets/pandas),列个清单不该付这个价。
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import ast
14
+ from dataclasses import dataclass
15
+ from pathlib import Path
16
+ from typing import Iterable
17
+
18
+ _PREFIX = "prepare_"
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class DataPrep:
23
+ name: str
24
+ script: Path
25
+ summary: str
26
+
27
+
28
+ def _summary(script: Path) -> str:
29
+ try:
30
+ doc = ast.get_docstring(ast.parse(script.read_text(encoding="utf-8")))
31
+ except (OSError, SyntaxError):
32
+ return ""
33
+ return doc.strip().splitlines()[0].strip() if doc else ""
34
+
35
+
36
+ def default_dirs() -> list[Path]:
37
+ """扫描目录:仓库内置的 common/data + 已安装插件(forge_plugins/<name>/)。
38
+
39
+ 内置目录在前:同名脚本内置优先,插件不能悄悄替换内置数据集的语义。
40
+ """
41
+ from starforge_cli.commands.common import ROOT
42
+
43
+ dirs = [ROOT / "common" / "data"]
44
+ plugins_root = ROOT / "forge_plugins"
45
+ if plugins_root.is_dir():
46
+ dirs += sorted(p for p in plugins_root.iterdir() if p.is_dir())
47
+ return dirs
48
+
49
+
50
+ def discover(dirs: Iterable[Path] | None = None) -> dict[str, DataPrep]:
51
+ """扫描目录下的 prepare_<name>.py,返回 name → DataPrep(同名先到先得)。"""
52
+ found: dict[str, DataPrep] = {}
53
+ for d in dirs if dirs is not None else default_dirs():
54
+ if not d.is_dir():
55
+ continue
56
+ for script in sorted(d.glob(f"{_PREFIX}*.py")):
57
+ name = script.stem[len(_PREFIX):]
58
+ if name and name not in found:
59
+ found[name] = DataPrep(name=name, script=script, summary=_summary(script))
60
+ return found
@@ -0,0 +1,195 @@
1
+ """跨平台新建 / fork 实验(唯一入口:sf new,macOS / Linux / Windows 共用)。"""
2
+ from __future__ import annotations
3
+
4
+ import re
5
+ import shutil
6
+ import tempfile
7
+ from pathlib import Path
8
+
9
+
10
+ class NewExperimentError(Exception):
11
+ pass
12
+
13
+
14
+ def _resolve_src_dir(repo_root: Path, src: str) -> Path:
15
+ for c in (src, f"experiments/{src}", f"projects/{src}"):
16
+ p = repo_root / c
17
+ if p.is_dir():
18
+ return p
19
+ raise NewExperimentError(
20
+ f"找不到来源实验: {src}(试过 {src} / experiments/{src} / projects/{src})"
21
+ )
22
+
23
+
24
+ def _patch_fork_metadata(dest: Path, name: str) -> None:
25
+ """fork 后改 swanlab project/name 与 README 标题(保留注释)。"""
26
+ cfg = dest / "config.yaml"
27
+ if cfg.is_file():
28
+ lines = cfg.read_text(encoding="utf-8").splitlines()
29
+ in_sw, sw_indent = False, 0
30
+ for i, ln in enumerate(lines):
31
+ s, indent = ln.strip(), len(ln) - len(ln.lstrip())
32
+ if s == "swanlab:":
33
+ in_sw, sw_indent = True, indent
34
+ continue
35
+ if in_sw:
36
+ if s and indent <= sw_indent:
37
+ in_sw = False
38
+ else:
39
+ m = re.match(r"^(\s*)(project|name):\s*.*$", ln)
40
+ if m:
41
+ lines[i] = f'{m.group(1)}{m.group(2)}: "{name}"'
42
+ cfg.write_text("\n".join(lines) + "\n", encoding="utf-8")
43
+
44
+ readme = dest / "README.md"
45
+ if readme.is_file():
46
+ rl = readme.read_text(encoding="utf-8").splitlines()
47
+ for i, ln in enumerate(rl):
48
+ if ln.startswith("# "):
49
+ rl[i] = f"# {name}"
50
+ break
51
+ readme.write_text("\n".join(rl) + "\n", encoding="utf-8")
52
+
53
+
54
+ def _validate_recipe_template(dest: Path, recipe) -> None:
55
+ """只校验 recipe 声明的结构,不跨框架执行别的 validator。"""
56
+ if recipe.framework in {"nemo-rl", "verl"} and not (dest / "config.yaml").is_file():
57
+ raise NewExperimentError(f"recipe {recipe.id} 模板缺少 config.yaml")
58
+ if recipe.entrypoint.kind == "experiment":
59
+ entry = (dest / recipe.entrypoint.value).resolve()
60
+ root = dest.resolve()
61
+ if not entry.is_relative_to(root) or not entry.is_file():
62
+ raise NewExperimentError(
63
+ f"recipe {recipe.id} 模板缺少实验入口 {recipe.entrypoint.value}"
64
+ )
65
+
66
+
67
+ def _copy_recipe_template(dest: Path, recipe) -> None:
68
+ from starforge_core.recipes import recipe_directory
69
+
70
+ template = recipe_directory(recipe.id) / recipe.template
71
+ if not template.is_dir():
72
+ raise NewExperimentError(f"recipe {recipe.id} 缺少模板目录: {template}")
73
+ shutil.copytree(template, dest, dirs_exist_ok=True)
74
+
75
+
76
+ def _fork_experiment(repo_root: Path, kind: str, name: str, src: str) -> None:
77
+ dest = repo_root / kind / name
78
+ if dest.exists():
79
+ raise NewExperimentError(f"已存在: {dest}")
80
+
81
+ src_dir = _resolve_src_dir(repo_root, src)
82
+ from starforge_cli.commands.exp import validate_exp_config
83
+ from starforge_cli.recipe_lock import RecipeLockError, RecipeLockManager
84
+ from starforge_cli.spec_builder import infer_recipe
85
+
86
+ recipe_name = infer_recipe(src_dir)
87
+ if not recipe_name:
88
+ raise NewExperimentError(f"来源实验缺少 recipe.lock.json(recipe 声明): {src_dir}")
89
+ shutil.copytree(src_dir, dest)
90
+ outputs = dest / "outputs"
91
+ if outputs.exists():
92
+ shutil.rmtree(outputs)
93
+
94
+ _patch_fork_metadata(dest, name)
95
+ try:
96
+ RecipeLockManager().upgrade(
97
+ dest,
98
+ recipe_name=recipe_name,
99
+ validate_config=lambda path, recipe: validate_exp_config(
100
+ path, recipe, repo_root=repo_root
101
+ ),
102
+ )
103
+ except RecipeLockError as exc:
104
+ shutil.rmtree(dest, ignore_errors=True)
105
+ raise NewExperimentError(
106
+ f"fork 后无法升级到当前 recipe:{exc}。"
107
+ f"先对来源执行 `sf recipe upgrade {src_dir.name}`"
108
+ ) from exc
109
+
110
+ print(f"已 fork 实验: {dest}(来源: {src})")
111
+ print(f" · config.yaml 的 swanlab project/name 与 README 标题已改为: {name}")
112
+ print(f"下一步: 改 {dest}/config.yaml 顶部【① 调参区】试你的超参,"
113
+ f"然后 sf submit {name} --profile <目标集群>")
114
+
115
+
116
+ def _create_from_template(
117
+ repo_root: Path,
118
+ kind: str,
119
+ name: str,
120
+ method: str,
121
+ framework_version: str = "",
122
+ ) -> None:
123
+ dest = repo_root / kind / name
124
+ if dest.exists():
125
+ raise NewExperimentError(f"已存在: {dest}")
126
+
127
+ from starforge_core.contract import SpecError
128
+ from starforge_core.recipes import get_recipe
129
+
130
+ try:
131
+ recipe = get_recipe(method)
132
+ selected_framework_version = framework_version.strip() or recipe.runtime.default_version
133
+ recipe.runtime.resolve(selected_framework_version)
134
+ except SpecError as exc:
135
+ # get_recipe 的报错已列出可用值,并对无前缀裸名给出两段式提示。
136
+ raise NewExperimentError(f"--method 非法: {exc}") from exc
137
+ # 实验模板随 CLI 包分发(starforge_cli/scaffold/),不依赖项目内容。
138
+ from starforge_cli.project import experiment_template
139
+
140
+ template = experiment_template()
141
+ if not template.is_dir():
142
+ raise NewExperimentError(f"缺少模板目录: {template}")
143
+
144
+ dest.parent.mkdir(parents=True, exist_ok=True)
145
+ staging = Path(tempfile.mkdtemp(prefix=f".{name}.staging-", dir=dest.parent))
146
+ try:
147
+ shutil.copytree(template, staging, dirs_exist_ok=True)
148
+ _copy_recipe_template(staging, recipe)
149
+ gitkeep = staging / ".gitkeep"
150
+ if gitkeep.is_file():
151
+ gitkeep.unlink()
152
+ # recipe 声明只存在于 recipe.lock.json(infer_recipe 从锁读),不再写 method 标注文件;
153
+ # 硬件 profile 在提交时用 --profile 指定(env/overrides/拓扑均由服务端注册表下发)。
154
+ from starforge_cli.recipe_lock import write_recipe_lock
155
+
156
+ write_recipe_lock(staging, recipe.id, selected_framework_version)
157
+ _validate_recipe_template(staging, recipe)
158
+ staging.replace(dest)
159
+ except Exception:
160
+ shutil.rmtree(staging, ignore_errors=True)
161
+ raise
162
+
163
+ print(
164
+ f"已创建实验: {dest}(method={recipe.id}, "
165
+ f"framework={recipe.framework}@{selected_framework_version})"
166
+ )
167
+ print("下一步:")
168
+ print(f" 1. 编辑 {dest}/README.md(目标 / 模型 / 数据 / 监控)")
169
+ print(f" 2. 按 {recipe.id} recipe 编辑模板文件")
170
+ print(f" 3. 用 sf submit {name} --profile <目标集群>[:总卡数] 提交(如 h200 / h200:4)")
171
+
172
+
173
+ def create_experiment(
174
+ repo_root: Path,
175
+ kind: str,
176
+ name: str,
177
+ *,
178
+ src: str = "",
179
+ method: str = "nemo-rl/grpo",
180
+ framework_version: str = "",
181
+ ) -> None:
182
+ """新建或 fork 实验;失败时抛 NewExperimentError。"""
183
+ if kind not in ("experiments", "projects"):
184
+ raise NewExperimentError("第一个参数必须是 experiments 或 projects")
185
+
186
+ if src:
187
+ _fork_experiment(repo_root, kind, name, src)
188
+ else:
189
+ _create_from_template(
190
+ repo_root,
191
+ kind,
192
+ name,
193
+ method,
194
+ framework_version=framework_version,
195
+ )
@@ -0,0 +1,179 @@
1
+ """作业负载打包:清单式文件列表 + tar.gz + git 溯源。
2
+
3
+ 打包是白名单(清单式)而非黑名单:一次提交只上传用户拥有的三类路径——
4
+ 实验目录本身、共享代码 common/、配置继承根 configs/。平台 runner 与入口
5
+ 由服务端在准入后注入,客户端不能覆盖。
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import fnmatch
10
+ import hashlib
11
+ import io
12
+ import subprocess
13
+ from pathlib import Path
14
+
15
+ from starforge_cli import cli_ui
16
+
17
+
18
+ def _git_out(args: list[str], cwd: Path) -> str:
19
+ """git 命令输出;失败直接报错——提交/打包必须在完好的 git 仓库内进行。"""
20
+ try:
21
+ return subprocess.run(
22
+ ["git", "-C", str(cwd), *args], capture_output=True, text=True, check=True
23
+ ).stdout
24
+ except FileNotFoundError:
25
+ cli_ui.fail("找不到 git 命令", hint="安装 git 后重试")
26
+ except subprocess.CalledProcessError as e:
27
+ cli_ui.fail(
28
+ f"git {' '.join(args)} 失败: {e.stderr.strip() or e}",
29
+ hint="请在 git 仓库目录内运行",
30
+ )
31
+
32
+
33
+ def git_provenance(repo_root: Path, exp_rel: str) -> dict:
34
+ """提交可追溯:git commit / dirty / config 指纹。git 不可用时直接报错。"""
35
+ commit = _git_out(["rev-parse", "--short", "HEAD"], repo_root).strip()
36
+ if not commit:
37
+ cli_ui.fail("无法确定 git commit", hint="请在有提交历史的 git 仓库内运行")
38
+ dirty = bool(_git_out(["status", "--porcelain"], repo_root).strip())
39
+ cfg = repo_root / exp_rel / "config.yaml"
40
+ if cfg.is_file():
41
+ config_sha = hashlib.sha256(cfg.read_bytes()).hexdigest()[:12]
42
+ else:
43
+ config_sha = "none"
44
+ return {"git_commit": commit, "git_dirty": dirty, "config_sha": config_sha}
45
+
46
+
47
+ # 清单内仍要剔除的非运行时产物(实验目录里偶尔混入的文档/报告)。
48
+ _UPLOAD_EXCLUDE_SUFFIXES = (".pdf",)
49
+
50
+
51
+ def _is_upload_excluded(rel: str) -> bool:
52
+ """该相对路径是否属于「不随作业上传」的非运行时产物。"""
53
+ return rel.replace("\\", "/").lower().endswith(_UPLOAD_EXCLUDE_SUFFIXES)
54
+
55
+
56
+ # 疑似密钥/凭据文件:命中即拒绝打包(fail-closed)。
57
+ # 清单用 `git ls-files --others` 收集未跟踪文件,.gitignore 覆盖不到的命名
58
+ # (hf_token.txt / credentials.json / *.pem …)会随 --allow-dirty 静默出网。
59
+ # 注意别用过宽的 "token*":会误伤 tokenizer_config.json 这类训练必需文件。
60
+ _SENSITIVE_EXACT = frozenset({
61
+ ".env", ".envrc", ".netrc",
62
+ "credentials.json", "token.txt", "api_key.txt",
63
+ })
64
+ _SENSITIVE_GLOBS = (
65
+ ".env.*", "*.pem", "*.key", "*.p12", "*.pfx",
66
+ "id_rsa*", "id_ed25519*", "id_ecdsa*", "id_dsa*",
67
+ "secrets.*", "*.secret", "hf_token*", "*_credentials.json",
68
+ )
69
+
70
+
71
+ def _is_sensitive(rel: str) -> bool:
72
+ base = rel.replace("\\", "/").rsplit("/", 1)[-1].lower()
73
+ if base in _SENSITIVE_EXACT:
74
+ return True
75
+ return any(fnmatch.fnmatch(base, pat) for pat in _SENSITIVE_GLOBS)
76
+
77
+
78
+ # 集群 Linux 侧会 source/读取;Windows 工作区可能是 CRLF,上传前须规范为 LF。
79
+ _UNIX_LF_SUFFIXES = (".sh", ".conf")
80
+ _UNIX_LF_BASENAMES = frozenset({"sf", "lab"})
81
+
82
+
83
+ def _needs_unix_lf(rel: str) -> bool:
84
+ r = rel.replace("\\", "/")
85
+ base = r.rsplit("/", 1)[-1]
86
+ return base in _UNIX_LF_BASENAMES or r.endswith(_UNIX_LF_SUFFIXES)
87
+
88
+
89
+ def _normalize_unix_lf(data: bytes) -> bytes:
90
+ return data.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
91
+
92
+
93
+ def upload_manifest(exp_rel: str, profile: str) -> tuple[tuple[str, ...], tuple[str, ...]]:
94
+ """作业包白名单:(目录前缀, 精确文件)。
95
+
96
+ 集群侧运行时的全部依赖:
97
+ experiments/<exp>/ 实验本体(config / recipe.lock / run.py …)
98
+ common/ 实验 run.py 通过 REPO_ROOT sys.path 引用的共享代码
99
+ configs/ 实验 config.yaml 用 ../../configs/ 相对路径继承的基底
100
+ profile 的 env/overrides 已服务端化(Console 硬件注册表经环境变量注入),
101
+ 不再随包上传 cluster/ 目录;profile 参数保留用于入参完整性校验。
102
+ """
103
+ if not exp_rel or not profile:
104
+ raise ValueError("打包清单需要显式的实验路径与硬件 profile")
105
+ prefixes = (
106
+ exp_rel.rstrip("/") + "/",
107
+ "common/",
108
+ "configs/",
109
+ )
110
+ exact: tuple[str, ...] = ()
111
+ return prefixes, exact
112
+
113
+
114
+ def list_working_files(repo_root: Path, *, exp_rel: str, profile: str,
115
+ with_stats: bool = False):
116
+ """作业负载文件清单:git 跟踪 + 未忽略(遵循 .gitignore),再按白名单收窄。
117
+
118
+ with_stats=True 时返回 (files, skipped_count),skipped 为 git 工作树中
119
+ 不属于本次作业负载而被略过的文件数。
120
+ """
121
+ prefixes, exact = upload_manifest(exp_rel, profile)
122
+ listing = _git_out(["ls-files", "--cached", "--others", "--exclude-standard"], repo_root)
123
+ raw = [f.strip() for f in listing.splitlines() if f.strip()]
124
+ files = [
125
+ f for f in raw
126
+ if (f.startswith(prefixes) or f in exact) and not _is_upload_excluded(f)
127
+ ]
128
+ if sensitive := [f for f in files if _is_sensitive(f)]:
129
+ cli_ui.fail(
130
+ "作业包中发现疑似密钥/凭据文件,拒绝打包上传",
131
+ title="作业包中发现疑似密钥/凭据文件,拒绝打包上传",
132
+ items=sensitive,
133
+ hint="移出上传目录或加入 .gitignore;确属训练必需请改名并确认其中不含密钥",
134
+ )
135
+ if not files:
136
+ cli_ui.fail(
137
+ "作业包为空:清单内没有可上传的文件。",
138
+ hint=f"确认实验目录 {exp_rel}/ 存在且已入 git",
139
+ )
140
+ # 未跟踪文件只会在 --allow-dirty 时走到这里(clean-tree 检查把 dirty 拦在前面)。
141
+ # 它们不受 commit 溯源约束,必须让用户看见都上传了什么。
142
+ tracked = {f.strip() for f in _git_out(["ls-files"], repo_root).splitlines() if f.strip()}
143
+ if untracked := [f for f in files if f not in tracked]:
144
+ cli_ui.emit_warning(
145
+ f"{len(untracked)} 个未跟踪文件将随作业上传(不受 commit 溯源约束)",
146
+ body="\n".join(untracked[:20]) + ("\n…" if len(untracked) > 20 else ""),
147
+ )
148
+ return (files, len(raw) - len(files)) if with_stats else files
149
+
150
+
151
+ def pack_working_dir(repo_root: Path, files: list[str], on_add=None) -> bytes:
152
+ """把清单文件打成 tar.gz;清单里的文件缺失直接报错,不静默跳过。
153
+
154
+ on_add(n):每加入一个文件回调一次,用于驱动进度条。
155
+ """
156
+ import tarfile
157
+
158
+ buf = io.BytesIO()
159
+ with tarfile.open(fileobj=buf, mode="w:gz") as tar:
160
+ for rel in files:
161
+ p = repo_root / rel
162
+ if not p.is_file():
163
+ cli_ui.fail(
164
+ f"打包失败:清单文件不存在或不是普通文件: {rel}",
165
+ hint="文件可能已删除但仍在 git 索引里;git add -A 后重试",
166
+ )
167
+ arcname = rel.replace("\\", "/")
168
+ if _needs_unix_lf(rel):
169
+ data = _normalize_unix_lf(p.read_bytes())
170
+ info = tarfile.TarInfo(name=arcname)
171
+ info.size = len(data)
172
+ info.mtime = int(p.stat().st_mtime)
173
+ info.mode = p.stat().st_mode
174
+ tar.addfile(info, io.BytesIO(data))
175
+ else:
176
+ tar.add(p, arcname=arcname)
177
+ if on_add:
178
+ on_add(1)
179
+ return buf.getvalue()
@@ -0,0 +1,73 @@
1
+ """实验插件锁文件(plugins.lock.json)。
2
+
3
+ 与 recipe.lock.json 同一哲学:实验用哪个插件的哪个精确版本、什么内容摘要,
4
+ 在 `sf plugin install --exp` 时固定下来;提交时原样写进 JobSpec,服务端与
5
+ 集群侧各自比对 —— 不一致就拒绝,绝不猜。
6
+
7
+ 锁定的是 (id, version, digest) 三元组,不含代码:插件包体由平台在提交时注入
8
+ 作业包,客户端不上传、也改不了。
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ from pathlib import Path
14
+
15
+ LOCK_FILE = "plugins.lock.json"
16
+ LOCK_VERSION = "forge/plugins-lock/v1"
17
+
18
+
19
+ def read_plugin_lock(exp_dir: Path) -> list[dict]:
20
+ """读取实验的插件引用列表;没有锁文件返回空表。"""
21
+ path = Path(exp_dir) / LOCK_FILE
22
+ if not path.is_file():
23
+ return []
24
+ try:
25
+ payload = json.loads(path.read_text(encoding="utf-8"))
26
+ except (OSError, json.JSONDecodeError) as exc:
27
+ raise ValueError(f"{path} 非法: {exc}") from exc
28
+ if payload.get("apiVersion") != LOCK_VERSION:
29
+ raise ValueError(
30
+ f"{path} 的 apiVersion 不是 {LOCK_VERSION};"
31
+ "重新执行 `sf plugin install <id> --exp <exp>` 重建锁文件"
32
+ )
33
+ plugins = payload.get("plugins") or []
34
+ for p in plugins:
35
+ if not all(str(p.get(k) or "").strip() for k in ("id", "version", "digest")):
36
+ raise ValueError(f"{path} 存在缺字段的插件引用: {p!r}")
37
+ return [
38
+ {"id": p["id"], "version": p["version"], "digest": p["digest"]}
39
+ for p in plugins
40
+ ]
41
+
42
+
43
+ def upsert_plugin_lock(exp_dir: Path, entry: dict) -> list[dict]:
44
+ """写入/更新一条插件引用(同 id 覆盖),返回更新后的列表。"""
45
+ plugins = [p for p in read_plugin_lock(exp_dir) if p["id"] != entry["id"]]
46
+ plugins.append({"id": entry["id"], "version": entry["version"], "digest": entry["digest"]})
47
+ plugins.sort(key=lambda p: p["id"])
48
+ _write(exp_dir, plugins)
49
+ return plugins
50
+
51
+
52
+ def remove_plugin_lock(exp_dir: Path, plugin_id: str) -> bool:
53
+ """移除一条插件引用;返回是否真的移除了。"""
54
+ plugins = read_plugin_lock(exp_dir)
55
+ kept = [p for p in plugins if p["id"] != plugin_id]
56
+ if len(kept) == len(plugins):
57
+ return False
58
+ _write(exp_dir, kept)
59
+ return True
60
+
61
+
62
+ def _write(exp_dir: Path, plugins: list[dict]) -> None:
63
+ path = Path(exp_dir) / LOCK_FILE
64
+ if not plugins:
65
+ path.unlink(missing_ok=True)
66
+ return
67
+ path.write_text(
68
+ json.dumps(
69
+ {"apiVersion": LOCK_VERSION, "plugins": plugins},
70
+ ensure_ascii=False, indent=2, sort_keys=True,
71
+ ) + "\n",
72
+ encoding="utf-8",
73
+ )
@@ -0,0 +1,130 @@
1
+ """StarForge 项目脚手架:`sf init` 的实现。
2
+
3
+ CLI 经 pip 分发(starforge-cli),用户不 clone 工具仓库;`sf init` 生成
4
+ 自包含的项目目录(自己的 git 仓库),布局与作业包白名单严格对应:
5
+
6
+ my-lab/
7
+ ├── starforge.yaml 项目标记(项目发现的唯一依据)
8
+ ├── experiments/ 实验目录(sf new 在此创建)
9
+ ├── configs/ 官方基底 + 模型片段(脚手架落地,用户可调可 pin)
10
+ ├── common/ 项目共享代码(数据脚本 / 环境 / 奖励)
11
+ └── .gitignore
12
+
13
+ 平台 bootstrap/runner 不落地到项目,也不由 CLI 上传;console 在准入后注入
14
+ 内容寻址 Job Capsule,用户代码不能覆盖平台入口。
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import re
19
+ import shutil
20
+ import subprocess
21
+ from importlib.metadata import version as _pkg_version
22
+ from importlib.resources import files
23
+ from pathlib import Path
24
+
25
+ import yaml
26
+
27
+ PROJECT_MARKER = "starforge.yaml"
28
+ # 与 console 分组键一致:会进 URL / 路径,只允许安全字符。
29
+ _PROJECT_NAME_RE = re.compile(r"^[A-Za-z0-9._-]+$")
30
+
31
+
32
+ class ProjectError(RuntimeError):
33
+ pass
34
+
35
+
36
+ def load_project_name(root: Path) -> str:
37
+ """读 starforge.yaml 的 name。提交到 console 的 project 只来自这里。"""
38
+ path = root / PROJECT_MARKER
39
+ if not path.is_file():
40
+ raise ProjectError(f"{root} 不是 StarForge 项目(缺 {PROJECT_MARKER})")
41
+ try:
42
+ data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
43
+ except yaml.YAMLError as exc:
44
+ raise ProjectError(f"{path} 无法解析:{exc}") from exc
45
+ if not isinstance(data, dict):
46
+ raise ProjectError(f"{path} 根节点必须是对象")
47
+ name = str(data.get("name") or "").strip()
48
+ if not name:
49
+ raise ProjectError(f"{path} 缺少 name")
50
+ if name in (".", "..") or not _PROJECT_NAME_RE.match(name):
51
+ raise ProjectError(f"非法项目名 {name!r}(仅允许字母数字与 . _ -)")
52
+ return name
53
+
54
+
55
+ def scaffold_root() -> Path:
56
+ """包内脚手架资源根(editable 与 wheel 安装均为真实目录)。"""
57
+ return Path(str(files("starforge_cli") / "scaffold"))
58
+
59
+
60
+ def experiment_template() -> Path:
61
+ """实验脚手架模板(sf new 用)。"""
62
+ return scaffold_root() / "experiment-template"
63
+
64
+
65
+ class InitError(RuntimeError):
66
+ pass
67
+
68
+
69
+ def init_project(dest: Path, *, name: str = "", git: bool = True) -> Path:
70
+ """在 dest 生成 StarForge 项目;目录已是项目或非空冲突时报错。"""
71
+ dest = dest.resolve()
72
+ if (dest / PROJECT_MARKER).is_file():
73
+ raise InitError(f"{dest} 已经是 StarForge 项目")
74
+ if dest.exists() and any(dest.iterdir()):
75
+ # 允许在非空目录初始化(如已有 README 的空仓库),但拒绝覆盖关键路径
76
+ for clash in ("experiments", "configs", "common", PROJECT_MARKER):
77
+ if (dest / clash).exists():
78
+ raise InitError(f"{dest} 下已存在 {clash},拒绝覆盖;换个目录或清理后重试")
79
+ dest.mkdir(parents=True, exist_ok=True)
80
+
81
+ src = scaffold_root() / "project"
82
+ shutil.copytree(src / "configs", dest / "configs")
83
+ shutil.copytree(src / "common", dest / "common")
84
+ if not (dest / ".gitignore").exists():
85
+ shutil.copyfile(src / "gitignore", dest / ".gitignore")
86
+ (dest / "experiments").mkdir(exist_ok=True)
87
+ keep = dest / "experiments" / ".gitkeep"
88
+ if not keep.exists():
89
+ keep.write_text("")
90
+
91
+ project_name = (name or dest.name).strip()
92
+ try:
93
+ cli_version = _pkg_version("starforge-cli")
94
+ except Exception: # noqa: BLE001 — 源码运行(未安装分发元数据)
95
+ cli_version = "dev"
96
+ (dest / PROJECT_MARKER).write_text(
97
+ "# StarForge 项目标记:`sf` 命令以此发现项目根,勿删除。\n"
98
+ f'apiVersion: forge/project/v1\n'
99
+ f'name: {project_name}\n'
100
+ f'created_by: starforge-cli {cli_version}\n',
101
+ encoding="utf-8",
102
+ )
103
+ readme = dest / "README.md"
104
+ if not readme.exists():
105
+ readme.write_text(
106
+ f"# {project_name}\n\n"
107
+ "StarForge(星锻)微调项目。常用命令:\n\n"
108
+ "```bash\n"
109
+ "sf new my-exp --method nemo-rl/grpo # 新建实验\n"
110
+ "sf validate my-exp # 本地校验\n"
111
+ "sf submit my-exp --profile h200:8 # 提交训练\n"
112
+ "sf job logs # 跟随日志\n"
113
+ "```\n\n"
114
+ "目录说明:`experiments/` 实验本体;`configs/` 官方基底与模型片段;\n"
115
+ "`common/` 项目共享代码(数据脚本 / Agent 环境 / 奖励函数)。\n",
116
+ encoding="utf-8",
117
+ )
118
+
119
+ if git and shutil.which("git") and not (dest / ".git").exists():
120
+ subprocess.run(["git", "init", "-q"], cwd=dest, check=False)
121
+ # 本地身份:CI / 干净机器没有 user.name 时也能完成首提交(submit 需要 git 溯源)
122
+ subprocess.run(["git", "config", "user.email", "starforge@localhost"], cwd=dest, check=False)
123
+ subprocess.run(["git", "config", "user.name", "sf init"], cwd=dest, check=False)
124
+ subprocess.run(["git", "config", "commit.gpgsign", "false"], cwd=dest, check=False)
125
+ subprocess.run(["git", "add", "-A"], cwd=dest, check=False)
126
+ subprocess.run(
127
+ ["git", "commit", "-q", "-m", "sf init: StarForge project scaffold"],
128
+ cwd=dest, check=False,
129
+ )
130
+ return dest