gatedtrain 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gatedtrain/__init__.py +16 -0
- gatedtrain/__main__.py +7 -0
- gatedtrain/backends/__init__.py +6 -0
- gatedtrain/backends/nemo_evaluator.py +34 -0
- gatedtrain/backends/nemo_rl.py +182 -0
- gatedtrain/canonical.py +236 -0
- gatedtrain/cli.py +370 -0
- gatedtrain/constraints.py +811 -0
- gatedtrain/errors.py +32 -0
- gatedtrain/ext/__init__.py +29 -0
- gatedtrain/ext/arm.py +360 -0
- gatedtrain/ext/collect.py +815 -0
- gatedtrain/ext/conformance.py +198 -0
- gatedtrain/ext/executor.py +14 -0
- gatedtrain/ext/fill.py +150 -0
- gatedtrain/ext/from_nel.py +255 -0
- gatedtrain/ext/launch.py +2057 -0
- gatedtrain/ext/merge.py +182 -0
- gatedtrain/ext/registry.py +179 -0
- gatedtrain/ext/rowkit.py +55 -0
- gatedtrain/ext/sdk.py +315 -0
- gatedtrain/ext/smoke.py +162 -0
- gatedtrain/families/__init__.py +5 -0
- gatedtrain/families/diffusers_lora.py +312 -0
- gatedtrain/families/flow_grpo.py +643 -0
- gatedtrain/families/nemo_rl.py +35 -0
- gatedtrain/gates.py +1002 -0
- gatedtrain/impact.py +348 -0
- gatedtrain/items.py +94 -0
- gatedtrain/ledger.py +247 -0
- gatedtrain/registration.py +1086 -0
- gatedtrain/renamed.py +89 -0
- gatedtrain/report.py +380 -0
- gatedtrain/runner.py +561 -0
- gatedtrain/spec.py +409 -0
- gatedtrain/stubs/__init__.py +19 -0
- gatedtrain/stubs/evaluator.py +88 -0
- gatedtrain/stubs/model.py +120 -0
- gatedtrain/stubs/task.py +85 -0
- gatedtrain/stubs/trainer.py +203 -0
- gatedtrain/testing/__init__.py +16 -0
- gatedtrain/testing/ext.py +87 -0
- gatedtrain/testing/support.py +108 -0
- gatedtrain/testing/synthetic.py +463 -0
- gatedtrain/tierb.py +1047 -0
- gatedtrain-0.3.0.dist-info/METADATA +239 -0
- gatedtrain-0.3.0.dist-info/RECORD +53 -0
- gatedtrain-0.3.0.dist-info/WHEEL +5 -0
- gatedtrain-0.3.0.dist-info/entry_points.txt +2 -0
- gatedtrain-0.3.0.dist-info/licenses/LICENSE +202 -0
- gatedtrain-0.3.0.dist-info/licenses/NOTICE +4 -0
- gatedtrain-0.3.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +48 -0
- gatedtrain-0.3.0.dist-info/top_level.txt +1 -0
gatedtrain/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Walking-skeleton prototype of the Composable Post-Training Framework.
|
|
2
|
+
|
|
3
|
+
prototype, unpromotable. The Tier A trainer and evaluator are stand-ins
|
|
4
|
+
marked skeleton-stub (MVP-SPEC section 3).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
__version__ = "0.3.0"
|
|
8
|
+
|
|
9
|
+
PACKAGE = "gatedtrain"
|
|
10
|
+
ANALYSIS_CODE_VERSION = f"{PACKAGE} {__version__}"
|
|
11
|
+
|
|
12
|
+
# Tags carried by every output (MVP-SPEC section 2.11).
|
|
13
|
+
TAGS = ("prototype", "unpromotable")
|
|
14
|
+
REPORT_TAGS = ("prototype", "unpromotable", "registration: unsigned")
|
|
15
|
+
STUB_MARK = "skeleton-stub"
|
|
16
|
+
NOT_ASSESSED = "not-assessed (skeleton)"
|
gatedtrain/__main__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Tier B backends (GPU, run by an operator; MVP-SPEC section 4).
|
|
2
|
+
|
|
3
|
+
Code only: the test suite never runs them. `nemo_rl` builds the launch command for
|
|
4
|
+
one NeMo RL recipe at 4f39fd8; `nemo_evaluator` reads NeMo Evaluator
|
|
5
|
+
`results.jsonl` at ea35ba3. Outputs are tagged `unpromotable` (01 section 8.1).
|
|
6
|
+
"""
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Tier B evaluator backend `nemo_evaluator` (MVP-SPEC section 4).
|
|
2
|
+
|
|
3
|
+
An operator runs NeMo Evaluator at ea35ba3 (`nel eval run`, cli/eval.py:71-88)
|
|
4
|
+
on the baseline and on the trained checkpoint with the registered repeats
|
|
5
|
+
(K) and sampler. NeMo Evaluator writes one per-sample record per line to
|
|
6
|
+
`<output dir>/results.jsonl` (engine/artifacts.py:96-102), with the keys of
|
|
7
|
+
engine/eval_loop.py:790-803. This module only reads those files; it pairs
|
|
8
|
+
records by (problem_idx, repeat) exactly as NeMo Evaluator's
|
|
9
|
+
`load_paired_records` does (engine/comparison.py:360).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from ..gates import load_paired_records
|
|
18
|
+
|
|
19
|
+
NEMO_EVALUATOR_COMMIT = "ea35ba3"
|
|
20
|
+
RESULT_KEYS = ("problem_idx", "repeat", "reward", "scorer_rewards", "prompt", "model_response", "extracted_answer",
|
|
21
|
+
"expected_answer", "scoring_details", "metadata", "tokens", "latency_ms")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def read_results(task_dir: Path) -> dict[tuple[int, int], dict[str, Any]]:
|
|
25
|
+
"""{(problem_idx, repeat): record} from a NeMo Evaluator task directory."""
|
|
26
|
+
return load_paired_records(Path(task_dir))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def build_eval_command(*, model_url: str, model_id: str, bench: str, repeats: int, temperature: float,
|
|
30
|
+
max_tokens: int, output_dir: str) -> list[str]:
|
|
31
|
+
"""argv for one NeMo Evaluator run (cli/eval.py:72-88). Never executed by gatedtrain."""
|
|
32
|
+
return ["nel", "eval", "run", "--bench", bench, "--model-url", model_url, "--model-id", model_id,
|
|
33
|
+
"--repeats", str(int(repeats)), "--temperature", str(float(temperature)),
|
|
34
|
+
"--max-tokens", str(int(max_tokens)), "--output-dir", output_dir]
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""Tier B trainer backend `nemo_rl` (MVP-SPEC section 4; TIER-B-SPEC H-5).
|
|
2
|
+
|
|
3
|
+
Builds, but never runs, the launch command for one NeMo RL recipe on
|
|
4
|
+
unmodified NeMo RL at 4f39fd8. Citations are file:line at 4f39fd8.
|
|
5
|
+
|
|
6
|
+
- **Recipe:** `examples/configs/grpo_smoke.yaml`. It layers on
|
|
7
|
+
`grpo_math_1B.yaml` (grpo_smoke.yaml:6), whose `cluster.gpus_per_node: 1`
|
|
8
|
+
and `num_nodes: 1` are at grpo_math_1B.yaml:690-692. Model
|
|
9
|
+
`Qwen/Qwen3-0.6B` (grpo_smoke.yaml:15).
|
|
10
|
+
- **Launch script:** `examples/run_grpo.py` with `--config` plus Hydra
|
|
11
|
+
overrides (run_grpo.py:53-63, :86-87). Hydra struct mode is on
|
|
12
|
+
(nemo_rl/utils/config.py:188), so keys the recipe lacks need `++`.
|
|
13
|
+
|
|
14
|
+
Overrides (`COMMON_OVERRIDES`):
|
|
15
|
+
|
|
16
|
+
| Override | Why |
|
|
17
|
+
|---|---|
|
|
18
|
+
| `data_plane.enabled=true` | selects `grpo_train_sync` (nemo_rl/data_plane/factory.py:51-55), whose outputs the ledger reads (grpo_sync.py:1323, :1353, :1407-1414, :1458-1460) |
|
|
19
|
+
| `data.shuffle=false` | file order (grpo_math_1B.yaml:617; StatefulDataLoader, grpo.py:883-890) |
|
|
20
|
+
| `checkpointing.enabled=true` | the smoke recipe turns checkpoints off (grpo_smoke.yaml:12-13); `consumed_samples` lives in them (grpo_sync.py:1353) |
|
|
21
|
+
| `checkpointing.keep_top_k=null` | keep every cut for the ledger (checkpoint.py:234 `NotRequired[int \\| None]`; deletion at :590-631) |
|
|
22
|
+
| `checkpointing.checkpoint_dir`, `logger.log_dir` | the stage's output directory (grpo_math_1B.yaml:138, :665) |
|
|
23
|
+
| `data.train.dataset_name=ResponseDataset` | local JSONL loader (response_datasets/__init__.py:99; response_dataset.py:32-61) |
|
|
24
|
+
| `++data.train.data_path`, `++data.train.input_key`, `++data.train.output_key` | ResponseDataset arguments absent from the recipe, so `++` (response_dataset.py:42-48) |
|
|
25
|
+
| `data.train.split_validation_size=0` | no validation split reorders rows (response_dataset.py:76) |
|
|
26
|
+
| `data.default.prompt_file` | NeMo Evaluator's GSM8K prompt, written by `export-gsm8k` (G-15; processors.py:574-576) |
|
|
27
|
+
| `policy.tokenizer.chat_template_kwargs={enable_thinking:false}` | Qwen3 thinking off (G-15; grpo_math_1B.yaml:158; nemo_rl/algorithms/utils.py:465-496) |
|
|
28
|
+
| `policy.automodel_cfg.checkpoint.save_consolidated=final` | HF safetensors under `step_<last>/policy/weights/model/consolidated` for serving (grpo_math_1B.yaml:178-181; Automodel config.py:64 `FINAL = "final"`) |
|
|
29
|
+
|
|
30
|
+
Profiles (`PROFILES`): `smoke` keeps the recipe's 10 steps of 2 x 4
|
|
31
|
+
(grpo_smoke.yaml:8-10); `long` is 150 steps of 32 x 8 with a global batch of
|
|
32
|
+
256 (an estimate, not a NeMo RL recipe; docs/tier-b-runbook.md step 7, the long run).
|
|
33
|
+
`samples_per_position` is the profile's `grpo.num_generations_per_prompt`;
|
|
34
|
+
after a run, read it from the checkpoint with
|
|
35
|
+
`samples_per_position_from_checkpoint` (utils/checkpoint.py:414-419), never
|
|
36
|
+
from a constant (interim rule G-7).
|
|
37
|
+
|
|
38
|
+
Launchers:
|
|
39
|
+
|
|
40
|
+
- `local`: absolute script and recipe, run with cwd `nemo_rl_dir`, because the
|
|
41
|
+
recipe has relative paths (grpo_math_1B.yaml:632);
|
|
42
|
+
- `docker`: `docker run` with the documented flags
|
|
43
|
+
(docs/about/installation.md:21-37), `nofile=65535` for Ray (ray.sub:161-182),
|
|
44
|
+
`TORCH_CUDA_ARCH_LIST=8.9` and `FLASHINFER_CUDA_ARCH_LIST=8.9` because the
|
|
45
|
+
image targets 9.0 and 10.0 (Dockerfile:231), the image's own source at
|
|
46
|
+
`/opt/nemo-rl` with no checkout mounted over it (the fingerprint check,
|
|
47
|
+
nemo_rl/__init__.py:105), and `-w /opt/nemo-rl`;
|
|
48
|
+
- `sbatch`: `sbatch --no-requeue` around NeMo RL's `ray.sub`, driven by
|
|
49
|
+
COMMAND, CONTAINER and MOUNTS (ray.sub:110-112; docs/cluster.md:10-24),
|
|
50
|
+
with COMMAND equal to the local argv. R9-OI-1
|
|
51
|
+
exposure: an administrator or an automated health-check service can still requeue the job.
|
|
52
|
+
|
|
53
|
+
Outputs are tagged `unpromotable`. The nemo_gym-responses path writes no step
|
|
54
|
+
log (grpo_sync.py:1428) and is unsupported.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
from __future__ import annotations
|
|
58
|
+
|
|
59
|
+
import shlex
|
|
60
|
+
from pathlib import Path
|
|
61
|
+
|
|
62
|
+
import yaml
|
|
63
|
+
|
|
64
|
+
NEMO_RL_COMMIT = "4f39fd8"
|
|
65
|
+
RECIPE = "examples/configs/grpo_smoke.yaml"
|
|
66
|
+
LAUNCH_SCRIPT = "examples/run_grpo.py"
|
|
67
|
+
CONTAINER_SRC = "/opt/nemo-rl"
|
|
68
|
+
TAGS = ("prototype", "unpromotable")
|
|
69
|
+
LAUNCHERS = ("local", "docker", "sbatch")
|
|
70
|
+
|
|
71
|
+
# Documented defaults for the keywords added in TIER-B-SPEC H-5.
|
|
72
|
+
DEFAULT_PROFILE = "smoke"
|
|
73
|
+
DEFAULT_IMAGE = "nemo-rl:4f39fd8"
|
|
74
|
+
DEFAULT_WORK_DIR = "/data/gatedtrain-tierb" # host directory mounted at /work by the docker launcher
|
|
75
|
+
DEFAULT_HF_CACHE = "/data/hf-cache" # host Hugging Face cache mounted at /hf_home
|
|
76
|
+
DEFAULT_SHM_SIZE = "32g" # keep at or below host RAM (docs/about/installation.md:25 uses 64g)
|
|
77
|
+
|
|
78
|
+
COMMON_OVERRIDES = (
|
|
79
|
+
"data_plane.enabled=true",
|
|
80
|
+
"data.shuffle=false",
|
|
81
|
+
"checkpointing.enabled=true",
|
|
82
|
+
"checkpointing.keep_top_k=null",
|
|
83
|
+
"checkpointing.checkpoint_dir={out}/checkpoints",
|
|
84
|
+
"logger.log_dir={out}/logs",
|
|
85
|
+
"data.train.dataset_name=ResponseDataset",
|
|
86
|
+
"++data.train.data_path={export}/train.jsonl",
|
|
87
|
+
"++data.train.input_key=question",
|
|
88
|
+
"++data.train.output_key=answer",
|
|
89
|
+
"data.train.split_validation_size=0",
|
|
90
|
+
"data.default.prompt_file={export}/nel_gsm8k_prompt.txt",
|
|
91
|
+
"policy.tokenizer.chat_template_kwargs={{enable_thinking:false}}",
|
|
92
|
+
"policy.automodel_cfg.checkpoint.save_consolidated=final",
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
PROFILES = {
|
|
96
|
+
"smoke": {"max_num_steps": 10, "save_period": 5, "num_prompts_per_step": 2, "num_generations_per_prompt": 4,
|
|
97
|
+
"extra": ()},
|
|
98
|
+
"long": {"max_num_steps": 150, "save_period": 25, "num_prompts_per_step": 32, "num_generations_per_prompt": 8,
|
|
99
|
+
"extra": ("policy.train_global_batch_size=256", "policy.train_micro_batch_size=8")},
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def overrides(*, out_dir: str, export_dir: str, profile: str, base_model: str | None = None) -> list[str]:
|
|
104
|
+
"""Hydra overrides for run_grpo.py: the common set, `policy.model_name` (F-17) and the profile's."""
|
|
105
|
+
if profile not in PROFILES:
|
|
106
|
+
raise ValueError(f"unknown profile {profile!r}; use one of {sorted(PROFILES)}")
|
|
107
|
+
p = PROFILES[profile]
|
|
108
|
+
out = [o.format(out=out_dir, export=export_dir) for o in COMMON_OVERRIDES]
|
|
109
|
+
if base_model is not None:
|
|
110
|
+
# F-17: train from the base-model directory whose manifest the gate records. The key is in
|
|
111
|
+
# the recipe (grpo_math_1B.yaml:155; grpo_smoke.yaml:15); the tokenizer follows it
|
|
112
|
+
# (`name: ${policy.model_name}`, grpo_math_1B.yaml:157). NeMo RL loads it with from_pretrained
|
|
113
|
+
# (nemo_rl/models/automodel/setup.py:374-375, :941-942) and passes it to vLLM (grpo.py:1426).
|
|
114
|
+
out.append(f"policy.model_name={base_model}")
|
|
115
|
+
out += [f"grpo.max_num_steps={p['max_num_steps']}", f"checkpointing.save_period={p['save_period']}",
|
|
116
|
+
f"grpo.num_prompts_per_step={p['num_prompts_per_step']}",
|
|
117
|
+
f"grpo.num_generations_per_prompt={p['num_generations_per_prompt']}", *p["extra"]]
|
|
118
|
+
return out
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def launch_spec(stage: dict, *, nemo_rl_dir: str, launcher: str, out_dir: str | None = None,
|
|
122
|
+
export_dir: str = "/work/export", profile: str = DEFAULT_PROFILE, image: str = DEFAULT_IMAGE,
|
|
123
|
+
work_dir: str = DEFAULT_WORK_DIR, hf_cache: str = DEFAULT_HF_CACHE, base_model: str | None = None,
|
|
124
|
+
account: str = "<account>", partition: str = "<partition>", time_limit: str = "01:00:00") -> dict:
|
|
125
|
+
"""{"argv", "cwd", "samples_per_position"} for one Tier B stage. Never executed by gatedtrain.
|
|
126
|
+
|
|
127
|
+
`out_dir` defaults to `<nemo_rl_dir>/results/gatedtrain/<stage id>`. For the docker launcher,
|
|
128
|
+
`out_dir`, `export_dir` and `base_model` are paths inside the container (under /work).
|
|
129
|
+
`base_model` is the spec's `base` directory as the trainer sees it; it becomes
|
|
130
|
+
`policy.model_name` (F-17). Pass it for every gated run. `None`, the default kept for older
|
|
131
|
+
callers, leaves the recipe's `Qwen/Qwen3-0.6B`, and the gate's baseline then is not the
|
|
132
|
+
trained parent.
|
|
133
|
+
"""
|
|
134
|
+
if launcher not in LAUNCHERS:
|
|
135
|
+
raise ValueError(f"unknown launcher {launcher!r}; use one of {LAUNCHERS} (NeMo-Run is not supported)")
|
|
136
|
+
stage_id = str(stage.get("id", "stage"))
|
|
137
|
+
out = out_dir or f"{nemo_rl_dir}/results/gatedtrain/{stage_id}"
|
|
138
|
+
ov = overrides(out_dir=out, export_dir=export_dir, profile=profile, base_model=base_model)
|
|
139
|
+
spp = PROFILES[profile]["num_generations_per_prompt"]
|
|
140
|
+
local = ["uv", "run", "python", f"{nemo_rl_dir}/{LAUNCH_SCRIPT}", "--config", f"{nemo_rl_dir}/{RECIPE}", *ov]
|
|
141
|
+
if launcher == "local":
|
|
142
|
+
return {"argv": local, "cwd": nemo_rl_dir, "samples_per_position": spp}
|
|
143
|
+
if launcher == "docker":
|
|
144
|
+
# HF_HUB_OFFLINE=1 (F-22, interim rule G-20): after F-17 the base model (policy.model_name) and the
|
|
145
|
+
# data (export/) are local, so the training container needs no Hub access and must not use it.
|
|
146
|
+
flags = [
|
|
147
|
+
"--rm", "--runtime=nvidia", "--gpus", "all", f"--shm-size={DEFAULT_SHM_SIZE}",
|
|
148
|
+
"--ulimit", "memlock=-1", "--ulimit", "nofile=65535:65535",
|
|
149
|
+
"-e", "TORCH_CUDA_ARCH_LIST=8.9", "-e", "FLASHINFER_CUDA_ARCH_LIST=8.9",
|
|
150
|
+
"-e", "HF_HOME=/hf_home", "-e", "HF_HUB_OFFLINE=1", "-v", f"{hf_cache}:/hf_home", "-v", f"{work_dir}:/work",
|
|
151
|
+
"-w", CONTAINER_SRC,
|
|
152
|
+
]
|
|
153
|
+
program = ["uv", "run", "python", LAUNCH_SCRIPT, "--config", RECIPE]
|
|
154
|
+
return {"argv": ["docker", "run", *flags, image, *program, *ov], "cwd": CONTAINER_SRC,
|
|
155
|
+
"samples_per_position": spp}
|
|
156
|
+
command = shlex.join(local)
|
|
157
|
+
mounts = f"{nemo_rl_dir}:{nemo_rl_dir}"
|
|
158
|
+
if "," in command + image + mounts:
|
|
159
|
+
raise ValueError("a comma in the command, image or mounts breaks sbatch --export")
|
|
160
|
+
env = f"ALL,COMMAND={command},CONTAINER={image},MOUNTS={mounts}"
|
|
161
|
+
argv = ["sbatch", "--no-requeue", "--nodes=1", "--gres=gpu:1", f"--account={account}",
|
|
162
|
+
f"--partition={partition}", f"--time={time_limit}", f"--job-name=gatedtrain-{stage_id}-unpromotable",
|
|
163
|
+
f"--export={env}", f"{nemo_rl_dir}/ray.sub"]
|
|
164
|
+
return {"argv": argv, "cwd": nemo_rl_dir, "samples_per_position": spp}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def build_command(stage: dict, *, nemo_rl_dir: str, launcher: str, **kwargs) -> list[str]:
|
|
168
|
+
"""argv only; see `launch_spec` for the keywords and their defaults."""
|
|
169
|
+
return launch_spec(stage, nemo_rl_dir=nemo_rl_dir, launcher=launcher, **kwargs)["argv"]
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def samples_per_position_from_checkpoint(step_dir: Path) -> int:
|
|
173
|
+
"""`grpo.num_generations_per_prompt` from `step_<N>/config.yaml`, which NeMo RL writes from the
|
|
174
|
+
run config (utils/checkpoint.py:414-419). Lines per loader position for the ledger (G-7)."""
|
|
175
|
+
cfg = yaml.safe_load((Path(step_dir) / "config.yaml").read_text(encoding="utf-8"))
|
|
176
|
+
try:
|
|
177
|
+
n = cfg["grpo"]["num_generations_per_prompt"]
|
|
178
|
+
except (KeyError, TypeError) as exc:
|
|
179
|
+
raise ValueError(f"{step_dir}/config.yaml has no grpo.num_generations_per_prompt") from exc
|
|
180
|
+
if not isinstance(n, int) or isinstance(n, bool) or n < 1:
|
|
181
|
+
raise ValueError(f"{step_dir}/config.yaml: grpo.num_generations_per_prompt = {n!r}")
|
|
182
|
+
return n
|
gatedtrain/canonical.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Canonical JSON and content-addressed manifests (MVP-SPEC section 2.8).
|
|
2
|
+
|
|
3
|
+
Follows the project's format conventions (rule ids C1-C6 for canonical JSON, H1-H8 for manifests):
|
|
4
|
+
|
|
5
|
+
- canonical JSON is RFC 8785 (JCS) with rules C1-C4: UTF-8, no
|
|
6
|
+
whitespace, keys in UTF-16 code-unit order, minimal string escaping,
|
|
7
|
+
ECMAScript number form; NaN, Infinity and integers outside +/-(2^53 - 1)
|
|
8
|
+
are rejected; sets serialize as sorted arrays (C6);
|
|
9
|
+
- a manifest lists `{path, sha256, size}` for every regular file under the
|
|
10
|
+
root (H1-H2), with '/'-joined NFC relative paths (H3-H4), rejects case-fold
|
|
11
|
+
collisions (H5), sorts by UTF-8 bytes (H6), and hashes the JCS bytes of the
|
|
12
|
+
entry array (H7). The kind is recorded as `p2_filelist_sha256` (H8).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import hashlib
|
|
18
|
+
import math
|
|
19
|
+
import re
|
|
20
|
+
import unicodedata
|
|
21
|
+
from decimal import Decimal
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
MAX_SAFE_INT = 2**53 - 1
|
|
26
|
+
MANIFEST_HASH_KIND = "p2_filelist_sha256"
|
|
27
|
+
SCHEMA_VERSION = "1.0" # format rule 2A, provisional initial value
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
_LINE_END = re.compile(r"\r\n|\r|\n")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def text_lines(text: str) -> list[str]:
|
|
34
|
+
"""The lines of a JSONL or line-oriented text file: `text.splitlines()` restricted to the line ends `\\n`,
|
|
35
|
+
`\\r\\n` and `\\r` (EXT-90). `str.splitlines` also breaks on U+2028, U+2029, U+0085, \\x0b, \\x0c and \\x1c to
|
|
36
|
+
\\x1e, which JSON allows raw inside a string, so it cuts a valid row in two; Python's file iteration
|
|
37
|
+
(`ledger._count_lines`) and JSON lines loaders end a line at `\\n` only. As with `splitlines`, a final line
|
|
38
|
+
end adds no empty line."""
|
|
39
|
+
lines = _LINE_END.split(text)
|
|
40
|
+
if lines[-1] == "":
|
|
41
|
+
lines.pop()
|
|
42
|
+
return lines
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _check_str(s: str) -> None:
|
|
46
|
+
for ch in s:
|
|
47
|
+
if 0xD800 <= ord(ch) <= 0xDFFF:
|
|
48
|
+
raise ValueError("lone surrogate in string (canonical-JSON rule C3)")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _encode_str(s: str) -> str:
|
|
52
|
+
_check_str(s)
|
|
53
|
+
out = ['"']
|
|
54
|
+
for ch in s:
|
|
55
|
+
o = ord(ch)
|
|
56
|
+
if ch == '"':
|
|
57
|
+
out.append('\\"')
|
|
58
|
+
elif ch == "\\":
|
|
59
|
+
out.append("\\\\")
|
|
60
|
+
elif o < 0x20:
|
|
61
|
+
out.append({"\b": "\\b", "\t": "\\t", "\n": "\\n", "\f": "\\f", "\r": "\\r"}.get(ch, f"\\u{o:04x}"))
|
|
62
|
+
else:
|
|
63
|
+
out.append(ch)
|
|
64
|
+
out.append('"')
|
|
65
|
+
return "".join(out)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _encode_float(x: float) -> str:
|
|
69
|
+
"""ECMAScript Number.prototype.toString form (RFC 8785 section 3.2.2.3)."""
|
|
70
|
+
if math.isnan(x) or math.isinf(x):
|
|
71
|
+
raise ValueError("NaN and Infinity are not allowed in canonical JSON (canonical-JSON rule C4)")
|
|
72
|
+
if x == 0:
|
|
73
|
+
return "0" # -0 is written 0 (C4)
|
|
74
|
+
sign = "-" if x < 0 else ""
|
|
75
|
+
tup = Decimal(repr(float(abs(x)))).normalize().as_tuple() # shortest round-trip digits
|
|
76
|
+
digits = "".join(map(str, tup.digits))
|
|
77
|
+
k = len(digits)
|
|
78
|
+
n = tup.exponent + k # value = 0.digits * 10^n
|
|
79
|
+
if k <= n <= 21:
|
|
80
|
+
body = digits + "0" * (n - k)
|
|
81
|
+
elif 0 < n <= 21:
|
|
82
|
+
body = digits[:n] + "." + digits[n:]
|
|
83
|
+
elif -6 < n <= 0:
|
|
84
|
+
body = "0." + "0" * (-n) + digits
|
|
85
|
+
else:
|
|
86
|
+
e = n - 1
|
|
87
|
+
mant = digits[0] + ("." + digits[1:] if k > 1 else "")
|
|
88
|
+
body = f"{mant}e{'+' if e > 0 else '-'}{abs(e)}"
|
|
89
|
+
return sign + body
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _encode(obj: Any) -> str:
|
|
93
|
+
if obj is None:
|
|
94
|
+
return "null"
|
|
95
|
+
if obj is True:
|
|
96
|
+
return "true"
|
|
97
|
+
if obj is False:
|
|
98
|
+
return "false"
|
|
99
|
+
if isinstance(obj, int):
|
|
100
|
+
if abs(obj) > MAX_SAFE_INT:
|
|
101
|
+
raise ValueError(f"integer {obj} outside +/-(2^53 - 1) (canonical-JSON rule C4)")
|
|
102
|
+
return str(obj)
|
|
103
|
+
if isinstance(obj, float):
|
|
104
|
+
return _encode_float(obj)
|
|
105
|
+
if isinstance(obj, str):
|
|
106
|
+
return _encode_str(obj)
|
|
107
|
+
if isinstance(obj, dict):
|
|
108
|
+
items = []
|
|
109
|
+
for k in obj:
|
|
110
|
+
if not isinstance(k, str):
|
|
111
|
+
raise ValueError(f"object key {k!r} is not a string")
|
|
112
|
+
_check_str(k)
|
|
113
|
+
if any(ord(ch) > 0xFFFF for ch in k):
|
|
114
|
+
raise ValueError(f"object key {k!r} has a supplementary-plane character (canonical-JSON rule C2)")
|
|
115
|
+
for k in sorted(obj, key=lambda s: s.encode("utf-16-be")): # C2: UTF-16 code units
|
|
116
|
+
items.append(_encode_str(k) + ":" + _encode(obj[k]))
|
|
117
|
+
return "{" + ",".join(items) + "}"
|
|
118
|
+
if isinstance(obj, (list, tuple)):
|
|
119
|
+
return "[" + ",".join(_encode(v) for v in obj) + "]"
|
|
120
|
+
if isinstance(obj, (set, frozenset)):
|
|
121
|
+
enc = sorted((_encode(v) for v in obj), key=lambda s: s.encode("utf-8")) # C6
|
|
122
|
+
return "[" + ",".join(enc) + "]"
|
|
123
|
+
raise ValueError(f"type {type(obj).__name__} is not allowed in canonical JSON")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def canonical_json(obj: Any) -> bytes:
|
|
127
|
+
"""RFC 8785 bytes, no trailing newline (canonical-JSON rule C1)."""
|
|
128
|
+
return _encode(obj).encode("utf-8")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def sha256_bytes(data: bytes) -> str:
|
|
132
|
+
return "sha256:" + hashlib.sha256(data).hexdigest()
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def file_sha256(path: Path) -> str:
|
|
136
|
+
h = hashlib.sha256()
|
|
137
|
+
with Path(path).open("rb") as f:
|
|
138
|
+
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
139
|
+
h.update(chunk)
|
|
140
|
+
return "sha256:" + h.hexdigest()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _check_path(rel: str) -> None:
|
|
144
|
+
"""Manifest rule H4: reject, never rewrite."""
|
|
145
|
+
if not unicodedata.is_normalized("NFC", rel):
|
|
146
|
+
raise ValueError(f"manifest path is not NFC (manifest rule H4): {rel!r}")
|
|
147
|
+
for seg in rel.split("/"):
|
|
148
|
+
if seg in ("", ".", ".."):
|
|
149
|
+
raise ValueError(f"bad path segment (manifest rule H4): {rel!r}")
|
|
150
|
+
if "\\" in rel or any(ord(c) < 0x20 or ord(c) == 0x7F for c in rel):
|
|
151
|
+
raise ValueError(f"bad character in manifest path (manifest rule H4): {rel!r}")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def manifest_entries(root: Path, *, exclude: tuple[str, ...] = ()) -> list[dict[str, Any]]:
|
|
155
|
+
"""Sorted `{path, sha256, size}` for every regular file under root (manifest rule H1-H6).
|
|
156
|
+
|
|
157
|
+
The walk raises on a directory it cannot list (`os.walk(onerror=...)`), so an unreadable
|
|
158
|
+
subtree is an error, never silently left out of the hash; an unreadable file raises OSError.
|
|
159
|
+
`exclude` names directories directly under root that are left out entirely; an adapter declares them
|
|
160
|
+
(tests/README.md section 11.22, EXT-162: the SD3.5 base's own `.cache/`). Default: none. An excluded name that is
|
|
161
|
+
present but is a symbolic link or not a directory raises ValueError (manifest rule H2).
|
|
162
|
+
"""
|
|
163
|
+
import os
|
|
164
|
+
|
|
165
|
+
root = Path(root)
|
|
166
|
+
if not root.is_dir():
|
|
167
|
+
raise ValueError(f"manifest root is not a directory: {root}")
|
|
168
|
+
|
|
169
|
+
def fail(err: OSError) -> None:
|
|
170
|
+
raise err
|
|
171
|
+
|
|
172
|
+
for name in exclude:
|
|
173
|
+
p = root / name
|
|
174
|
+
if p.is_symlink() or (p.exists() and not p.is_dir()):
|
|
175
|
+
raise ValueError(f"the excluded {p} is not a plain directory (a symbolic link or a file; manifest rule H2)")
|
|
176
|
+
entries = []
|
|
177
|
+
folded: dict[str, str] = {}
|
|
178
|
+
top = True
|
|
179
|
+
for dirpath, dirnames, filenames in os.walk(root, onerror=fail, followlinks=False):
|
|
180
|
+
if top and exclude: # top-down walk: pruning dirnames skips the subtree
|
|
181
|
+
dirnames[:] = [d for d in dirnames if d not in exclude]
|
|
182
|
+
top = False
|
|
183
|
+
for d in dirnames:
|
|
184
|
+
if os.path.islink(os.path.join(dirpath, d)):
|
|
185
|
+
raise ValueError(f"symbolic link inside a manifest root (manifest rule H2): {Path(dirpath, d)}")
|
|
186
|
+
for name in filenames:
|
|
187
|
+
p = Path(dirpath, name)
|
|
188
|
+
if p.is_symlink():
|
|
189
|
+
raise ValueError(f"symbolic link inside a manifest root (manifest rule H2): {p}")
|
|
190
|
+
if not p.is_file():
|
|
191
|
+
raise ValueError(f"not a regular file (manifest rule H2): {p}")
|
|
192
|
+
rel = p.relative_to(root).as_posix()
|
|
193
|
+
_check_path(rel)
|
|
194
|
+
key = rel.casefold()
|
|
195
|
+
if key in folded:
|
|
196
|
+
raise ValueError(f"paths collide after case folding (manifest rule H5): {folded[key]!r}, {rel!r}")
|
|
197
|
+
folded[key] = rel
|
|
198
|
+
data = p.read_bytes()
|
|
199
|
+
entries.append({"path": rel, "sha256": hashlib.sha256(data).hexdigest(), "size": len(data)})
|
|
200
|
+
entries.sort(key=lambda e: e["path"].encode("utf-8")) # H6
|
|
201
|
+
return entries
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def entries_hash(entries: list[dict[str, Any]]) -> str:
|
|
205
|
+
"""H7: "sha256:" + hex(SHA-256(JCS(entries)))."""
|
|
206
|
+
return sha256_bytes(canonical_json(entries))
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def manifest_hash(root: Path) -> str:
|
|
210
|
+
return entries_hash(manifest_entries(root))
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def manifest_document(name: str, type_: str, root: Path, tags: tuple[str, ...],
|
|
214
|
+
entries: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
|
215
|
+
"""The manifest document of tests/README.md section 5.1. `root` is recorded, not hashed.
|
|
216
|
+
`entries` may be passed when the root was already walked (Tier B collect pre-checks)."""
|
|
217
|
+
if entries is None:
|
|
218
|
+
entries = manifest_entries(root)
|
|
219
|
+
return {
|
|
220
|
+
"schema": "gatedtrain.manifest",
|
|
221
|
+
"schema_version": SCHEMA_VERSION,
|
|
222
|
+
"artifact": name,
|
|
223
|
+
"type": type_,
|
|
224
|
+
"root": str(Path(root).resolve()),
|
|
225
|
+
"entries": entries,
|
|
226
|
+
"manifest_hash": entries_hash(entries),
|
|
227
|
+
"manifest_hash_kind": MANIFEST_HASH_KIND,
|
|
228
|
+
"tags": list(tags),
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def write_canonical(path: Path, obj: Any) -> bytes:
|
|
233
|
+
data = canonical_json(obj)
|
|
234
|
+
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
235
|
+
Path(path).write_bytes(data)
|
|
236
|
+
return data
|