gatedtrain 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. gatedtrain/__init__.py +16 -0
  2. gatedtrain/__main__.py +7 -0
  3. gatedtrain/backends/__init__.py +6 -0
  4. gatedtrain/backends/nemo_evaluator.py +34 -0
  5. gatedtrain/backends/nemo_rl.py +182 -0
  6. gatedtrain/canonical.py +236 -0
  7. gatedtrain/cli.py +370 -0
  8. gatedtrain/constraints.py +811 -0
  9. gatedtrain/errors.py +32 -0
  10. gatedtrain/ext/__init__.py +29 -0
  11. gatedtrain/ext/arm.py +360 -0
  12. gatedtrain/ext/collect.py +815 -0
  13. gatedtrain/ext/conformance.py +198 -0
  14. gatedtrain/ext/executor.py +14 -0
  15. gatedtrain/ext/fill.py +150 -0
  16. gatedtrain/ext/from_nel.py +255 -0
  17. gatedtrain/ext/launch.py +2057 -0
  18. gatedtrain/ext/merge.py +182 -0
  19. gatedtrain/ext/registry.py +179 -0
  20. gatedtrain/ext/rowkit.py +55 -0
  21. gatedtrain/ext/sdk.py +315 -0
  22. gatedtrain/ext/smoke.py +162 -0
  23. gatedtrain/families/__init__.py +5 -0
  24. gatedtrain/families/diffusers_lora.py +312 -0
  25. gatedtrain/families/flow_grpo.py +643 -0
  26. gatedtrain/families/nemo_rl.py +35 -0
  27. gatedtrain/gates.py +1002 -0
  28. gatedtrain/impact.py +348 -0
  29. gatedtrain/items.py +94 -0
  30. gatedtrain/ledger.py +247 -0
  31. gatedtrain/registration.py +1086 -0
  32. gatedtrain/renamed.py +89 -0
  33. gatedtrain/report.py +380 -0
  34. gatedtrain/runner.py +561 -0
  35. gatedtrain/spec.py +409 -0
  36. gatedtrain/stubs/__init__.py +19 -0
  37. gatedtrain/stubs/evaluator.py +88 -0
  38. gatedtrain/stubs/model.py +120 -0
  39. gatedtrain/stubs/task.py +85 -0
  40. gatedtrain/stubs/trainer.py +203 -0
  41. gatedtrain/testing/__init__.py +16 -0
  42. gatedtrain/testing/ext.py +87 -0
  43. gatedtrain/testing/support.py +108 -0
  44. gatedtrain/testing/synthetic.py +463 -0
  45. gatedtrain/tierb.py +1047 -0
  46. gatedtrain-0.3.0.dist-info/METADATA +239 -0
  47. gatedtrain-0.3.0.dist-info/RECORD +53 -0
  48. gatedtrain-0.3.0.dist-info/WHEEL +5 -0
  49. gatedtrain-0.3.0.dist-info/entry_points.txt +2 -0
  50. gatedtrain-0.3.0.dist-info/licenses/LICENSE +202 -0
  51. gatedtrain-0.3.0.dist-info/licenses/NOTICE +4 -0
  52. gatedtrain-0.3.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +48 -0
  53. gatedtrain-0.3.0.dist-info/top_level.txt +1 -0
gatedtrain/__init__.py ADDED
@@ -0,0 +1,16 @@
1
+ """Walking-skeleton prototype of the Composable Post-Training Framework.
2
+
3
+ prototype, unpromotable. The Tier A trainer and evaluator are stand-ins
4
+ marked skeleton-stub (MVP-SPEC section 3).
5
+ """
6
+
7
+ __version__ = "0.3.0"
8
+
9
+ PACKAGE = "gatedtrain"
10
+ ANALYSIS_CODE_VERSION = f"{PACKAGE} {__version__}"
11
+
12
+ # Tags carried by every output (MVP-SPEC section 2.11).
13
+ TAGS = ("prototype", "unpromotable")
14
+ REPORT_TAGS = ("prototype", "unpromotable", "registration: unsigned")
15
+ STUB_MARK = "skeleton-stub"
16
+ NOT_ASSESSED = "not-assessed (skeleton)"
gatedtrain/__main__.py ADDED
@@ -0,0 +1,7 @@
1
+ """`python -m gatedtrain` is the `gatedtrain` CLI (tests/README.md section 2)."""
2
+
3
+ import sys
4
+
5
+ from .cli import main
6
+
7
+ sys.exit(main())
@@ -0,0 +1,6 @@
1
+ """Tier B backends (GPU, run by an operator; MVP-SPEC section 4).
2
+
3
+ Code only: the test suite never runs them. `nemo_rl` builds the launch command for
4
+ one NeMo RL recipe at 4f39fd8; `nemo_evaluator` reads NeMo Evaluator
5
+ `results.jsonl` at ea35ba3. Outputs are tagged `unpromotable` (01 section 8.1).
6
+ """
@@ -0,0 +1,34 @@
1
+ """Tier B evaluator backend `nemo_evaluator` (MVP-SPEC section 4).
2
+
3
+ An operator runs NeMo Evaluator at ea35ba3 (`nel eval run`, cli/eval.py:71-88)
4
+ on the baseline and on the trained checkpoint with the registered repeats
5
+ (K) and sampler. NeMo Evaluator writes one per-sample record per line to
6
+ `<output dir>/results.jsonl` (engine/artifacts.py:96-102), with the keys of
7
+ engine/eval_loop.py:790-803. This module only reads those files; it pairs
8
+ records by (problem_idx, repeat) exactly as NeMo Evaluator's
9
+ `load_paired_records` does (engine/comparison.py:360).
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from ..gates import load_paired_records
18
+
19
+ NEMO_EVALUATOR_COMMIT = "ea35ba3"
20
+ RESULT_KEYS = ("problem_idx", "repeat", "reward", "scorer_rewards", "prompt", "model_response", "extracted_answer",
21
+ "expected_answer", "scoring_details", "metadata", "tokens", "latency_ms")
22
+
23
+
24
+ def read_results(task_dir: Path) -> dict[tuple[int, int], dict[str, Any]]:
25
+ """{(problem_idx, repeat): record} from a NeMo Evaluator task directory."""
26
+ return load_paired_records(Path(task_dir))
27
+
28
+
29
+ def build_eval_command(*, model_url: str, model_id: str, bench: str, repeats: int, temperature: float,
30
+ max_tokens: int, output_dir: str) -> list[str]:
31
+ """argv for one NeMo Evaluator run (cli/eval.py:72-88). Never executed by gatedtrain."""
32
+ return ["nel", "eval", "run", "--bench", bench, "--model-url", model_url, "--model-id", model_id,
33
+ "--repeats", str(int(repeats)), "--temperature", str(float(temperature)),
34
+ "--max-tokens", str(int(max_tokens)), "--output-dir", output_dir]
@@ -0,0 +1,182 @@
1
+ """Tier B trainer backend `nemo_rl` (MVP-SPEC section 4; TIER-B-SPEC H-5).
2
+
3
+ Builds, but never runs, the launch command for one NeMo RL recipe on
4
+ unmodified NeMo RL at 4f39fd8. Citations are file:line at 4f39fd8.
5
+
6
+ - **Recipe:** `examples/configs/grpo_smoke.yaml`. It layers on
7
+ `grpo_math_1B.yaml` (grpo_smoke.yaml:6), whose `cluster.gpus_per_node: 1`
8
+ and `num_nodes: 1` are at grpo_math_1B.yaml:690-692. Model
9
+ `Qwen/Qwen3-0.6B` (grpo_smoke.yaml:15).
10
+ - **Launch script:** `examples/run_grpo.py` with `--config` plus Hydra
11
+ overrides (run_grpo.py:53-63, :86-87). Hydra struct mode is on
12
+ (nemo_rl/utils/config.py:188), so keys the recipe lacks need `++`.
13
+
14
+ Overrides (`COMMON_OVERRIDES`):
15
+
16
+ | Override | Why |
17
+ |---|---|
18
+ | `data_plane.enabled=true` | selects `grpo_train_sync` (nemo_rl/data_plane/factory.py:51-55), whose outputs the ledger reads (grpo_sync.py:1323, :1353, :1407-1414, :1458-1460) |
19
+ | `data.shuffle=false` | file order (grpo_math_1B.yaml:617; StatefulDataLoader, grpo.py:883-890) |
20
+ | `checkpointing.enabled=true` | the smoke recipe turns checkpoints off (grpo_smoke.yaml:12-13); `consumed_samples` lives in them (grpo_sync.py:1353) |
21
+ | `checkpointing.keep_top_k=null` | keep every cut for the ledger (checkpoint.py:234 `NotRequired[int \\| None]`; deletion at :590-631) |
22
+ | `checkpointing.checkpoint_dir`, `logger.log_dir` | the stage's output directory (grpo_math_1B.yaml:138, :665) |
23
+ | `data.train.dataset_name=ResponseDataset` | local JSONL loader (response_datasets/__init__.py:99; response_dataset.py:32-61) |
24
+ | `++data.train.data_path`, `++data.train.input_key`, `++data.train.output_key` | ResponseDataset arguments absent from the recipe, so `++` (response_dataset.py:42-48) |
25
+ | `data.train.split_validation_size=0` | no validation split reorders rows (response_dataset.py:76) |
26
+ | `data.default.prompt_file` | NeMo Evaluator's GSM8K prompt, written by `export-gsm8k` (G-15; processors.py:574-576) |
27
+ | `policy.tokenizer.chat_template_kwargs={enable_thinking:false}` | Qwen3 thinking off (G-15; grpo_math_1B.yaml:158; nemo_rl/algorithms/utils.py:465-496) |
28
+ | `policy.automodel_cfg.checkpoint.save_consolidated=final` | HF safetensors under `step_<last>/policy/weights/model/consolidated` for serving (grpo_math_1B.yaml:178-181; Automodel config.py:64 `FINAL = "final"`) |
29
+
30
+ Profiles (`PROFILES`): `smoke` keeps the recipe's 10 steps of 2 x 4
31
+ (grpo_smoke.yaml:8-10); `long` is 150 steps of 32 x 8 with a global batch of
32
+ 256 (an estimate, not a NeMo RL recipe; docs/tier-b-runbook.md step 7, the long run).
33
+ `samples_per_position` is the profile's `grpo.num_generations_per_prompt`;
34
+ after a run, read it from the checkpoint with
35
+ `samples_per_position_from_checkpoint` (utils/checkpoint.py:414-419), never
36
+ from a constant (interim rule G-7).
37
+
38
+ Launchers:
39
+
40
+ - `local`: absolute script and recipe, run with cwd `nemo_rl_dir`, because the
41
+ recipe has relative paths (grpo_math_1B.yaml:632);
42
+ - `docker`: `docker run` with the documented flags
43
+ (docs/about/installation.md:21-37), `nofile=65535` for Ray (ray.sub:161-182),
44
+ `TORCH_CUDA_ARCH_LIST=8.9` and `FLASHINFER_CUDA_ARCH_LIST=8.9` because the
45
+ image targets 9.0 and 10.0 (Dockerfile:231), the image's own source at
46
+ `/opt/nemo-rl` with no checkout mounted over it (the fingerprint check,
47
+ nemo_rl/__init__.py:105), and `-w /opt/nemo-rl`;
48
+ - `sbatch`: `sbatch --no-requeue` around NeMo RL's `ray.sub`, driven by
49
+ COMMAND, CONTAINER and MOUNTS (ray.sub:110-112; docs/cluster.md:10-24),
50
+ with COMMAND equal to the local argv. R9-OI-1
51
+ exposure: an administrator or an automated health-check service can still requeue the job.
52
+
53
+ Outputs are tagged `unpromotable`. The nemo_gym-responses path writes no step
54
+ log (grpo_sync.py:1428) and is unsupported.
55
+ """
56
+
57
+ from __future__ import annotations
58
+
59
+ import shlex
60
+ from pathlib import Path
61
+
62
+ import yaml
63
+
64
+ NEMO_RL_COMMIT = "4f39fd8"
65
+ RECIPE = "examples/configs/grpo_smoke.yaml"
66
+ LAUNCH_SCRIPT = "examples/run_grpo.py"
67
+ CONTAINER_SRC = "/opt/nemo-rl"
68
+ TAGS = ("prototype", "unpromotable")
69
+ LAUNCHERS = ("local", "docker", "sbatch")
70
+
71
+ # Documented defaults for the keywords added in TIER-B-SPEC H-5.
72
+ DEFAULT_PROFILE = "smoke"
73
+ DEFAULT_IMAGE = "nemo-rl:4f39fd8"
74
+ DEFAULT_WORK_DIR = "/data/gatedtrain-tierb" # host directory mounted at /work by the docker launcher
75
+ DEFAULT_HF_CACHE = "/data/hf-cache" # host Hugging Face cache mounted at /hf_home
76
+ DEFAULT_SHM_SIZE = "32g" # keep at or below host RAM (docs/about/installation.md:25 uses 64g)
77
+
78
+ COMMON_OVERRIDES = (
79
+ "data_plane.enabled=true",
80
+ "data.shuffle=false",
81
+ "checkpointing.enabled=true",
82
+ "checkpointing.keep_top_k=null",
83
+ "checkpointing.checkpoint_dir={out}/checkpoints",
84
+ "logger.log_dir={out}/logs",
85
+ "data.train.dataset_name=ResponseDataset",
86
+ "++data.train.data_path={export}/train.jsonl",
87
+ "++data.train.input_key=question",
88
+ "++data.train.output_key=answer",
89
+ "data.train.split_validation_size=0",
90
+ "data.default.prompt_file={export}/nel_gsm8k_prompt.txt",
91
+ "policy.tokenizer.chat_template_kwargs={{enable_thinking:false}}",
92
+ "policy.automodel_cfg.checkpoint.save_consolidated=final",
93
+ )
94
+
95
+ PROFILES = {
96
+ "smoke": {"max_num_steps": 10, "save_period": 5, "num_prompts_per_step": 2, "num_generations_per_prompt": 4,
97
+ "extra": ()},
98
+ "long": {"max_num_steps": 150, "save_period": 25, "num_prompts_per_step": 32, "num_generations_per_prompt": 8,
99
+ "extra": ("policy.train_global_batch_size=256", "policy.train_micro_batch_size=8")},
100
+ }
101
+
102
+
103
+ def overrides(*, out_dir: str, export_dir: str, profile: str, base_model: str | None = None) -> list[str]:
104
+ """Hydra overrides for run_grpo.py: the common set, `policy.model_name` (F-17) and the profile's."""
105
+ if profile not in PROFILES:
106
+ raise ValueError(f"unknown profile {profile!r}; use one of {sorted(PROFILES)}")
107
+ p = PROFILES[profile]
108
+ out = [o.format(out=out_dir, export=export_dir) for o in COMMON_OVERRIDES]
109
+ if base_model is not None:
110
+ # F-17: train from the base-model directory whose manifest the gate records. The key is in
111
+ # the recipe (grpo_math_1B.yaml:155; grpo_smoke.yaml:15); the tokenizer follows it
112
+ # (`name: ${policy.model_name}`, grpo_math_1B.yaml:157). NeMo RL loads it with from_pretrained
113
+ # (nemo_rl/models/automodel/setup.py:374-375, :941-942) and passes it to vLLM (grpo.py:1426).
114
+ out.append(f"policy.model_name={base_model}")
115
+ out += [f"grpo.max_num_steps={p['max_num_steps']}", f"checkpointing.save_period={p['save_period']}",
116
+ f"grpo.num_prompts_per_step={p['num_prompts_per_step']}",
117
+ f"grpo.num_generations_per_prompt={p['num_generations_per_prompt']}", *p["extra"]]
118
+ return out
119
+
120
+
121
+ def launch_spec(stage: dict, *, nemo_rl_dir: str, launcher: str, out_dir: str | None = None,
122
+ export_dir: str = "/work/export", profile: str = DEFAULT_PROFILE, image: str = DEFAULT_IMAGE,
123
+ work_dir: str = DEFAULT_WORK_DIR, hf_cache: str = DEFAULT_HF_CACHE, base_model: str | None = None,
124
+ account: str = "<account>", partition: str = "<partition>", time_limit: str = "01:00:00") -> dict:
125
+ """{"argv", "cwd", "samples_per_position"} for one Tier B stage. Never executed by gatedtrain.
126
+
127
+ `out_dir` defaults to `<nemo_rl_dir>/results/gatedtrain/<stage id>`. For the docker launcher,
128
+ `out_dir`, `export_dir` and `base_model` are paths inside the container (under /work).
129
+ `base_model` is the spec's `base` directory as the trainer sees it; it becomes
130
+ `policy.model_name` (F-17). Pass it for every gated run. `None`, the default kept for older
131
+ callers, leaves the recipe's `Qwen/Qwen3-0.6B`, and the gate's baseline then is not the
132
+ trained parent.
133
+ """
134
+ if launcher not in LAUNCHERS:
135
+ raise ValueError(f"unknown launcher {launcher!r}; use one of {LAUNCHERS} (NeMo-Run is not supported)")
136
+ stage_id = str(stage.get("id", "stage"))
137
+ out = out_dir or f"{nemo_rl_dir}/results/gatedtrain/{stage_id}"
138
+ ov = overrides(out_dir=out, export_dir=export_dir, profile=profile, base_model=base_model)
139
+ spp = PROFILES[profile]["num_generations_per_prompt"]
140
+ local = ["uv", "run", "python", f"{nemo_rl_dir}/{LAUNCH_SCRIPT}", "--config", f"{nemo_rl_dir}/{RECIPE}", *ov]
141
+ if launcher == "local":
142
+ return {"argv": local, "cwd": nemo_rl_dir, "samples_per_position": spp}
143
+ if launcher == "docker":
144
+ # HF_HUB_OFFLINE=1 (F-22, interim rule G-20): after F-17 the base model (policy.model_name) and the
145
+ # data (export/) are local, so the training container needs no Hub access and must not use it.
146
+ flags = [
147
+ "--rm", "--runtime=nvidia", "--gpus", "all", f"--shm-size={DEFAULT_SHM_SIZE}",
148
+ "--ulimit", "memlock=-1", "--ulimit", "nofile=65535:65535",
149
+ "-e", "TORCH_CUDA_ARCH_LIST=8.9", "-e", "FLASHINFER_CUDA_ARCH_LIST=8.9",
150
+ "-e", "HF_HOME=/hf_home", "-e", "HF_HUB_OFFLINE=1", "-v", f"{hf_cache}:/hf_home", "-v", f"{work_dir}:/work",
151
+ "-w", CONTAINER_SRC,
152
+ ]
153
+ program = ["uv", "run", "python", LAUNCH_SCRIPT, "--config", RECIPE]
154
+ return {"argv": ["docker", "run", *flags, image, *program, *ov], "cwd": CONTAINER_SRC,
155
+ "samples_per_position": spp}
156
+ command = shlex.join(local)
157
+ mounts = f"{nemo_rl_dir}:{nemo_rl_dir}"
158
+ if "," in command + image + mounts:
159
+ raise ValueError("a comma in the command, image or mounts breaks sbatch --export")
160
+ env = f"ALL,COMMAND={command},CONTAINER={image},MOUNTS={mounts}"
161
+ argv = ["sbatch", "--no-requeue", "--nodes=1", "--gres=gpu:1", f"--account={account}",
162
+ f"--partition={partition}", f"--time={time_limit}", f"--job-name=gatedtrain-{stage_id}-unpromotable",
163
+ f"--export={env}", f"{nemo_rl_dir}/ray.sub"]
164
+ return {"argv": argv, "cwd": nemo_rl_dir, "samples_per_position": spp}
165
+
166
+
167
+ def build_command(stage: dict, *, nemo_rl_dir: str, launcher: str, **kwargs) -> list[str]:
168
+ """argv only; see `launch_spec` for the keywords and their defaults."""
169
+ return launch_spec(stage, nemo_rl_dir=nemo_rl_dir, launcher=launcher, **kwargs)["argv"]
170
+
171
+
172
+ def samples_per_position_from_checkpoint(step_dir: Path) -> int:
173
+ """`grpo.num_generations_per_prompt` from `step_<N>/config.yaml`, which NeMo RL writes from the
174
+ run config (utils/checkpoint.py:414-419). Lines per loader position for the ledger (G-7)."""
175
+ cfg = yaml.safe_load((Path(step_dir) / "config.yaml").read_text(encoding="utf-8"))
176
+ try:
177
+ n = cfg["grpo"]["num_generations_per_prompt"]
178
+ except (KeyError, TypeError) as exc:
179
+ raise ValueError(f"{step_dir}/config.yaml has no grpo.num_generations_per_prompt") from exc
180
+ if not isinstance(n, int) or isinstance(n, bool) or n < 1:
181
+ raise ValueError(f"{step_dir}/config.yaml: grpo.num_generations_per_prompt = {n!r}")
182
+ return n
@@ -0,0 +1,236 @@
1
+ """Canonical JSON and content-addressed manifests (MVP-SPEC section 2.8).
2
+
3
+ Follows the project's format conventions (rule ids C1-C6 for canonical JSON, H1-H8 for manifests):
4
+
5
+ - canonical JSON is RFC 8785 (JCS) with rules C1-C4: UTF-8, no
6
+ whitespace, keys in UTF-16 code-unit order, minimal string escaping,
7
+ ECMAScript number form; NaN, Infinity and integers outside +/-(2^53 - 1)
8
+ are rejected; sets serialize as sorted arrays (C6);
9
+ - a manifest lists `{path, sha256, size}` for every regular file under the
10
+ root (H1-H2), with '/'-joined NFC relative paths (H3-H4), rejects case-fold
11
+ collisions (H5), sorts by UTF-8 bytes (H6), and hashes the JCS bytes of the
12
+ entry array (H7). The kind is recorded as `p2_filelist_sha256` (H8).
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import hashlib
18
+ import math
19
+ import re
20
+ import unicodedata
21
+ from decimal import Decimal
22
+ from pathlib import Path
23
+ from typing import Any
24
+
25
+ MAX_SAFE_INT = 2**53 - 1
26
+ MANIFEST_HASH_KIND = "p2_filelist_sha256"
27
+ SCHEMA_VERSION = "1.0" # format rule 2A, provisional initial value
28
+
29
+
30
+ _LINE_END = re.compile(r"\r\n|\r|\n")
31
+
32
+
33
+ def text_lines(text: str) -> list[str]:
34
+ """The lines of a JSONL or line-oriented text file: `text.splitlines()` restricted to the line ends `\\n`,
35
+ `\\r\\n` and `\\r` (EXT-90). `str.splitlines` also breaks on U+2028, U+2029, U+0085, \\x0b, \\x0c and \\x1c to
36
+ \\x1e, which JSON allows raw inside a string, so it cuts a valid row in two; Python's file iteration
37
+ (`ledger._count_lines`) and JSON lines loaders end a line at `\\n` only. As with `splitlines`, a final line
38
+ end adds no empty line."""
39
+ lines = _LINE_END.split(text)
40
+ if lines[-1] == "":
41
+ lines.pop()
42
+ return lines
43
+
44
+
45
+ def _check_str(s: str) -> None:
46
+ for ch in s:
47
+ if 0xD800 <= ord(ch) <= 0xDFFF:
48
+ raise ValueError("lone surrogate in string (canonical-JSON rule C3)")
49
+
50
+
51
+ def _encode_str(s: str) -> str:
52
+ _check_str(s)
53
+ out = ['"']
54
+ for ch in s:
55
+ o = ord(ch)
56
+ if ch == '"':
57
+ out.append('\\"')
58
+ elif ch == "\\":
59
+ out.append("\\\\")
60
+ elif o < 0x20:
61
+ out.append({"\b": "\\b", "\t": "\\t", "\n": "\\n", "\f": "\\f", "\r": "\\r"}.get(ch, f"\\u{o:04x}"))
62
+ else:
63
+ out.append(ch)
64
+ out.append('"')
65
+ return "".join(out)
66
+
67
+
68
+ def _encode_float(x: float) -> str:
69
+ """ECMAScript Number.prototype.toString form (RFC 8785 section 3.2.2.3)."""
70
+ if math.isnan(x) or math.isinf(x):
71
+ raise ValueError("NaN and Infinity are not allowed in canonical JSON (canonical-JSON rule C4)")
72
+ if x == 0:
73
+ return "0" # -0 is written 0 (C4)
74
+ sign = "-" if x < 0 else ""
75
+ tup = Decimal(repr(float(abs(x)))).normalize().as_tuple() # shortest round-trip digits
76
+ digits = "".join(map(str, tup.digits))
77
+ k = len(digits)
78
+ n = tup.exponent + k # value = 0.digits * 10^n
79
+ if k <= n <= 21:
80
+ body = digits + "0" * (n - k)
81
+ elif 0 < n <= 21:
82
+ body = digits[:n] + "." + digits[n:]
83
+ elif -6 < n <= 0:
84
+ body = "0." + "0" * (-n) + digits
85
+ else:
86
+ e = n - 1
87
+ mant = digits[0] + ("." + digits[1:] if k > 1 else "")
88
+ body = f"{mant}e{'+' if e > 0 else '-'}{abs(e)}"
89
+ return sign + body
90
+
91
+
92
+ def _encode(obj: Any) -> str:
93
+ if obj is None:
94
+ return "null"
95
+ if obj is True:
96
+ return "true"
97
+ if obj is False:
98
+ return "false"
99
+ if isinstance(obj, int):
100
+ if abs(obj) > MAX_SAFE_INT:
101
+ raise ValueError(f"integer {obj} outside +/-(2^53 - 1) (canonical-JSON rule C4)")
102
+ return str(obj)
103
+ if isinstance(obj, float):
104
+ return _encode_float(obj)
105
+ if isinstance(obj, str):
106
+ return _encode_str(obj)
107
+ if isinstance(obj, dict):
108
+ items = []
109
+ for k in obj:
110
+ if not isinstance(k, str):
111
+ raise ValueError(f"object key {k!r} is not a string")
112
+ _check_str(k)
113
+ if any(ord(ch) > 0xFFFF for ch in k):
114
+ raise ValueError(f"object key {k!r} has a supplementary-plane character (canonical-JSON rule C2)")
115
+ for k in sorted(obj, key=lambda s: s.encode("utf-16-be")): # C2: UTF-16 code units
116
+ items.append(_encode_str(k) + ":" + _encode(obj[k]))
117
+ return "{" + ",".join(items) + "}"
118
+ if isinstance(obj, (list, tuple)):
119
+ return "[" + ",".join(_encode(v) for v in obj) + "]"
120
+ if isinstance(obj, (set, frozenset)):
121
+ enc = sorted((_encode(v) for v in obj), key=lambda s: s.encode("utf-8")) # C6
122
+ return "[" + ",".join(enc) + "]"
123
+ raise ValueError(f"type {type(obj).__name__} is not allowed in canonical JSON")
124
+
125
+
126
+ def canonical_json(obj: Any) -> bytes:
127
+ """RFC 8785 bytes, no trailing newline (canonical-JSON rule C1)."""
128
+ return _encode(obj).encode("utf-8")
129
+
130
+
131
+ def sha256_bytes(data: bytes) -> str:
132
+ return "sha256:" + hashlib.sha256(data).hexdigest()
133
+
134
+
135
+ def file_sha256(path: Path) -> str:
136
+ h = hashlib.sha256()
137
+ with Path(path).open("rb") as f:
138
+ for chunk in iter(lambda: f.read(1 << 20), b""):
139
+ h.update(chunk)
140
+ return "sha256:" + h.hexdigest()
141
+
142
+
143
+ def _check_path(rel: str) -> None:
144
+ """Manifest rule H4: reject, never rewrite."""
145
+ if not unicodedata.is_normalized("NFC", rel):
146
+ raise ValueError(f"manifest path is not NFC (manifest rule H4): {rel!r}")
147
+ for seg in rel.split("/"):
148
+ if seg in ("", ".", ".."):
149
+ raise ValueError(f"bad path segment (manifest rule H4): {rel!r}")
150
+ if "\\" in rel or any(ord(c) < 0x20 or ord(c) == 0x7F for c in rel):
151
+ raise ValueError(f"bad character in manifest path (manifest rule H4): {rel!r}")
152
+
153
+
154
+ def manifest_entries(root: Path, *, exclude: tuple[str, ...] = ()) -> list[dict[str, Any]]:
155
+ """Sorted `{path, sha256, size}` for every regular file under root (manifest rule H1-H6).
156
+
157
+ The walk raises on a directory it cannot list (`os.walk(onerror=...)`), so an unreadable
158
+ subtree is an error, never silently left out of the hash; an unreadable file raises OSError.
159
+ `exclude` names directories directly under root that are left out entirely; an adapter declares them
160
+ (tests/README.md section 11.22, EXT-162: the SD3.5 base's own `.cache/`). Default: none. An excluded name that is
161
+ present but is a symbolic link or not a directory raises ValueError (manifest rule H2).
162
+ """
163
+ import os
164
+
165
+ root = Path(root)
166
+ if not root.is_dir():
167
+ raise ValueError(f"manifest root is not a directory: {root}")
168
+
169
+ def fail(err: OSError) -> None:
170
+ raise err
171
+
172
+ for name in exclude:
173
+ p = root / name
174
+ if p.is_symlink() or (p.exists() and not p.is_dir()):
175
+ raise ValueError(f"the excluded {p} is not a plain directory (a symbolic link or a file; manifest rule H2)")
176
+ entries = []
177
+ folded: dict[str, str] = {}
178
+ top = True
179
+ for dirpath, dirnames, filenames in os.walk(root, onerror=fail, followlinks=False):
180
+ if top and exclude: # top-down walk: pruning dirnames skips the subtree
181
+ dirnames[:] = [d for d in dirnames if d not in exclude]
182
+ top = False
183
+ for d in dirnames:
184
+ if os.path.islink(os.path.join(dirpath, d)):
185
+ raise ValueError(f"symbolic link inside a manifest root (manifest rule H2): {Path(dirpath, d)}")
186
+ for name in filenames:
187
+ p = Path(dirpath, name)
188
+ if p.is_symlink():
189
+ raise ValueError(f"symbolic link inside a manifest root (manifest rule H2): {p}")
190
+ if not p.is_file():
191
+ raise ValueError(f"not a regular file (manifest rule H2): {p}")
192
+ rel = p.relative_to(root).as_posix()
193
+ _check_path(rel)
194
+ key = rel.casefold()
195
+ if key in folded:
196
+ raise ValueError(f"paths collide after case folding (manifest rule H5): {folded[key]!r}, {rel!r}")
197
+ folded[key] = rel
198
+ data = p.read_bytes()
199
+ entries.append({"path": rel, "sha256": hashlib.sha256(data).hexdigest(), "size": len(data)})
200
+ entries.sort(key=lambda e: e["path"].encode("utf-8")) # H6
201
+ return entries
202
+
203
+
204
+ def entries_hash(entries: list[dict[str, Any]]) -> str:
205
+ """H7: "sha256:" + hex(SHA-256(JCS(entries)))."""
206
+ return sha256_bytes(canonical_json(entries))
207
+
208
+
209
+ def manifest_hash(root: Path) -> str:
210
+ return entries_hash(manifest_entries(root))
211
+
212
+
213
+ def manifest_document(name: str, type_: str, root: Path, tags: tuple[str, ...],
214
+ entries: list[dict[str, Any]] | None = None) -> dict[str, Any]:
215
+ """The manifest document of tests/README.md section 5.1. `root` is recorded, not hashed.
216
+ `entries` may be passed when the root was already walked (Tier B collect pre-checks)."""
217
+ if entries is None:
218
+ entries = manifest_entries(root)
219
+ return {
220
+ "schema": "gatedtrain.manifest",
221
+ "schema_version": SCHEMA_VERSION,
222
+ "artifact": name,
223
+ "type": type_,
224
+ "root": str(Path(root).resolve()),
225
+ "entries": entries,
226
+ "manifest_hash": entries_hash(entries),
227
+ "manifest_hash_kind": MANIFEST_HASH_KIND,
228
+ "tags": list(tags),
229
+ }
230
+
231
+
232
+ def write_canonical(path: Path, obj: Any) -> bytes:
233
+ data = canonical_json(obj)
234
+ Path(path).parent.mkdir(parents=True, exist_ok=True)
235
+ Path(path).write_bytes(data)
236
+ return data