dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
dromeus/application.py
ADDED
|
@@ -0,0 +1,523 @@
|
|
|
1
|
+
"""Turn a training task into a shared draft and a node's prepared training."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import importlib.metadata
|
|
7
|
+
import inspect
|
|
8
|
+
import json
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from types import ModuleType
|
|
12
|
+
from typing import Any, Literal
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
from pydantic import ValidationError
|
|
16
|
+
from torch.utils.data import Dataset
|
|
17
|
+
|
|
18
|
+
from dromeus.errors import ConfigError, DraftError, PrivacyError, TaskError
|
|
19
|
+
from dromeus.manifests.canonical import (
|
|
20
|
+
canonical_hash,
|
|
21
|
+
parse_draft_yaml,
|
|
22
|
+
validate_sealed_draft,
|
|
23
|
+
)
|
|
24
|
+
from dromeus.manifests.models import (
|
|
25
|
+
ApplicationTaskContract,
|
|
26
|
+
ArtifactCodec,
|
|
27
|
+
ConsensusSketchConfig,
|
|
28
|
+
DivergencePolicy,
|
|
29
|
+
DraftRunSpec,
|
|
30
|
+
EnvironmentFingerprint,
|
|
31
|
+
NoLoCoConfig,
|
|
32
|
+
PrivacyPolicy,
|
|
33
|
+
TaskTrainingSpec,
|
|
34
|
+
TensorSchema,
|
|
35
|
+
TransportLimits,
|
|
36
|
+
)
|
|
37
|
+
from dromeus.membership.formation import FormationResult
|
|
38
|
+
from dromeus.runtime import (
|
|
39
|
+
FailureConfig,
|
|
40
|
+
MetricsService,
|
|
41
|
+
TrainingConfig,
|
|
42
|
+
build_algorithm,
|
|
43
|
+
)
|
|
44
|
+
from dromeus.training.privacy import ReservationStore
|
|
45
|
+
from dromeus.training.private_model import describe_private_model
|
|
46
|
+
from dromeus.training.private_trainer import PrivateTrainer
|
|
47
|
+
from dromeus.training.state import InitialCheckpoint
|
|
48
|
+
from dromeus.training.task import Optimizer, TrainingTask
|
|
49
|
+
from dromeus.training.trainer import PyTorchTrainer
|
|
50
|
+
from dromeus.training.versions import torch_release
|
|
51
|
+
|
|
52
|
+
DEFAULT_TRANSPORT = TransportLimits(
|
|
53
|
+
max_payload_bytes=8 * 1024 * 1024,
|
|
54
|
+
max_retries=3,
|
|
55
|
+
retry_timeout_seconds=5.0,
|
|
56
|
+
chunk_size_bytes=64 * 1024,
|
|
57
|
+
window_size=2,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def definition_hash(definition: str) -> str:
|
|
62
|
+
"""Identify an application-supplied semantic definition."""
|
|
63
|
+
return hashlib.sha256(definition.encode()).hexdigest()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def code_hash(*modules: ModuleType | Path) -> str:
|
|
67
|
+
"""Identify the code that implements a task, for `implementation_hash`.
|
|
68
|
+
|
|
69
|
+
Hashes the source files of the given modules (or paths) in order. Every node
|
|
70
|
+
must install identical files. Agreement shows the files match; it does not
|
|
71
|
+
verify what the code does.
|
|
72
|
+
"""
|
|
73
|
+
if not modules:
|
|
74
|
+
raise TaskError("code_hash needs at least one module or path")
|
|
75
|
+
digest = hashlib.sha256()
|
|
76
|
+
for module in modules:
|
|
77
|
+
if isinstance(module, Path):
|
|
78
|
+
path = module
|
|
79
|
+
else:
|
|
80
|
+
source = inspect.getsourcefile(module)
|
|
81
|
+
if source is None:
|
|
82
|
+
raise TaskError(f"module has no source file: {module.__name__}")
|
|
83
|
+
path = Path(source)
|
|
84
|
+
data = path.read_bytes()
|
|
85
|
+
digest.update(len(data).to_bytes(8, "big"))
|
|
86
|
+
digest.update(data)
|
|
87
|
+
return digest.hexdigest()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True)
|
|
91
|
+
class NoLoCo:
|
|
92
|
+
"""NoLoCo outer-step settings shared by the group.
|
|
93
|
+
|
|
94
|
+
Set `top_k_fraction` to compress updates: the outer gradient sends its largest
|
|
95
|
+
entries as int8 and slow weights are sent as int8. None sends exact values.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
alpha: float = 0.5
|
|
99
|
+
beta: float = 0.7
|
|
100
|
+
gamma: float = 0.7
|
|
101
|
+
top_k_fraction: float | None = None
|
|
102
|
+
|
|
103
|
+
def codecs(self) -> tuple[ArtifactCodec, ...]:
|
|
104
|
+
if self.top_k_fraction is None:
|
|
105
|
+
return (
|
|
106
|
+
ArtifactCodec(artifact_name="outer_gradient", codec_id="identity"),
|
|
107
|
+
ArtifactCodec(artifact_name="slow_weights", codec_id="identity"),
|
|
108
|
+
)
|
|
109
|
+
return (
|
|
110
|
+
ArtifactCodec(
|
|
111
|
+
artifact_name="outer_gradient",
|
|
112
|
+
codec_id="topk-bitmap-int8",
|
|
113
|
+
top_k_fraction=self.top_k_fraction,
|
|
114
|
+
lossy_allowed=True,
|
|
115
|
+
),
|
|
116
|
+
ArtifactCodec(
|
|
117
|
+
artifact_name="slow_weights",
|
|
118
|
+
codec_id="dense-int8",
|
|
119
|
+
lossy_allowed=True,
|
|
120
|
+
),
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass(frozen=True)
|
|
125
|
+
class Privacy:
|
|
126
|
+
"""Record-level differential privacy for every node's local training.
|
|
127
|
+
|
|
128
|
+
The privacy horizon is the run's total local steps (rounds x local steps).
|
|
129
|
+
Use `randomness="reproducible_public_benchmark"` only for seeded research
|
|
130
|
+
runs on public data; real private runs use the default `"secure"`.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
max_grad_norm: float
|
|
134
|
+
noise_multiplier: float
|
|
135
|
+
delta: float
|
|
136
|
+
epsilon_limit: float | None = None
|
|
137
|
+
randomness: Literal["secure", "reproducible_public_benchmark"] = "secure"
|
|
138
|
+
|
|
139
|
+
def policy(self, max_logical_steps: int) -> PrivacyPolicy:
|
|
140
|
+
try:
|
|
141
|
+
return PrivacyPolicy(
|
|
142
|
+
max_grad_norm=self.max_grad_norm,
|
|
143
|
+
noise_multiplier=self.noise_multiplier,
|
|
144
|
+
delta=self.delta,
|
|
145
|
+
max_logical_steps=max_logical_steps,
|
|
146
|
+
epsilon_limit=self.epsilon_limit,
|
|
147
|
+
randomness_profile=self.randomness,
|
|
148
|
+
)
|
|
149
|
+
except ValidationError as error:
|
|
150
|
+
raise PrivacyError(f"invalid privacy settings: {error}") from error
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True)
|
|
154
|
+
class LocalData:
|
|
155
|
+
"""This node's own records and limits; never sent to peers.
|
|
156
|
+
|
|
157
|
+
`data_identity` names this exact copy of the records (for example a file
|
|
158
|
+
hash); checkpoints and privacy spending are bound to it. A private run needs
|
|
159
|
+
`ledger`, the node's durable privacy-spending record. `physical_batch_size`
|
|
160
|
+
caps how many records are processed at once. `research_seed` makes ordinary
|
|
161
|
+
or public-benchmark runs reproducible; secure private runs refuse it.
|
|
162
|
+
"""
|
|
163
|
+
|
|
164
|
+
dataset: Dataset[Any]
|
|
165
|
+
data_identity: str
|
|
166
|
+
ledger: ReservationStore | None = None
|
|
167
|
+
physical_batch_size: int = 1
|
|
168
|
+
research_seed: int | None = None
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def installed_versions() -> tuple[str, str]:
|
|
172
|
+
"""Dromeus and PyTorch versions installed on this machine."""
|
|
173
|
+
return importlib.metadata.version("dromeus"), torch_release()
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def describe_task(
|
|
177
|
+
task: TrainingTask, *, sampling: Literal["poisson", "shuffle"]
|
|
178
|
+
) -> TaskTrainingSpec:
|
|
179
|
+
"""The shared training specification for a task, without opening records."""
|
|
180
|
+
evidence = task.initialization_evidence
|
|
181
|
+
provenance = evidence.provenance if evidence is not None else None
|
|
182
|
+
task.check_initialization(task.initialization_mode, provenance)
|
|
183
|
+
model = task.model_factory()
|
|
184
|
+
report = describe_private_model(model, gradient_backend=task.gradient_backend)
|
|
185
|
+
optimizer = task.optimizer
|
|
186
|
+
if isinstance(optimizer, Optimizer):
|
|
187
|
+
selected = optimizer.specification(report.trainable_parameter_names)
|
|
188
|
+
else:
|
|
189
|
+
selected = optimizer.describe(model)
|
|
190
|
+
try:
|
|
191
|
+
return TaskTrainingSpec(
|
|
192
|
+
optimizer=selected,
|
|
193
|
+
batch_size=task.batch_size,
|
|
194
|
+
sampling=sampling,
|
|
195
|
+
schedule=task.schedule,
|
|
196
|
+
initialization=task.initialization_mode,
|
|
197
|
+
initialization_provenance=provenance,
|
|
198
|
+
evaluation_provenance="public_fixed_holdout"
|
|
199
|
+
if task.public_evaluate
|
|
200
|
+
else "disabled",
|
|
201
|
+
task_definition_hash=definition_hash(task.task_definition),
|
|
202
|
+
implementation_hash=task.implementation_hash,
|
|
203
|
+
batch_definition_hash=task.batch_adapter.definition_hash,
|
|
204
|
+
loss_definition_hash=definition_hash(task.loss_definition),
|
|
205
|
+
record_unit=task.record_unit,
|
|
206
|
+
gradient_backend=report.gradient_backend,
|
|
207
|
+
trainable_parameter_names=report.trainable_parameter_names,
|
|
208
|
+
parameter_aliases=report.parameter_aliases,
|
|
209
|
+
public_buffer_hashes=dict(report.public_buffer_hashes),
|
|
210
|
+
frozen_parameter_hashes=dict(report.frozen_parameter_hashes),
|
|
211
|
+
)
|
|
212
|
+
except ValidationError as error:
|
|
213
|
+
raise TaskError(f"task cannot be described: {error}") from error
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def draft_for_task(
|
|
217
|
+
task: TrainingTask,
|
|
218
|
+
*,
|
|
219
|
+
run_id: str,
|
|
220
|
+
participants: int,
|
|
221
|
+
rounds: int,
|
|
222
|
+
local_steps: int,
|
|
223
|
+
axl_version: str,
|
|
224
|
+
noloco: NoLoCo | None = None,
|
|
225
|
+
privacy: Privacy | None = None,
|
|
226
|
+
transport: TransportLimits | None = None,
|
|
227
|
+
divergence: DivergencePolicy | None = None,
|
|
228
|
+
seed: int = 0,
|
|
229
|
+
) -> DraftRunSpec:
|
|
230
|
+
"""Build the shared draft for a task; every hash is computed here."""
|
|
231
|
+
noloco = noloco or NoLoCo()
|
|
232
|
+
dromeus_version, pytorch_version = installed_versions()
|
|
233
|
+
model_hash = definition_hash(task.model_definition)
|
|
234
|
+
training = describe_task(task, sampling="poisson" if privacy else "shuffle")
|
|
235
|
+
try:
|
|
236
|
+
return DraftRunSpec(
|
|
237
|
+
run_id=run_id,
|
|
238
|
+
model_id="application",
|
|
239
|
+
model_definition_hash=model_hash,
|
|
240
|
+
dataset=ApplicationTaskContract(
|
|
241
|
+
task_id="application",
|
|
242
|
+
definition_hash=training.task_definition_hash,
|
|
243
|
+
),
|
|
244
|
+
expected_participant_count=participants,
|
|
245
|
+
environment=EnvironmentFingerprint(
|
|
246
|
+
dromeus_version=dromeus_version,
|
|
247
|
+
pytorch_version=pytorch_version,
|
|
248
|
+
axl_version=axl_version,
|
|
249
|
+
model_definition_hash=model_hash,
|
|
250
|
+
),
|
|
251
|
+
local_steps=local_steps,
|
|
252
|
+
round_count=rounds,
|
|
253
|
+
peer_scheduler_seed=seed,
|
|
254
|
+
codec_id="safetensors",
|
|
255
|
+
transport=transport or DEFAULT_TRANSPORT,
|
|
256
|
+
consensus_sketch=ConsensusSketchConfig(seed=seed),
|
|
257
|
+
training=training,
|
|
258
|
+
algorithm_config=NoLoCoConfig(
|
|
259
|
+
alpha=noloco.alpha,
|
|
260
|
+
beta=noloco.beta,
|
|
261
|
+
gamma=noloco.gamma,
|
|
262
|
+
inner_steps=local_steps,
|
|
263
|
+
),
|
|
264
|
+
artifact_codecs=noloco.codecs(),
|
|
265
|
+
privacy=privacy.policy(local_steps * rounds) if privacy else None,
|
|
266
|
+
divergence_detection=divergence,
|
|
267
|
+
)
|
|
268
|
+
except ValidationError as error:
|
|
269
|
+
raise DraftError(f"invalid draft settings: {error}") from error
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def make_draft(
|
|
273
|
+
task: TrainingTask,
|
|
274
|
+
path: Path | str,
|
|
275
|
+
*,
|
|
276
|
+
run_id: str,
|
|
277
|
+
participants: int,
|
|
278
|
+
rounds: int,
|
|
279
|
+
local_steps: int,
|
|
280
|
+
axl_version: str,
|
|
281
|
+
noloco: NoLoCo | None = None,
|
|
282
|
+
privacy: Privacy | None = None,
|
|
283
|
+
transport: TransportLimits | None = None,
|
|
284
|
+
divergence: DivergencePolicy | None = None,
|
|
285
|
+
seed: int = 0,
|
|
286
|
+
) -> str:
|
|
287
|
+
"""Write the group's shared draft for `task` to `path`; return its hash.
|
|
288
|
+
|
|
289
|
+
Run once, by one person, then give the file to every node. Every hash and
|
|
290
|
+
the Dromeus and PyTorch versions are filled in from this machine.
|
|
291
|
+
"""
|
|
292
|
+
draft = draft_for_task(
|
|
293
|
+
task,
|
|
294
|
+
run_id=run_id,
|
|
295
|
+
participants=participants,
|
|
296
|
+
rounds=rounds,
|
|
297
|
+
local_steps=local_steps,
|
|
298
|
+
axl_version=axl_version,
|
|
299
|
+
noloco=noloco,
|
|
300
|
+
privacy=privacy,
|
|
301
|
+
transport=transport,
|
|
302
|
+
divergence=divergence,
|
|
303
|
+
seed=seed,
|
|
304
|
+
)
|
|
305
|
+
text = yaml.safe_dump(
|
|
306
|
+
draft.model_dump(mode="json", exclude_none=True), sort_keys=False
|
|
307
|
+
)
|
|
308
|
+
Path(path).write_text(text, encoding="utf-8")
|
|
309
|
+
return canonical_hash(draft)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def load_draft(path: Path | str) -> DraftRunSpec:
|
|
313
|
+
"""Read and validate a draft file."""
|
|
314
|
+
try:
|
|
315
|
+
return parse_draft_yaml(Path(path))
|
|
316
|
+
except OSError as error:
|
|
317
|
+
raise ConfigError(f"cannot read draft {path}: {error}") from error
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def check_environment(draft: DraftRunSpec) -> None:
|
|
321
|
+
"""Refuse a draft made for a different Dromeus or PyTorch version."""
|
|
322
|
+
dromeus_version, pytorch_version = installed_versions()
|
|
323
|
+
for name, declared, installed in (
|
|
324
|
+
("Dromeus", draft.environment.dromeus_version, dromeus_version),
|
|
325
|
+
("PyTorch", draft.environment.pytorch_version, pytorch_version),
|
|
326
|
+
):
|
|
327
|
+
if declared != installed:
|
|
328
|
+
raise DraftError(
|
|
329
|
+
f"draft requires {name} {declared}; this machine has {installed}"
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def require_task_matches(draft: DraftRunSpec, task: TrainingTask) -> None:
|
|
334
|
+
"""Refuse a draft that was not made from this task."""
|
|
335
|
+
if definition_hash(task.model_definition) != draft.model_definition_hash:
|
|
336
|
+
raise DraftError("draft model definition differs from this task")
|
|
337
|
+
local = describe_task(task, sampling=draft.training.sampling)
|
|
338
|
+
if local != draft.training:
|
|
339
|
+
shared = draft.training.model_dump()
|
|
340
|
+
differing = sorted(
|
|
341
|
+
name for name, value in local.model_dump().items() if shared[name] != value
|
|
342
|
+
)
|
|
343
|
+
raise DraftError(
|
|
344
|
+
"draft training differs from this task: " + ", ".join(differing)
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def check_draft(path: Path | str, task: TrainingTask) -> None:
|
|
349
|
+
"""Check a draft file against this task and this machine before running.
|
|
350
|
+
|
|
351
|
+
Raises `DraftError` naming what differs. Every node should run this check;
|
|
352
|
+
`Node.run` performs it too.
|
|
353
|
+
"""
|
|
354
|
+
draft = load_draft(path)
|
|
355
|
+
check_environment(draft)
|
|
356
|
+
require_task_matches(draft, task)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def prepare_task(
|
|
360
|
+
*,
|
|
361
|
+
draft: DraftRunSpec,
|
|
362
|
+
task: TrainingTask,
|
|
363
|
+
local: LocalData,
|
|
364
|
+
evaluation_interval: int = 1,
|
|
365
|
+
) -> PreparedApplication:
|
|
366
|
+
"""Prepare this node's training for the group's draft."""
|
|
367
|
+
require_task_matches(draft, task)
|
|
368
|
+
specification = draft.training
|
|
369
|
+
trainer: PyTorchTrainer
|
|
370
|
+
if draft.privacy is not None:
|
|
371
|
+
if local.ledger is None:
|
|
372
|
+
raise PrivacyError("private task requires a persistent privacy ledger")
|
|
373
|
+
trainer = PrivateTrainer(
|
|
374
|
+
task=task,
|
|
375
|
+
dataset=local.dataset,
|
|
376
|
+
policy=draft.privacy,
|
|
377
|
+
specification=specification,
|
|
378
|
+
run_id=draft.run_id,
|
|
379
|
+
local_data_identity=local.data_identity,
|
|
380
|
+
reservation_store=local.ledger,
|
|
381
|
+
max_physical_batch_size=local.physical_batch_size,
|
|
382
|
+
research_seed=local.research_seed,
|
|
383
|
+
)
|
|
384
|
+
else:
|
|
385
|
+
from dromeus.training.ordinary_task import OrdinaryTaskTrainer
|
|
386
|
+
|
|
387
|
+
trainer = OrdinaryTaskTrainer(
|
|
388
|
+
task=task,
|
|
389
|
+
dataset=local.dataset,
|
|
390
|
+
specification=specification,
|
|
391
|
+
local_data_identity=local.data_identity,
|
|
392
|
+
max_logical_steps=draft.local_steps * draft.round_count,
|
|
393
|
+
research_seed=local.research_seed if local.research_seed is not None else 0,
|
|
394
|
+
max_physical_batch_size=local.physical_batch_size,
|
|
395
|
+
)
|
|
396
|
+
return prepare_trainer(
|
|
397
|
+
draft=draft,
|
|
398
|
+
trainer=trainer,
|
|
399
|
+
model_definition=task.model_definition,
|
|
400
|
+
evaluation_interval=evaluation_interval,
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
@dataclass(frozen=True, slots=True)
|
|
405
|
+
class PreparedApplication:
|
|
406
|
+
"""A validated application workload ready for fixed-group formation."""
|
|
407
|
+
|
|
408
|
+
_draft_hash: str
|
|
409
|
+
_trainer: PyTorchTrainer
|
|
410
|
+
_model_definition: str
|
|
411
|
+
_evaluation_interval: int
|
|
412
|
+
|
|
413
|
+
@property
|
|
414
|
+
def tensor_schema(self) -> TensorSchema:
|
|
415
|
+
return self._trainer.tensor_schema
|
|
416
|
+
|
|
417
|
+
def validate_draft(self, draft: DraftRunSpec) -> None:
|
|
418
|
+
if draft.privacy is not None:
|
|
419
|
+
if (
|
|
420
|
+
not isinstance(self._trainer, PrivateTrainer)
|
|
421
|
+
or self._trainer.policy != draft.privacy
|
|
422
|
+
or self._trainer.specification != draft.training
|
|
423
|
+
):
|
|
424
|
+
raise PrivacyError("private preparation does not match draft")
|
|
425
|
+
self._trainer.weights()
|
|
426
|
+
if canonical_hash(draft) != self._draft_hash:
|
|
427
|
+
raise DraftError("prepared application draft does not match")
|
|
428
|
+
|
|
429
|
+
def create_initial_checkpoint(self, path: Path) -> InitialCheckpoint:
|
|
430
|
+
return self._trainer.create_initial_checkpoint(
|
|
431
|
+
path, model_definition=self._model_definition
|
|
432
|
+
)
|
|
433
|
+
|
|
434
|
+
@property
|
|
435
|
+
def private_trainer(self) -> PrivateTrainer | None:
|
|
436
|
+
return self._trainer if isinstance(self._trainer, PrivateTrainer) else None
|
|
437
|
+
|
|
438
|
+
def export_model(self, path: Path) -> None:
|
|
439
|
+
"""Explicit model-only private export; training archives remain local."""
|
|
440
|
+
if not isinstance(self._trainer, PrivateTrainer):
|
|
441
|
+
raise PrivacyError("model-only private export requires a private adapter")
|
|
442
|
+
self._trainer.export_model(path)
|
|
443
|
+
|
|
444
|
+
def build_config(
|
|
445
|
+
self,
|
|
446
|
+
*,
|
|
447
|
+
result: FormationResult,
|
|
448
|
+
local_public_key: str,
|
|
449
|
+
run_root: Path,
|
|
450
|
+
metrics_publisher: MetricsService | None = None,
|
|
451
|
+
) -> TrainingConfig:
|
|
452
|
+
validate_sealed_draft(result.manifest, expected_draft_hash=self._draft_hash)
|
|
453
|
+
if canonical_hash(result.manifest) != result.manifest_hash:
|
|
454
|
+
raise DraftError("formed manifest hash does not match its contents")
|
|
455
|
+
if result.manifest.tensor_schema != self.tensor_schema:
|
|
456
|
+
raise DraftError("formed tensor schema does not match application")
|
|
457
|
+
if local_public_key not in {
|
|
458
|
+
member.public_key for member in result.manifest.participants
|
|
459
|
+
}:
|
|
460
|
+
raise DraftError("local key is not a formed participant")
|
|
461
|
+
durable = FailureConfig.for_run_root(run_root)
|
|
462
|
+
return TrainingConfig(
|
|
463
|
+
algorithm=build_algorithm(manifest=result.manifest, trainer=self._trainer),
|
|
464
|
+
load_checkpoint=self._trainer.load_checkpoint,
|
|
465
|
+
run_store=durable.run_store,
|
|
466
|
+
artifact_root=durable.artifact_root,
|
|
467
|
+
metrics_publisher=metrics_publisher,
|
|
468
|
+
evaluation_interval=self._evaluation_interval,
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def prepare_trainer(
|
|
473
|
+
*,
|
|
474
|
+
draft: DraftRunSpec,
|
|
475
|
+
trainer: PyTorchTrainer,
|
|
476
|
+
model_definition: str,
|
|
477
|
+
evaluation_interval: int = 1,
|
|
478
|
+
) -> PreparedApplication:
|
|
479
|
+
"""Bind a fresh trainer to the draft, before any AXL or formation work."""
|
|
480
|
+
if draft.privacy is not None:
|
|
481
|
+
if not isinstance(trainer, PrivateTrainer):
|
|
482
|
+
raise PrivacyError("private policy requires the owned private adapter")
|
|
483
|
+
if draft.privacy != trainer.policy or draft.training != trainer.specification:
|
|
484
|
+
raise PrivacyError("private trainer policy or specification mismatch")
|
|
485
|
+
elif isinstance(trainer, PrivateTrainer):
|
|
486
|
+
raise PrivacyError("private adapter requires a private manifest")
|
|
487
|
+
if definition_hash(model_definition) != draft.model_definition_hash:
|
|
488
|
+
raise DraftError("model definition does not match shared draft")
|
|
489
|
+
if evaluation_interval <= 0:
|
|
490
|
+
raise ConfigError("evaluation interval must be positive")
|
|
491
|
+
if trainer.completed_steps:
|
|
492
|
+
raise TaskError("prepare a fresh trainer before formation")
|
|
493
|
+
trainer.weights()
|
|
494
|
+
outer_policy = draft.model_dump(
|
|
495
|
+
mode="json",
|
|
496
|
+
include={
|
|
497
|
+
"algorithm_id",
|
|
498
|
+
"algorithm_config",
|
|
499
|
+
"artifact_codecs",
|
|
500
|
+
"codec_id",
|
|
501
|
+
"local_steps",
|
|
502
|
+
"round_count",
|
|
503
|
+
"privacy",
|
|
504
|
+
"training",
|
|
505
|
+
},
|
|
506
|
+
)
|
|
507
|
+
outer_identity = definition_hash(
|
|
508
|
+
json.dumps(outer_policy, sort_keys=True, separators=(",", ":"), allow_nan=False)
|
|
509
|
+
)
|
|
510
|
+
trainer.bind_contract(
|
|
511
|
+
":".join(
|
|
512
|
+
(
|
|
513
|
+
"application-restore",
|
|
514
|
+
draft.model_definition_hash,
|
|
515
|
+
canonical_hash(draft.dataset),
|
|
516
|
+
canonical_hash(draft.training),
|
|
517
|
+
outer_identity,
|
|
518
|
+
)
|
|
519
|
+
)
|
|
520
|
+
)
|
|
521
|
+
return PreparedApplication(
|
|
522
|
+
canonical_hash(draft), trainer, model_definition, evaluation_interval
|
|
523
|
+
)
|