dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,694 @@
|
|
|
1
|
+
"""Validated domain models for run formation and update exchange."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from typing import Annotated, Literal, Self, cast
|
|
9
|
+
|
|
10
|
+
from pydantic import (
|
|
11
|
+
Field,
|
|
12
|
+
JsonValue,
|
|
13
|
+
StringConstraints,
|
|
14
|
+
field_validator,
|
|
15
|
+
model_validator,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
from dromeus.errors import DraftError
|
|
19
|
+
from dromeus.protocol.models import (
|
|
20
|
+
CHUNK_FRAMING_BYTES,
|
|
21
|
+
AlgorithmId,
|
|
22
|
+
DomainModel,
|
|
23
|
+
Identifier,
|
|
24
|
+
PublicKey,
|
|
25
|
+
RoundId,
|
|
26
|
+
RunId,
|
|
27
|
+
Sha256,
|
|
28
|
+
TensorSchema,
|
|
29
|
+
)
|
|
30
|
+
from dromeus.protocol.models import (
|
|
31
|
+
MessageId as MessageId,
|
|
32
|
+
)
|
|
33
|
+
from dromeus.protocol.models import (
|
|
34
|
+
Tensor as Tensor,
|
|
35
|
+
)
|
|
36
|
+
from dromeus.protocol.models import (
|
|
37
|
+
TransferId as TransferId,
|
|
38
|
+
)
|
|
39
|
+
from dromeus.protocol.version import PROTOCOL_VERSION
|
|
40
|
+
|
|
41
|
+
MANIFEST_VERSION = 7
|
|
42
|
+
MIN_PARTICIPANT_COUNT = 4
|
|
43
|
+
MAX_PARTICIPANT_COUNT = 16
|
|
44
|
+
NOLOCO_ALGORITHM_ID = "noloco"
|
|
45
|
+
|
|
46
|
+
PackageVersion = Annotated[
|
|
47
|
+
str,
|
|
48
|
+
StringConstraints(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:+-]+$"),
|
|
49
|
+
]
|
|
50
|
+
NodeIndex = Annotated[int, Field(ge=0, lt=MAX_PARTICIPANT_COUNT)]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class ParticipantCountError(DraftError):
|
|
54
|
+
"""A run member count is outside the supported fixed-group range."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def validate_participant_count(count: int) -> None:
|
|
58
|
+
"""Validate the fixed even membership range shared by manifests and gossip."""
|
|
59
|
+
if count < MIN_PARTICIPANT_COUNT or count > MAX_PARTICIPANT_COUNT:
|
|
60
|
+
raise ParticipantCountError(
|
|
61
|
+
"participant count must be between 4 and 16 inclusive"
|
|
62
|
+
)
|
|
63
|
+
if count % 2:
|
|
64
|
+
raise ParticipantCountError("participant count must be even")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Participant(DomainModel):
|
|
68
|
+
public_key: PublicKey
|
|
69
|
+
node_index: NodeIndex
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class ConsensusSketchMessage(DomainModel):
|
|
73
|
+
sender_public_key: PublicKey
|
|
74
|
+
round_id: RoundId
|
|
75
|
+
payload: Annotated[bytes, Field(min_length=1)]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class ApplicationTaskContract(DomainModel):
|
|
79
|
+
"""Application-defined input, target, preprocessing, and objective semantics.
|
|
80
|
+
|
|
81
|
+
The definition hash identifies the shared task, never node-local records.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
dataset_id: Literal["application"] = "application"
|
|
85
|
+
task_id: Identifier
|
|
86
|
+
definition_hash: Sha256
|
|
87
|
+
objective: Literal["equal-node"] = "equal-node"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class EnvironmentFingerprint(DomainModel):
|
|
91
|
+
dromeus_version: Identifier
|
|
92
|
+
protocol_version: Literal[1] = PROTOCOL_VERSION
|
|
93
|
+
pytorch_version: PackageVersion
|
|
94
|
+
axl_version: Identifier
|
|
95
|
+
model_definition_hash: Sha256
|
|
96
|
+
container_image_digest: (
|
|
97
|
+
Annotated[str, StringConstraints(pattern=r"^sha256:[0-9a-f]{64}$")] | None
|
|
98
|
+
) = None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class TransportLimits(DomainModel):
|
|
102
|
+
max_payload_bytes: Annotated[int, Field(gt=0)]
|
|
103
|
+
max_retries: Annotated[int, Field(ge=0)]
|
|
104
|
+
retry_timeout_seconds: Annotated[float, Field(gt=0)]
|
|
105
|
+
chunk_size_bytes: Annotated[int, Field(gt=0)] | None = None
|
|
106
|
+
window_size: Annotated[int, Field(gt=0)] | None = None
|
|
107
|
+
max_message_payload_bytes: Annotated[int, Field(gt=0)] | None = None
|
|
108
|
+
max_artifact_bytes: Annotated[int, Field(gt=0)] | None = None
|
|
109
|
+
max_concurrent_transfers: Annotated[int, Field(gt=0)] | None = None
|
|
110
|
+
max_inflight_bytes: Annotated[int, Field(gt=0)] | None = None
|
|
111
|
+
transfer_lifetime_seconds: Annotated[float, Field(gt=0)] | None = None
|
|
112
|
+
artifact_store_capacity_bytes: Annotated[int, Field(gt=0)] | None = None
|
|
113
|
+
|
|
114
|
+
@model_validator(mode="after")
|
|
115
|
+
def valid_transfer_bounds(self) -> Self:
|
|
116
|
+
if self.effective_chunk_size + CHUNK_FRAMING_BYTES > self.message_payload_limit:
|
|
117
|
+
raise ValueError(
|
|
118
|
+
"chunk size plus chunk framing must not exceed message payload limit"
|
|
119
|
+
)
|
|
120
|
+
if self.message_payload_limit > self.max_payload_bytes:
|
|
121
|
+
raise ValueError("message payload limit must not exceed bundle limit")
|
|
122
|
+
if self.artifact_size_limit > self.max_payload_bytes:
|
|
123
|
+
raise ValueError("artifact limit must not exceed bundle limit")
|
|
124
|
+
if (
|
|
125
|
+
self.effective_chunk_size * self.effective_window_size
|
|
126
|
+
> self.inflight_byte_limit
|
|
127
|
+
):
|
|
128
|
+
raise ValueError("window bytes must not exceed in-flight byte limit")
|
|
129
|
+
if self.artifact_size_limit > self.artifact_store_capacity_limit:
|
|
130
|
+
raise ValueError("artifact limit must not exceed store capacity")
|
|
131
|
+
if self.inflight_byte_limit > self.artifact_store_capacity_limit:
|
|
132
|
+
raise ValueError("in-flight limit must not exceed store capacity")
|
|
133
|
+
minimum_lifetime = self.retry_timeout_seconds * (self.max_retries + 1)
|
|
134
|
+
if self.transfer_lifetime_limit < minimum_lifetime:
|
|
135
|
+
raise ValueError("transfer lifetime is shorter than retry budget")
|
|
136
|
+
return self
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def max_update_bundle_bytes(self) -> int:
|
|
140
|
+
"""Return the byte limit for one whole update bundle."""
|
|
141
|
+
return self.max_payload_bytes
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def message_payload_limit(self) -> int:
|
|
145
|
+
return self.max_message_payload_bytes or self.max_payload_bytes
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def artifact_size_limit(self) -> int:
|
|
149
|
+
return self.max_artifact_bytes or self.max_payload_bytes
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def effective_chunk_size(self) -> int:
|
|
153
|
+
if self.chunk_size_bytes is not None:
|
|
154
|
+
return self.chunk_size_bytes
|
|
155
|
+
return max(
|
|
156
|
+
1,
|
|
157
|
+
min(
|
|
158
|
+
self.artifact_size_limit,
|
|
159
|
+
self.message_payload_limit - CHUNK_FRAMING_BYTES,
|
|
160
|
+
),
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def effective_window_size(self) -> int:
|
|
165
|
+
return self.window_size or 1
|
|
166
|
+
|
|
167
|
+
@property
|
|
168
|
+
def concurrent_transfer_limit(self) -> int:
|
|
169
|
+
return self.max_concurrent_transfers or 1
|
|
170
|
+
|
|
171
|
+
@property
|
|
172
|
+
def inflight_byte_limit(self) -> int:
|
|
173
|
+
return self.max_inflight_bytes or self.artifact_size_limit
|
|
174
|
+
|
|
175
|
+
@property
|
|
176
|
+
def transfer_lifetime_limit(self) -> float:
|
|
177
|
+
return self.transfer_lifetime_seconds or (
|
|
178
|
+
self.retry_timeout_seconds * (self.max_retries + 2)
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def artifact_store_capacity_limit(self) -> int:
|
|
183
|
+
return self.artifact_store_capacity_bytes or self.artifact_size_limit
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class ConsensusSketchConfig(DomainModel):
|
|
187
|
+
size: Literal[4096] = 4096
|
|
188
|
+
seed: int
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class NoLoCoConfig(DomainModel):
|
|
192
|
+
"""Outer NoLoCo settings; the application owns its inner optimizer."""
|
|
193
|
+
|
|
194
|
+
alpha: Annotated[float, Field(ge=0.0, lt=1.0)]
|
|
195
|
+
beta: Annotated[float, Field(gt=0.0)]
|
|
196
|
+
gamma: Annotated[float, Field(gt=0.0)]
|
|
197
|
+
inner_steps: Annotated[int, Field(gt=0)]
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class ArtifactCodec(DomainModel):
|
|
201
|
+
"""Codec identity bound to one logical update artifact."""
|
|
202
|
+
|
|
203
|
+
artifact_name: Identifier
|
|
204
|
+
codec_id: Identifier
|
|
205
|
+
top_k_fraction: Annotated[float, Field(gt=0.0, le=1.0)] | None = None
|
|
206
|
+
lossy_allowed: bool | None = None
|
|
207
|
+
|
|
208
|
+
@model_validator(mode="after")
|
|
209
|
+
def valid_codec_settings(self) -> Self:
|
|
210
|
+
if self.codec_id == "identity":
|
|
211
|
+
if self.top_k_fraction is not None or self.lossy_allowed is True:
|
|
212
|
+
raise ValueError("identity codec cannot declare lossy settings")
|
|
213
|
+
return self
|
|
214
|
+
if self.codec_id == "topk-bitmap-int8":
|
|
215
|
+
if self.artifact_name != "outer_gradient":
|
|
216
|
+
raise ValueError("top-k codec is only valid for outer gradient")
|
|
217
|
+
if self.top_k_fraction is None or not self.lossy_allowed:
|
|
218
|
+
raise ValueError("top-k codec requires fraction and lossy permission")
|
|
219
|
+
return self
|
|
220
|
+
if self.codec_id == "dense-int8":
|
|
221
|
+
if self.artifact_name != "slow_weights":
|
|
222
|
+
raise ValueError("dense int8 codec is only valid for slow weights")
|
|
223
|
+
if self.top_k_fraction is not None or not self.lossy_allowed:
|
|
224
|
+
raise ValueError("dense int8 codec requires lossy permission")
|
|
225
|
+
return self
|
|
226
|
+
raise ValueError("unsupported NoLoCo artifact codec")
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class UpdateCodecBinding(DomainModel):
|
|
230
|
+
"""Logical codec identity and schema bound into one update artifact."""
|
|
231
|
+
|
|
232
|
+
codec_id: Identifier
|
|
233
|
+
logical_schema: TensorSchema
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
class PrivacyPolicy(DomainModel):
|
|
237
|
+
"""Fixed record-level Poisson Gaussian mechanism and release policy."""
|
|
238
|
+
|
|
239
|
+
mode: Literal["record_dp"] = "record_dp"
|
|
240
|
+
adjacency: Literal["add_remove"] = "add_remove"
|
|
241
|
+
sampling: Literal["poisson"] = "poisson"
|
|
242
|
+
clipping: Literal["flat"] = "flat"
|
|
243
|
+
max_grad_norm: Annotated[float, Field(gt=0, allow_inf_nan=False)]
|
|
244
|
+
noise_multiplier: Annotated[float, Field(gt=0, allow_inf_nan=False)]
|
|
245
|
+
accountant: Literal["prv"] = "prv"
|
|
246
|
+
delta: Annotated[float, Field(gt=0, lt=1, allow_inf_nan=False)]
|
|
247
|
+
max_logical_steps: Annotated[int, Field(gt=0)]
|
|
248
|
+
epsilon_limit: Annotated[float, Field(gt=0, allow_inf_nan=False)] | None = None
|
|
249
|
+
randomness_profile: Literal["secure", "reproducible_public_benchmark"]
|
|
250
|
+
metric_policy: Literal["public_eval_only"] = "public_eval_only"
|
|
251
|
+
prv_epsilon_error: Annotated[float, Field(gt=0, allow_inf_nan=False)] = 0.01
|
|
252
|
+
prv_delta_error_ratio: Annotated[float, Field(gt=0, lt=1)] = 0.001
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
class DivergencePolicy(DomainModel):
|
|
256
|
+
"""Explicit pilot thresholds; no universal detector defaults."""
|
|
257
|
+
|
|
258
|
+
version: Literal[1] = 1
|
|
259
|
+
mode: Literal["warn"] = "warn"
|
|
260
|
+
threshold_set_id: Identifier
|
|
261
|
+
warmup_rounds: Annotated[int, Field(ge=0)]
|
|
262
|
+
window_rounds: Annotated[int, Field(ge=4, le=4096)]
|
|
263
|
+
patience_windows: Annotated[int, Field(gt=0)]
|
|
264
|
+
distance_floor: Annotated[float, Field(gt=0, allow_inf_nan=False)]
|
|
265
|
+
growth_ratio: Annotated[float, Field(gt=1, allow_inf_nan=False)]
|
|
266
|
+
min_log_slope: Annotated[float, Field(gt=0, allow_inf_nan=False)]
|
|
267
|
+
severe_distance: Annotated[float, Field(gt=0, allow_inf_nan=False)]
|
|
268
|
+
severe_patience: Annotated[int, Field(gt=0)]
|
|
269
|
+
recovery_windows: Annotated[int, Field(gt=0)]
|
|
270
|
+
recovery_distance: Annotated[float, Field(gt=0, allow_inf_nan=False)]
|
|
271
|
+
max_lag_rounds: Annotated[int, Field(ge=0, le=4096)]
|
|
272
|
+
absolute_distance_floor: (
|
|
273
|
+
Annotated[float, Field(gt=0, allow_inf_nan=False)] | None
|
|
274
|
+
) = None
|
|
275
|
+
absolute_severe_distance: (
|
|
276
|
+
Annotated[float, Field(gt=0, allow_inf_nan=False)] | None
|
|
277
|
+
) = None
|
|
278
|
+
absolute_recovery_distance: (
|
|
279
|
+
Annotated[float, Field(gt=0, allow_inf_nan=False)] | None
|
|
280
|
+
) = None
|
|
281
|
+
|
|
282
|
+
@model_validator(mode="after")
|
|
283
|
+
def ordered_thresholds(self) -> Self:
|
|
284
|
+
if self.recovery_distance >= self.severe_distance:
|
|
285
|
+
raise ValueError("recovery threshold must be below severe threshold")
|
|
286
|
+
absolute = (
|
|
287
|
+
self.absolute_distance_floor,
|
|
288
|
+
self.absolute_severe_distance,
|
|
289
|
+
self.absolute_recovery_distance,
|
|
290
|
+
)
|
|
291
|
+
if any(x is not None for x in absolute):
|
|
292
|
+
if any(x is None for x in absolute):
|
|
293
|
+
raise ValueError("absolute scale thresholds must be provided together")
|
|
294
|
+
assert (
|
|
295
|
+
self.absolute_severe_distance is not None
|
|
296
|
+
and self.absolute_recovery_distance is not None
|
|
297
|
+
)
|
|
298
|
+
if self.absolute_recovery_distance >= self.absolute_severe_distance:
|
|
299
|
+
raise ValueError("absolute recovery must be below severe threshold")
|
|
300
|
+
return self
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
class PrivateOptimizerSpec(DomainModel):
|
|
304
|
+
"""One complete named group, ordinary dense FP32 PyTorch execution."""
|
|
305
|
+
|
|
306
|
+
name: Literal["sgd", "adam", "adamw"]
|
|
307
|
+
torch_version: PackageVersion
|
|
308
|
+
parameter_names: tuple[Identifier, ...] = Field(min_length=1)
|
|
309
|
+
learning_rate: Annotated[float, Field(gt=0, allow_inf_nan=False)]
|
|
310
|
+
weight_decay: Annotated[float, Field(ge=0, allow_inf_nan=False)] = 0
|
|
311
|
+
momentum: Annotated[float, Field(ge=0, lt=1)] = 0
|
|
312
|
+
dampening: Annotated[float, Field(ge=0)] = 0
|
|
313
|
+
nesterov: bool = False
|
|
314
|
+
betas: tuple[
|
|
315
|
+
Annotated[float, Field(ge=0, lt=1)], Annotated[float, Field(ge=0, lt=1)]
|
|
316
|
+
] = (0.9, 0.999)
|
|
317
|
+
numerical_epsilon: Annotated[float, Field(gt=0, allow_inf_nan=False)] = 1e-8
|
|
318
|
+
amsgrad: Literal[False] = False
|
|
319
|
+
foreach: Literal[False] = False
|
|
320
|
+
fused: Literal[False] = False
|
|
321
|
+
capturable: Literal[False] = False
|
|
322
|
+
differentiable: Literal[False] = False
|
|
323
|
+
maximize: Literal[False] = False
|
|
324
|
+
|
|
325
|
+
@model_validator(mode="after")
|
|
326
|
+
def valid_options(self) -> Self:
|
|
327
|
+
if len(set(self.parameter_names)) != len(self.parameter_names):
|
|
328
|
+
raise ValueError("parameter names must be unique and ordered")
|
|
329
|
+
if self.name != "sgd" and (self.momentum or self.dampening or self.nesterov):
|
|
330
|
+
raise ValueError("momentum options require SGD")
|
|
331
|
+
if self.nesterov and (not self.momentum or self.dampening):
|
|
332
|
+
raise ValueError("Nesterov requires momentum and zero dampening")
|
|
333
|
+
return self
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
class OptimizerParameterGroup(DomainModel):
|
|
337
|
+
"""Ordered parameter membership and resolved JSON optimizer options."""
|
|
338
|
+
|
|
339
|
+
parameter_names: tuple[Identifier, ...] = Field(min_length=1, max_length=65536)
|
|
340
|
+
options: dict[str, JsonValue]
|
|
341
|
+
|
|
342
|
+
@field_validator("options", mode="before")
|
|
343
|
+
@classmethod
|
|
344
|
+
def bounded_json_options(cls, value: object) -> object:
|
|
345
|
+
"""Bound untrusted metadata before Pydantic's recursive JSON parsing."""
|
|
346
|
+
pending: list[tuple[object, int]] = [(value, 0)]
|
|
347
|
+
remaining = 4096
|
|
348
|
+
while pending:
|
|
349
|
+
item, depth = pending.pop()
|
|
350
|
+
remaining -= 1
|
|
351
|
+
if depth > 8 or remaining < 0:
|
|
352
|
+
raise ValueError(
|
|
353
|
+
"optimizer options exceed metadata depth or size limit"
|
|
354
|
+
)
|
|
355
|
+
if isinstance(item, dict):
|
|
356
|
+
mapping = cast(dict[object, object], item)
|
|
357
|
+
if len(mapping) > remaining:
|
|
358
|
+
raise ValueError("optimizer options exceed metadata size limit")
|
|
359
|
+
for key, child in mapping.items():
|
|
360
|
+
if not isinstance(key, str) or not key:
|
|
361
|
+
raise ValueError(
|
|
362
|
+
"optimizer option keys must be nonempty strings"
|
|
363
|
+
)
|
|
364
|
+
pending.append((child, depth + 1))
|
|
365
|
+
elif isinstance(item, list):
|
|
366
|
+
sequence = cast(list[object], item)
|
|
367
|
+
if len(sequence) > remaining:
|
|
368
|
+
raise ValueError("optimizer options exceed metadata size limit")
|
|
369
|
+
pending.extend((child, depth + 1) for child in sequence)
|
|
370
|
+
elif item is not None and type(item) not in (str, bool, int, float):
|
|
371
|
+
raise ValueError("optimizer options require JSON values")
|
|
372
|
+
elif isinstance(item, float) and not math.isfinite(item):
|
|
373
|
+
raise ValueError("optimizer options must contain finite numbers")
|
|
374
|
+
if len(json.dumps(value, allow_nan=False).encode()) > 65536:
|
|
375
|
+
raise ValueError("optimizer options exceed metadata size limit")
|
|
376
|
+
return value
|
|
377
|
+
|
|
378
|
+
@model_validator(mode="after")
|
|
379
|
+
def valid_options(self) -> Self:
|
|
380
|
+
if len(set(self.parameter_names)) != len(self.parameter_names):
|
|
381
|
+
raise ValueError("parameter names must be unique and ordered")
|
|
382
|
+
if "params" in self.options:
|
|
383
|
+
raise ValueError("parameter membership belongs in parameter_names")
|
|
384
|
+
learning_rate = self.options.get("lr")
|
|
385
|
+
if (
|
|
386
|
+
isinstance(learning_rate, bool)
|
|
387
|
+
or not isinstance(learning_rate, (int, float))
|
|
388
|
+
or learning_rate <= 0
|
|
389
|
+
):
|
|
390
|
+
raise ValueError("optimizer lr must be a positive finite scalar")
|
|
391
|
+
for flag in ("foreach", "fused", "capturable", "differentiable", "maximize"):
|
|
392
|
+
if flag in self.options and self.options[flag] is not False:
|
|
393
|
+
raise ValueError(f"optimizer {flag} must be explicitly false")
|
|
394
|
+
return self
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
class FactoryOptimizerSpec(DomainModel):
|
|
398
|
+
"""Group-agreed implementation and state-validator identity for a factory."""
|
|
399
|
+
|
|
400
|
+
kind: Literal["factory"] = "factory"
|
|
401
|
+
implementation_id: Annotated[
|
|
402
|
+
str, StringConstraints(min_length=1, max_length=256, pattern=r"^\S+$")
|
|
403
|
+
]
|
|
404
|
+
implementation_hash: Sha256
|
|
405
|
+
torch_version: PackageVersion
|
|
406
|
+
groups: tuple[OptimizerParameterGroup, ...] = Field(min_length=1, max_length=32)
|
|
407
|
+
|
|
408
|
+
@property
|
|
409
|
+
def parameter_names(self) -> tuple[str, ...]:
|
|
410
|
+
return tuple(name for group in self.groups for name in group.parameter_names)
|
|
411
|
+
|
|
412
|
+
@model_validator(mode="after")
|
|
413
|
+
def unique_parameters(self) -> Self:
|
|
414
|
+
names = self.parameter_names
|
|
415
|
+
if len(set(names)) != len(names):
|
|
416
|
+
raise ValueError("optimizer parameter groups must not overlap")
|
|
417
|
+
return self
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
GradientBackend = Literal["opacus_hooks", "opacus_native_b1"]
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
class AccountedInitializationAuthority(DomainModel):
|
|
424
|
+
"""An existing protected lineage whose spending precedes initialization."""
|
|
425
|
+
|
|
426
|
+
lineage_id: Annotated[
|
|
427
|
+
str, StringConstraints(min_length=1, max_length=256, pattern=r"\S")
|
|
428
|
+
]
|
|
429
|
+
ledger_sha256: Sha256
|
|
430
|
+
reserved_steps: Annotated[int, Field(gt=0)]
|
|
431
|
+
reservation_count: Annotated[int, Field(gt=0)]
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
class AccountedInitializationProvenance(DomainModel):
|
|
435
|
+
"""Bound ancestry of a sanitized model, never a fresh privacy allowance."""
|
|
436
|
+
|
|
437
|
+
parent_run_id: RunId
|
|
438
|
+
parent_model_sha256: Sha256
|
|
439
|
+
parent_audit_sha256: Sha256
|
|
440
|
+
accounting_authorities: tuple[AccountedInitializationAuthority, ...] = Field(
|
|
441
|
+
min_length=1, max_length=16
|
|
442
|
+
)
|
|
443
|
+
|
|
444
|
+
@model_validator(mode="after")
|
|
445
|
+
def distinct_lineages(self) -> Self:
|
|
446
|
+
names = [row.lineage_id for row in self.accounting_authorities]
|
|
447
|
+
if len(names) != len(set(names)):
|
|
448
|
+
raise ValueError("initialization accounting lineages must be unique")
|
|
449
|
+
return self
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
class TaskTrainingSpec(DomainModel):
|
|
453
|
+
"""Shared semantics for an application-defined finite-record training task."""
|
|
454
|
+
|
|
455
|
+
optimizer: PrivateOptimizerSpec | FactoryOptimizerSpec
|
|
456
|
+
batch_size: Annotated[int, Field(gt=0)]
|
|
457
|
+
sampling: Literal["poisson", "shuffle"] = "poisson"
|
|
458
|
+
schedule: Literal["constant", "linear_decay"] = "constant"
|
|
459
|
+
initialization: Literal[
|
|
460
|
+
"public_untrained", "public_pretrained", "accounted_private_pretrained"
|
|
461
|
+
] = "public_untrained"
|
|
462
|
+
initialization_provenance: AccountedInitializationProvenance | None = None
|
|
463
|
+
evaluation_provenance: Literal["public_fixed_holdout", "disabled"] = "disabled"
|
|
464
|
+
task_definition_hash: Sha256
|
|
465
|
+
implementation_hash: Sha256
|
|
466
|
+
batch_definition_hash: Sha256
|
|
467
|
+
loss_definition_hash: Sha256
|
|
468
|
+
record_unit: Annotated[
|
|
469
|
+
str, StringConstraints(min_length=1, max_length=256, pattern=r"\S")
|
|
470
|
+
]
|
|
471
|
+
gradient_backend: GradientBackend = "opacus_hooks"
|
|
472
|
+
trainable_parameter_names: tuple[Identifier, ...] = Field(
|
|
473
|
+
min_length=1, max_length=65536
|
|
474
|
+
)
|
|
475
|
+
parameter_aliases: tuple[tuple[Identifier, ...], ...] = Field(
|
|
476
|
+
default=(), max_length=65536
|
|
477
|
+
)
|
|
478
|
+
public_buffer_hashes: dict[Identifier, Sha256] = Field(
|
|
479
|
+
default_factory=dict, max_length=65536
|
|
480
|
+
)
|
|
481
|
+
frozen_parameter_hashes: dict[Identifier, Sha256] = Field(
|
|
482
|
+
default_factory=dict, max_length=65536
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
@model_validator(mode="after")
|
|
486
|
+
def parameter_state_is_unambiguous(self) -> Self:
|
|
487
|
+
if (self.initialization == "accounted_private_pretrained") != (
|
|
488
|
+
self.initialization_provenance is not None
|
|
489
|
+
):
|
|
490
|
+
raise ValueError(
|
|
491
|
+
"accounted private initialization requires exclusive provenance"
|
|
492
|
+
)
|
|
493
|
+
trainable = set(self.trainable_parameter_names)
|
|
494
|
+
if len(trainable) != len(self.trainable_parameter_names):
|
|
495
|
+
raise ValueError("trainable parameter names must be unique")
|
|
496
|
+
if trainable != set(self.optimizer.parameter_names):
|
|
497
|
+
raise ValueError(
|
|
498
|
+
"optimizer parameter names must match trainable parameters"
|
|
499
|
+
)
|
|
500
|
+
frozen = set(self.frozen_parameter_hashes)
|
|
501
|
+
if trainable & frozen:
|
|
502
|
+
raise ValueError("trainable and frozen parameter names must not overlap")
|
|
503
|
+
canonical = trainable | frozen
|
|
504
|
+
aliases: set[str] = set()
|
|
505
|
+
for group in self.parameter_aliases:
|
|
506
|
+
if len(group) < 2:
|
|
507
|
+
raise ValueError("parameter alias groups require at least two names")
|
|
508
|
+
names = set(group)
|
|
509
|
+
if len(names) != len(group):
|
|
510
|
+
raise ValueError("parameter alias names must be distinct")
|
|
511
|
+
if aliases & names:
|
|
512
|
+
raise ValueError("parameter alias groups must be disjoint")
|
|
513
|
+
if len(names & canonical) != 1:
|
|
514
|
+
raise ValueError(
|
|
515
|
+
"alias group must identify exactly one canonical parameter"
|
|
516
|
+
)
|
|
517
|
+
aliases.update(names)
|
|
518
|
+
if set(self.public_buffer_hashes) & (canonical | aliases):
|
|
519
|
+
raise ValueError("buffer and parameter names must not overlap")
|
|
520
|
+
return self
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
class PrivacyReservation(DomainModel):
|
|
524
|
+
version: Literal[1] = 1
|
|
525
|
+
run_id: RunId
|
|
526
|
+
policy_hash: Sha256
|
|
527
|
+
sample_rate: Annotated[float, Field(gt=0, le=1, allow_inf_nan=False)]
|
|
528
|
+
noise_multiplier: Annotated[float, Field(gt=0, allow_inf_nan=False)]
|
|
529
|
+
logical_steps: Annotated[int, Field(gt=0)]
|
|
530
|
+
accountant_version: Literal["opacus-1.6.0-prv"] = "opacus-1.6.0-prv"
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
class DraftRunSpec(DomainModel):
|
|
534
|
+
"""Shared run definition every participant must agree on before formation."""
|
|
535
|
+
|
|
536
|
+
manifest_version: Literal[7] = MANIFEST_VERSION
|
|
537
|
+
protocol_version: Literal[1] = PROTOCOL_VERSION
|
|
538
|
+
run_id: RunId
|
|
539
|
+
algorithm_id: AlgorithmId = NOLOCO_ALGORITHM_ID
|
|
540
|
+
model_id: Identifier
|
|
541
|
+
model_definition_hash: Sha256
|
|
542
|
+
dataset: ApplicationTaskContract
|
|
543
|
+
expected_participant_count: Annotated[
|
|
544
|
+
int, Field(ge=MIN_PARTICIPANT_COUNT, le=MAX_PARTICIPANT_COUNT)
|
|
545
|
+
]
|
|
546
|
+
environment: EnvironmentFingerprint
|
|
547
|
+
local_steps: Annotated[int, Field(gt=0)]
|
|
548
|
+
round_count: Annotated[int, Field(gt=0)]
|
|
549
|
+
peer_scheduler_seed: int
|
|
550
|
+
codec_id: Literal["safetensors"]
|
|
551
|
+
transport: TransportLimits
|
|
552
|
+
consensus_sketch: ConsensusSketchConfig
|
|
553
|
+
training: TaskTrainingSpec
|
|
554
|
+
algorithm_config: NoLoCoConfig
|
|
555
|
+
artifact_codecs: tuple[ArtifactCodec, ...]
|
|
556
|
+
|
|
557
|
+
privacy: PrivacyPolicy | None = None
|
|
558
|
+
divergence_detection: DivergencePolicy | None = None
|
|
559
|
+
|
|
560
|
+
@property
|
|
561
|
+
def participant_count(self) -> int:
|
|
562
|
+
return self.expected_participant_count
|
|
563
|
+
|
|
564
|
+
@model_validator(mode="after")
|
|
565
|
+
def valid_manifest(self) -> Self:
|
|
566
|
+
validate_participant_count(self.expected_participant_count)
|
|
567
|
+
if self.algorithm_id != NOLOCO_ALGORITHM_ID:
|
|
568
|
+
raise ValueError("manifest requires the noloco algorithm")
|
|
569
|
+
if self.environment.model_definition_hash != self.model_definition_hash:
|
|
570
|
+
raise ValueError("environment model hash does not match draft")
|
|
571
|
+
if self.training.task_definition_hash != self.dataset.definition_hash:
|
|
572
|
+
raise ValueError("training task definition must match dataset contract")
|
|
573
|
+
if self.privacy is not None:
|
|
574
|
+
if self.training.sampling != "poisson":
|
|
575
|
+
raise ValueError("private tasks require Poisson sampling")
|
|
576
|
+
if self.privacy.max_logical_steps != self.local_steps * self.round_count:
|
|
577
|
+
raise ValueError("privacy horizon must match local steps and rounds")
|
|
578
|
+
if self.local_steps != self.algorithm_config.inner_steps:
|
|
579
|
+
raise ValueError("local steps must match NoLoCo inner steps")
|
|
580
|
+
names = [codec.artifact_name for codec in self.artifact_codecs]
|
|
581
|
+
if len(names) != len(set(names)) or set(names) != {
|
|
582
|
+
"outer_gradient",
|
|
583
|
+
"slow_weights",
|
|
584
|
+
}:
|
|
585
|
+
raise ValueError(
|
|
586
|
+
"NoLoCo codec IDs must cover outer_gradient and slow_weights"
|
|
587
|
+
)
|
|
588
|
+
codecs = {codec.artifact_name: codec for codec in self.artifact_codecs}
|
|
589
|
+
codec_pair = (
|
|
590
|
+
codecs["outer_gradient"].codec_id,
|
|
591
|
+
codecs["slow_weights"].codec_id,
|
|
592
|
+
)
|
|
593
|
+
if codec_pair not in {
|
|
594
|
+
("identity", "identity"),
|
|
595
|
+
("topk-bitmap-int8", "dense-int8"),
|
|
596
|
+
}:
|
|
597
|
+
raise ValueError("NoLoCo artifact codec combination is invalid")
|
|
598
|
+
if (
|
|
599
|
+
self.transport.chunk_size_bytes is None
|
|
600
|
+
or self.transport.window_size is None
|
|
601
|
+
):
|
|
602
|
+
raise ValueError("NoLoCo requires chunk size and window size")
|
|
603
|
+
return self
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
class Invitation(DomainModel):
|
|
607
|
+
protocol_version: Literal[1] = PROTOCOL_VERSION
|
|
608
|
+
run_id: RunId
|
|
609
|
+
initiator_public_key: PublicKey
|
|
610
|
+
bootstrap_uri: Annotated[str, StringConstraints(min_length=1, max_length=2048)]
|
|
611
|
+
draft_hash: Sha256
|
|
612
|
+
expected_participant_count: Annotated[
|
|
613
|
+
int,
|
|
614
|
+
Field(ge=MIN_PARTICIPANT_COUNT, le=MAX_PARTICIPANT_COUNT),
|
|
615
|
+
] = MIN_PARTICIPANT_COUNT
|
|
616
|
+
enrollment_expires_at: datetime | None = None
|
|
617
|
+
|
|
618
|
+
@model_validator(mode="after")
|
|
619
|
+
def valid_expected_membership(self) -> Self:
|
|
620
|
+
validate_participant_count(self.expected_participant_count)
|
|
621
|
+
return self
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
class SealedManifest(DraftRunSpec):
|
|
625
|
+
# Defaults are a draft-authoring convenience; sealed authority is explicit.
|
|
626
|
+
# Pydantic permits required overrides of defaulted, keyword-only fields.
|
|
627
|
+
algorithm_id: AlgorithmId = Field( # pyright: ignore[reportGeneralTypeIssues]
|
|
628
|
+
...
|
|
629
|
+
)
|
|
630
|
+
draft_hash: Sha256
|
|
631
|
+
participants: tuple[Participant, ...] = Field(
|
|
632
|
+
min_length=MIN_PARTICIPANT_COUNT,
|
|
633
|
+
max_length=MAX_PARTICIPANT_COUNT,
|
|
634
|
+
)
|
|
635
|
+
initial_checkpoint_hash: Sha256
|
|
636
|
+
tensor_schema: TensorSchema
|
|
637
|
+
|
|
638
|
+
@model_validator(mode="after")
|
|
639
|
+
def valid_membership(self) -> Self:
|
|
640
|
+
participant_count = len(self.participants)
|
|
641
|
+
validate_participant_count(participant_count)
|
|
642
|
+
keys = {participant.public_key for participant in self.participants}
|
|
643
|
+
indices = {participant.node_index for participant in self.participants}
|
|
644
|
+
if len(keys) != participant_count:
|
|
645
|
+
raise ValueError("participant public keys must be unique")
|
|
646
|
+
if indices != set(range(participant_count)):
|
|
647
|
+
raise ValueError(
|
|
648
|
+
"participant node indices must be exactly 0 through "
|
|
649
|
+
f"{participant_count - 1}"
|
|
650
|
+
)
|
|
651
|
+
if self.participant_count != participant_count:
|
|
652
|
+
raise ValueError(
|
|
653
|
+
"declared participant count must match sealed participant count"
|
|
654
|
+
)
|
|
655
|
+
return self
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
class SealedManifestExpectation(DomainModel):
|
|
659
|
+
"""Machine-local preflight constraints for one formed manifest."""
|
|
660
|
+
|
|
661
|
+
draft_hash: Sha256
|
|
662
|
+
participants: tuple[Participant, ...] = Field(
|
|
663
|
+
min_length=MIN_PARTICIPANT_COUNT,
|
|
664
|
+
max_length=MAX_PARTICIPANT_COUNT,
|
|
665
|
+
)
|
|
666
|
+
initial_checkpoint_hash: Sha256
|
|
667
|
+
tensor_schema_hash: Sha256
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
class OpaqueArtifactMetadata(DomainModel):
|
|
671
|
+
name: Identifier
|
|
672
|
+
size_bytes: Annotated[int, Field(gt=0)]
|
|
673
|
+
sha256: Sha256
|
|
674
|
+
codec_id: Identifier
|
|
675
|
+
logical_schema_hash: Sha256
|
|
676
|
+
encoded_schema_hash: Sha256
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
class OpaqueUpdateBundleMetadata(DomainModel):
|
|
680
|
+
"""Bundle metadata carried as the first artifact of every update."""
|
|
681
|
+
|
|
682
|
+
run_id: RunId
|
|
683
|
+
manifest_hash: Sha256
|
|
684
|
+
sender_public_key: PublicKey
|
|
685
|
+
algorithm_id: AlgorithmId
|
|
686
|
+
round_id: RoundId
|
|
687
|
+
artifacts: tuple[OpaqueArtifactMetadata, ...] = Field(min_length=1, max_length=16)
|
|
688
|
+
|
|
689
|
+
@model_validator(mode="after")
|
|
690
|
+
def unique_artifacts(self) -> Self:
|
|
691
|
+
names = [artifact.name for artifact in self.artifacts]
|
|
692
|
+
if len(names) != len(set(names)):
|
|
693
|
+
raise ValueError("artifact names must be unique")
|
|
694
|
+
return self
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Fixed run membership."""
|