dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,694 @@
1
+ """Validated domain models for run formation and update exchange."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import math
7
+ from datetime import datetime
8
+ from typing import Annotated, Literal, Self, cast
9
+
10
+ from pydantic import (
11
+ Field,
12
+ JsonValue,
13
+ StringConstraints,
14
+ field_validator,
15
+ model_validator,
16
+ )
17
+
18
+ from dromeus.errors import DraftError
19
+ from dromeus.protocol.models import (
20
+ CHUNK_FRAMING_BYTES,
21
+ AlgorithmId,
22
+ DomainModel,
23
+ Identifier,
24
+ PublicKey,
25
+ RoundId,
26
+ RunId,
27
+ Sha256,
28
+ TensorSchema,
29
+ )
30
+ from dromeus.protocol.models import (
31
+ MessageId as MessageId,
32
+ )
33
+ from dromeus.protocol.models import (
34
+ Tensor as Tensor,
35
+ )
36
+ from dromeus.protocol.models import (
37
+ TransferId as TransferId,
38
+ )
39
+ from dromeus.protocol.version import PROTOCOL_VERSION
40
+
41
+ MANIFEST_VERSION = 7
42
+ MIN_PARTICIPANT_COUNT = 4
43
+ MAX_PARTICIPANT_COUNT = 16
44
+ NOLOCO_ALGORITHM_ID = "noloco"
45
+
46
+ PackageVersion = Annotated[
47
+ str,
48
+ StringConstraints(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:+-]+$"),
49
+ ]
50
+ NodeIndex = Annotated[int, Field(ge=0, lt=MAX_PARTICIPANT_COUNT)]
51
+
52
+
53
+ class ParticipantCountError(DraftError):
54
+ """A run member count is outside the supported fixed-group range."""
55
+
56
+
57
+ def validate_participant_count(count: int) -> None:
58
+ """Validate the fixed even membership range shared by manifests and gossip."""
59
+ if count < MIN_PARTICIPANT_COUNT or count > MAX_PARTICIPANT_COUNT:
60
+ raise ParticipantCountError(
61
+ "participant count must be between 4 and 16 inclusive"
62
+ )
63
+ if count % 2:
64
+ raise ParticipantCountError("participant count must be even")
65
+
66
+
67
+ class Participant(DomainModel):
68
+ public_key: PublicKey
69
+ node_index: NodeIndex
70
+
71
+
72
+ class ConsensusSketchMessage(DomainModel):
73
+ sender_public_key: PublicKey
74
+ round_id: RoundId
75
+ payload: Annotated[bytes, Field(min_length=1)]
76
+
77
+
78
+ class ApplicationTaskContract(DomainModel):
79
+ """Application-defined input, target, preprocessing, and objective semantics.
80
+
81
+ The definition hash identifies the shared task, never node-local records.
82
+ """
83
+
84
+ dataset_id: Literal["application"] = "application"
85
+ task_id: Identifier
86
+ definition_hash: Sha256
87
+ objective: Literal["equal-node"] = "equal-node"
88
+
89
+
90
+ class EnvironmentFingerprint(DomainModel):
91
+ dromeus_version: Identifier
92
+ protocol_version: Literal[1] = PROTOCOL_VERSION
93
+ pytorch_version: PackageVersion
94
+ axl_version: Identifier
95
+ model_definition_hash: Sha256
96
+ container_image_digest: (
97
+ Annotated[str, StringConstraints(pattern=r"^sha256:[0-9a-f]{64}$")] | None
98
+ ) = None
99
+
100
+
101
+ class TransportLimits(DomainModel):
102
+ max_payload_bytes: Annotated[int, Field(gt=0)]
103
+ max_retries: Annotated[int, Field(ge=0)]
104
+ retry_timeout_seconds: Annotated[float, Field(gt=0)]
105
+ chunk_size_bytes: Annotated[int, Field(gt=0)] | None = None
106
+ window_size: Annotated[int, Field(gt=0)] | None = None
107
+ max_message_payload_bytes: Annotated[int, Field(gt=0)] | None = None
108
+ max_artifact_bytes: Annotated[int, Field(gt=0)] | None = None
109
+ max_concurrent_transfers: Annotated[int, Field(gt=0)] | None = None
110
+ max_inflight_bytes: Annotated[int, Field(gt=0)] | None = None
111
+ transfer_lifetime_seconds: Annotated[float, Field(gt=0)] | None = None
112
+ artifact_store_capacity_bytes: Annotated[int, Field(gt=0)] | None = None
113
+
114
+ @model_validator(mode="after")
115
+ def valid_transfer_bounds(self) -> Self:
116
+ if self.effective_chunk_size + CHUNK_FRAMING_BYTES > self.message_payload_limit:
117
+ raise ValueError(
118
+ "chunk size plus chunk framing must not exceed message payload limit"
119
+ )
120
+ if self.message_payload_limit > self.max_payload_bytes:
121
+ raise ValueError("message payload limit must not exceed bundle limit")
122
+ if self.artifact_size_limit > self.max_payload_bytes:
123
+ raise ValueError("artifact limit must not exceed bundle limit")
124
+ if (
125
+ self.effective_chunk_size * self.effective_window_size
126
+ > self.inflight_byte_limit
127
+ ):
128
+ raise ValueError("window bytes must not exceed in-flight byte limit")
129
+ if self.artifact_size_limit > self.artifact_store_capacity_limit:
130
+ raise ValueError("artifact limit must not exceed store capacity")
131
+ if self.inflight_byte_limit > self.artifact_store_capacity_limit:
132
+ raise ValueError("in-flight limit must not exceed store capacity")
133
+ minimum_lifetime = self.retry_timeout_seconds * (self.max_retries + 1)
134
+ if self.transfer_lifetime_limit < minimum_lifetime:
135
+ raise ValueError("transfer lifetime is shorter than retry budget")
136
+ return self
137
+
138
+ @property
139
+ def max_update_bundle_bytes(self) -> int:
140
+ """Return the byte limit for one whole update bundle."""
141
+ return self.max_payload_bytes
142
+
143
+ @property
144
+ def message_payload_limit(self) -> int:
145
+ return self.max_message_payload_bytes or self.max_payload_bytes
146
+
147
+ @property
148
+ def artifact_size_limit(self) -> int:
149
+ return self.max_artifact_bytes or self.max_payload_bytes
150
+
151
+ @property
152
+ def effective_chunk_size(self) -> int:
153
+ if self.chunk_size_bytes is not None:
154
+ return self.chunk_size_bytes
155
+ return max(
156
+ 1,
157
+ min(
158
+ self.artifact_size_limit,
159
+ self.message_payload_limit - CHUNK_FRAMING_BYTES,
160
+ ),
161
+ )
162
+
163
+ @property
164
+ def effective_window_size(self) -> int:
165
+ return self.window_size or 1
166
+
167
+ @property
168
+ def concurrent_transfer_limit(self) -> int:
169
+ return self.max_concurrent_transfers or 1
170
+
171
+ @property
172
+ def inflight_byte_limit(self) -> int:
173
+ return self.max_inflight_bytes or self.artifact_size_limit
174
+
175
+ @property
176
+ def transfer_lifetime_limit(self) -> float:
177
+ return self.transfer_lifetime_seconds or (
178
+ self.retry_timeout_seconds * (self.max_retries + 2)
179
+ )
180
+
181
+ @property
182
+ def artifact_store_capacity_limit(self) -> int:
183
+ return self.artifact_store_capacity_bytes or self.artifact_size_limit
184
+
185
+
186
+ class ConsensusSketchConfig(DomainModel):
187
+ size: Literal[4096] = 4096
188
+ seed: int
189
+
190
+
191
+ class NoLoCoConfig(DomainModel):
192
+ """Outer NoLoCo settings; the application owns its inner optimizer."""
193
+
194
+ alpha: Annotated[float, Field(ge=0.0, lt=1.0)]
195
+ beta: Annotated[float, Field(gt=0.0)]
196
+ gamma: Annotated[float, Field(gt=0.0)]
197
+ inner_steps: Annotated[int, Field(gt=0)]
198
+
199
+
200
+ class ArtifactCodec(DomainModel):
201
+ """Codec identity bound to one logical update artifact."""
202
+
203
+ artifact_name: Identifier
204
+ codec_id: Identifier
205
+ top_k_fraction: Annotated[float, Field(gt=0.0, le=1.0)] | None = None
206
+ lossy_allowed: bool | None = None
207
+
208
+ @model_validator(mode="after")
209
+ def valid_codec_settings(self) -> Self:
210
+ if self.codec_id == "identity":
211
+ if self.top_k_fraction is not None or self.lossy_allowed is True:
212
+ raise ValueError("identity codec cannot declare lossy settings")
213
+ return self
214
+ if self.codec_id == "topk-bitmap-int8":
215
+ if self.artifact_name != "outer_gradient":
216
+ raise ValueError("top-k codec is only valid for outer gradient")
217
+ if self.top_k_fraction is None or not self.lossy_allowed:
218
+ raise ValueError("top-k codec requires fraction and lossy permission")
219
+ return self
220
+ if self.codec_id == "dense-int8":
221
+ if self.artifact_name != "slow_weights":
222
+ raise ValueError("dense int8 codec is only valid for slow weights")
223
+ if self.top_k_fraction is not None or not self.lossy_allowed:
224
+ raise ValueError("dense int8 codec requires lossy permission")
225
+ return self
226
+ raise ValueError("unsupported NoLoCo artifact codec")
227
+
228
+
229
+ class UpdateCodecBinding(DomainModel):
230
+ """Logical codec identity and schema bound into one update artifact."""
231
+
232
+ codec_id: Identifier
233
+ logical_schema: TensorSchema
234
+
235
+
236
+ class PrivacyPolicy(DomainModel):
237
+ """Fixed record-level Poisson Gaussian mechanism and release policy."""
238
+
239
+ mode: Literal["record_dp"] = "record_dp"
240
+ adjacency: Literal["add_remove"] = "add_remove"
241
+ sampling: Literal["poisson"] = "poisson"
242
+ clipping: Literal["flat"] = "flat"
243
+ max_grad_norm: Annotated[float, Field(gt=0, allow_inf_nan=False)]
244
+ noise_multiplier: Annotated[float, Field(gt=0, allow_inf_nan=False)]
245
+ accountant: Literal["prv"] = "prv"
246
+ delta: Annotated[float, Field(gt=0, lt=1, allow_inf_nan=False)]
247
+ max_logical_steps: Annotated[int, Field(gt=0)]
248
+ epsilon_limit: Annotated[float, Field(gt=0, allow_inf_nan=False)] | None = None
249
+ randomness_profile: Literal["secure", "reproducible_public_benchmark"]
250
+ metric_policy: Literal["public_eval_only"] = "public_eval_only"
251
+ prv_epsilon_error: Annotated[float, Field(gt=0, allow_inf_nan=False)] = 0.01
252
+ prv_delta_error_ratio: Annotated[float, Field(gt=0, lt=1)] = 0.001
253
+
254
+
255
+ class DivergencePolicy(DomainModel):
256
+ """Explicit pilot thresholds; no universal detector defaults."""
257
+
258
+ version: Literal[1] = 1
259
+ mode: Literal["warn"] = "warn"
260
+ threshold_set_id: Identifier
261
+ warmup_rounds: Annotated[int, Field(ge=0)]
262
+ window_rounds: Annotated[int, Field(ge=4, le=4096)]
263
+ patience_windows: Annotated[int, Field(gt=0)]
264
+ distance_floor: Annotated[float, Field(gt=0, allow_inf_nan=False)]
265
+ growth_ratio: Annotated[float, Field(gt=1, allow_inf_nan=False)]
266
+ min_log_slope: Annotated[float, Field(gt=0, allow_inf_nan=False)]
267
+ severe_distance: Annotated[float, Field(gt=0, allow_inf_nan=False)]
268
+ severe_patience: Annotated[int, Field(gt=0)]
269
+ recovery_windows: Annotated[int, Field(gt=0)]
270
+ recovery_distance: Annotated[float, Field(gt=0, allow_inf_nan=False)]
271
+ max_lag_rounds: Annotated[int, Field(ge=0, le=4096)]
272
+ absolute_distance_floor: (
273
+ Annotated[float, Field(gt=0, allow_inf_nan=False)] | None
274
+ ) = None
275
+ absolute_severe_distance: (
276
+ Annotated[float, Field(gt=0, allow_inf_nan=False)] | None
277
+ ) = None
278
+ absolute_recovery_distance: (
279
+ Annotated[float, Field(gt=0, allow_inf_nan=False)] | None
280
+ ) = None
281
+
282
+ @model_validator(mode="after")
283
+ def ordered_thresholds(self) -> Self:
284
+ if self.recovery_distance >= self.severe_distance:
285
+ raise ValueError("recovery threshold must be below severe threshold")
286
+ absolute = (
287
+ self.absolute_distance_floor,
288
+ self.absolute_severe_distance,
289
+ self.absolute_recovery_distance,
290
+ )
291
+ if any(x is not None for x in absolute):
292
+ if any(x is None for x in absolute):
293
+ raise ValueError("absolute scale thresholds must be provided together")
294
+ assert (
295
+ self.absolute_severe_distance is not None
296
+ and self.absolute_recovery_distance is not None
297
+ )
298
+ if self.absolute_recovery_distance >= self.absolute_severe_distance:
299
+ raise ValueError("absolute recovery must be below severe threshold")
300
+ return self
301
+
302
+
303
+ class PrivateOptimizerSpec(DomainModel):
304
+ """One complete named group, ordinary dense FP32 PyTorch execution."""
305
+
306
+ name: Literal["sgd", "adam", "adamw"]
307
+ torch_version: PackageVersion
308
+ parameter_names: tuple[Identifier, ...] = Field(min_length=1)
309
+ learning_rate: Annotated[float, Field(gt=0, allow_inf_nan=False)]
310
+ weight_decay: Annotated[float, Field(ge=0, allow_inf_nan=False)] = 0
311
+ momentum: Annotated[float, Field(ge=0, lt=1)] = 0
312
+ dampening: Annotated[float, Field(ge=0)] = 0
313
+ nesterov: bool = False
314
+ betas: tuple[
315
+ Annotated[float, Field(ge=0, lt=1)], Annotated[float, Field(ge=0, lt=1)]
316
+ ] = (0.9, 0.999)
317
+ numerical_epsilon: Annotated[float, Field(gt=0, allow_inf_nan=False)] = 1e-8
318
+ amsgrad: Literal[False] = False
319
+ foreach: Literal[False] = False
320
+ fused: Literal[False] = False
321
+ capturable: Literal[False] = False
322
+ differentiable: Literal[False] = False
323
+ maximize: Literal[False] = False
324
+
325
+ @model_validator(mode="after")
326
+ def valid_options(self) -> Self:
327
+ if len(set(self.parameter_names)) != len(self.parameter_names):
328
+ raise ValueError("parameter names must be unique and ordered")
329
+ if self.name != "sgd" and (self.momentum or self.dampening or self.nesterov):
330
+ raise ValueError("momentum options require SGD")
331
+ if self.nesterov and (not self.momentum or self.dampening):
332
+ raise ValueError("Nesterov requires momentum and zero dampening")
333
+ return self
334
+
335
+
336
+ class OptimizerParameterGroup(DomainModel):
337
+ """Ordered parameter membership and resolved JSON optimizer options."""
338
+
339
+ parameter_names: tuple[Identifier, ...] = Field(min_length=1, max_length=65536)
340
+ options: dict[str, JsonValue]
341
+
342
+ @field_validator("options", mode="before")
343
+ @classmethod
344
+ def bounded_json_options(cls, value: object) -> object:
345
+ """Bound untrusted metadata before Pydantic's recursive JSON parsing."""
346
+ pending: list[tuple[object, int]] = [(value, 0)]
347
+ remaining = 4096
348
+ while pending:
349
+ item, depth = pending.pop()
350
+ remaining -= 1
351
+ if depth > 8 or remaining < 0:
352
+ raise ValueError(
353
+ "optimizer options exceed metadata depth or size limit"
354
+ )
355
+ if isinstance(item, dict):
356
+ mapping = cast(dict[object, object], item)
357
+ if len(mapping) > remaining:
358
+ raise ValueError("optimizer options exceed metadata size limit")
359
+ for key, child in mapping.items():
360
+ if not isinstance(key, str) or not key:
361
+ raise ValueError(
362
+ "optimizer option keys must be nonempty strings"
363
+ )
364
+ pending.append((child, depth + 1))
365
+ elif isinstance(item, list):
366
+ sequence = cast(list[object], item)
367
+ if len(sequence) > remaining:
368
+ raise ValueError("optimizer options exceed metadata size limit")
369
+ pending.extend((child, depth + 1) for child in sequence)
370
+ elif item is not None and type(item) not in (str, bool, int, float):
371
+ raise ValueError("optimizer options require JSON values")
372
+ elif isinstance(item, float) and not math.isfinite(item):
373
+ raise ValueError("optimizer options must contain finite numbers")
374
+ if len(json.dumps(value, allow_nan=False).encode()) > 65536:
375
+ raise ValueError("optimizer options exceed metadata size limit")
376
+ return value
377
+
378
+ @model_validator(mode="after")
379
+ def valid_options(self) -> Self:
380
+ if len(set(self.parameter_names)) != len(self.parameter_names):
381
+ raise ValueError("parameter names must be unique and ordered")
382
+ if "params" in self.options:
383
+ raise ValueError("parameter membership belongs in parameter_names")
384
+ learning_rate = self.options.get("lr")
385
+ if (
386
+ isinstance(learning_rate, bool)
387
+ or not isinstance(learning_rate, (int, float))
388
+ or learning_rate <= 0
389
+ ):
390
+ raise ValueError("optimizer lr must be a positive finite scalar")
391
+ for flag in ("foreach", "fused", "capturable", "differentiable", "maximize"):
392
+ if flag in self.options and self.options[flag] is not False:
393
+ raise ValueError(f"optimizer {flag} must be explicitly false")
394
+ return self
395
+
396
+
397
+ class FactoryOptimizerSpec(DomainModel):
398
+ """Group-agreed implementation and state-validator identity for a factory."""
399
+
400
+ kind: Literal["factory"] = "factory"
401
+ implementation_id: Annotated[
402
+ str, StringConstraints(min_length=1, max_length=256, pattern=r"^\S+$")
403
+ ]
404
+ implementation_hash: Sha256
405
+ torch_version: PackageVersion
406
+ groups: tuple[OptimizerParameterGroup, ...] = Field(min_length=1, max_length=32)
407
+
408
+ @property
409
+ def parameter_names(self) -> tuple[str, ...]:
410
+ return tuple(name for group in self.groups for name in group.parameter_names)
411
+
412
+ @model_validator(mode="after")
413
+ def unique_parameters(self) -> Self:
414
+ names = self.parameter_names
415
+ if len(set(names)) != len(names):
416
+ raise ValueError("optimizer parameter groups must not overlap")
417
+ return self
418
+
419
+
420
+ GradientBackend = Literal["opacus_hooks", "opacus_native_b1"]
421
+
422
+
423
+ class AccountedInitializationAuthority(DomainModel):
424
+ """An existing protected lineage whose spending precedes initialization."""
425
+
426
+ lineage_id: Annotated[
427
+ str, StringConstraints(min_length=1, max_length=256, pattern=r"\S")
428
+ ]
429
+ ledger_sha256: Sha256
430
+ reserved_steps: Annotated[int, Field(gt=0)]
431
+ reservation_count: Annotated[int, Field(gt=0)]
432
+
433
+
434
+ class AccountedInitializationProvenance(DomainModel):
435
+ """Bound ancestry of a sanitized model, never a fresh privacy allowance."""
436
+
437
+ parent_run_id: RunId
438
+ parent_model_sha256: Sha256
439
+ parent_audit_sha256: Sha256
440
+ accounting_authorities: tuple[AccountedInitializationAuthority, ...] = Field(
441
+ min_length=1, max_length=16
442
+ )
443
+
444
+ @model_validator(mode="after")
445
+ def distinct_lineages(self) -> Self:
446
+ names = [row.lineage_id for row in self.accounting_authorities]
447
+ if len(names) != len(set(names)):
448
+ raise ValueError("initialization accounting lineages must be unique")
449
+ return self
450
+
451
+
452
+ class TaskTrainingSpec(DomainModel):
453
+ """Shared semantics for an application-defined finite-record training task."""
454
+
455
+ optimizer: PrivateOptimizerSpec | FactoryOptimizerSpec
456
+ batch_size: Annotated[int, Field(gt=0)]
457
+ sampling: Literal["poisson", "shuffle"] = "poisson"
458
+ schedule: Literal["constant", "linear_decay"] = "constant"
459
+ initialization: Literal[
460
+ "public_untrained", "public_pretrained", "accounted_private_pretrained"
461
+ ] = "public_untrained"
462
+ initialization_provenance: AccountedInitializationProvenance | None = None
463
+ evaluation_provenance: Literal["public_fixed_holdout", "disabled"] = "disabled"
464
+ task_definition_hash: Sha256
465
+ implementation_hash: Sha256
466
+ batch_definition_hash: Sha256
467
+ loss_definition_hash: Sha256
468
+ record_unit: Annotated[
469
+ str, StringConstraints(min_length=1, max_length=256, pattern=r"\S")
470
+ ]
471
+ gradient_backend: GradientBackend = "opacus_hooks"
472
+ trainable_parameter_names: tuple[Identifier, ...] = Field(
473
+ min_length=1, max_length=65536
474
+ )
475
+ parameter_aliases: tuple[tuple[Identifier, ...], ...] = Field(
476
+ default=(), max_length=65536
477
+ )
478
+ public_buffer_hashes: dict[Identifier, Sha256] = Field(
479
+ default_factory=dict, max_length=65536
480
+ )
481
+ frozen_parameter_hashes: dict[Identifier, Sha256] = Field(
482
+ default_factory=dict, max_length=65536
483
+ )
484
+
485
+ @model_validator(mode="after")
486
+ def parameter_state_is_unambiguous(self) -> Self:
487
+ if (self.initialization == "accounted_private_pretrained") != (
488
+ self.initialization_provenance is not None
489
+ ):
490
+ raise ValueError(
491
+ "accounted private initialization requires exclusive provenance"
492
+ )
493
+ trainable = set(self.trainable_parameter_names)
494
+ if len(trainable) != len(self.trainable_parameter_names):
495
+ raise ValueError("trainable parameter names must be unique")
496
+ if trainable != set(self.optimizer.parameter_names):
497
+ raise ValueError(
498
+ "optimizer parameter names must match trainable parameters"
499
+ )
500
+ frozen = set(self.frozen_parameter_hashes)
501
+ if trainable & frozen:
502
+ raise ValueError("trainable and frozen parameter names must not overlap")
503
+ canonical = trainable | frozen
504
+ aliases: set[str] = set()
505
+ for group in self.parameter_aliases:
506
+ if len(group) < 2:
507
+ raise ValueError("parameter alias groups require at least two names")
508
+ names = set(group)
509
+ if len(names) != len(group):
510
+ raise ValueError("parameter alias names must be distinct")
511
+ if aliases & names:
512
+ raise ValueError("parameter alias groups must be disjoint")
513
+ if len(names & canonical) != 1:
514
+ raise ValueError(
515
+ "alias group must identify exactly one canonical parameter"
516
+ )
517
+ aliases.update(names)
518
+ if set(self.public_buffer_hashes) & (canonical | aliases):
519
+ raise ValueError("buffer and parameter names must not overlap")
520
+ return self
521
+
522
+
523
+ class PrivacyReservation(DomainModel):
524
+ version: Literal[1] = 1
525
+ run_id: RunId
526
+ policy_hash: Sha256
527
+ sample_rate: Annotated[float, Field(gt=0, le=1, allow_inf_nan=False)]
528
+ noise_multiplier: Annotated[float, Field(gt=0, allow_inf_nan=False)]
529
+ logical_steps: Annotated[int, Field(gt=0)]
530
+ accountant_version: Literal["opacus-1.6.0-prv"] = "opacus-1.6.0-prv"
531
+
532
+
533
+ class DraftRunSpec(DomainModel):
534
+ """Shared run definition every participant must agree on before formation."""
535
+
536
+ manifest_version: Literal[7] = MANIFEST_VERSION
537
+ protocol_version: Literal[1] = PROTOCOL_VERSION
538
+ run_id: RunId
539
+ algorithm_id: AlgorithmId = NOLOCO_ALGORITHM_ID
540
+ model_id: Identifier
541
+ model_definition_hash: Sha256
542
+ dataset: ApplicationTaskContract
543
+ expected_participant_count: Annotated[
544
+ int, Field(ge=MIN_PARTICIPANT_COUNT, le=MAX_PARTICIPANT_COUNT)
545
+ ]
546
+ environment: EnvironmentFingerprint
547
+ local_steps: Annotated[int, Field(gt=0)]
548
+ round_count: Annotated[int, Field(gt=0)]
549
+ peer_scheduler_seed: int
550
+ codec_id: Literal["safetensors"]
551
+ transport: TransportLimits
552
+ consensus_sketch: ConsensusSketchConfig
553
+ training: TaskTrainingSpec
554
+ algorithm_config: NoLoCoConfig
555
+ artifact_codecs: tuple[ArtifactCodec, ...]
556
+
557
+ privacy: PrivacyPolicy | None = None
558
+ divergence_detection: DivergencePolicy | None = None
559
+
560
+ @property
561
+ def participant_count(self) -> int:
562
+ return self.expected_participant_count
563
+
564
+ @model_validator(mode="after")
565
+ def valid_manifest(self) -> Self:
566
+ validate_participant_count(self.expected_participant_count)
567
+ if self.algorithm_id != NOLOCO_ALGORITHM_ID:
568
+ raise ValueError("manifest requires the noloco algorithm")
569
+ if self.environment.model_definition_hash != self.model_definition_hash:
570
+ raise ValueError("environment model hash does not match draft")
571
+ if self.training.task_definition_hash != self.dataset.definition_hash:
572
+ raise ValueError("training task definition must match dataset contract")
573
+ if self.privacy is not None:
574
+ if self.training.sampling != "poisson":
575
+ raise ValueError("private tasks require Poisson sampling")
576
+ if self.privacy.max_logical_steps != self.local_steps * self.round_count:
577
+ raise ValueError("privacy horizon must match local steps and rounds")
578
+ if self.local_steps != self.algorithm_config.inner_steps:
579
+ raise ValueError("local steps must match NoLoCo inner steps")
580
+ names = [codec.artifact_name for codec in self.artifact_codecs]
581
+ if len(names) != len(set(names)) or set(names) != {
582
+ "outer_gradient",
583
+ "slow_weights",
584
+ }:
585
+ raise ValueError(
586
+ "NoLoCo codec IDs must cover outer_gradient and slow_weights"
587
+ )
588
+ codecs = {codec.artifact_name: codec for codec in self.artifact_codecs}
589
+ codec_pair = (
590
+ codecs["outer_gradient"].codec_id,
591
+ codecs["slow_weights"].codec_id,
592
+ )
593
+ if codec_pair not in {
594
+ ("identity", "identity"),
595
+ ("topk-bitmap-int8", "dense-int8"),
596
+ }:
597
+ raise ValueError("NoLoCo artifact codec combination is invalid")
598
+ if (
599
+ self.transport.chunk_size_bytes is None
600
+ or self.transport.window_size is None
601
+ ):
602
+ raise ValueError("NoLoCo requires chunk size and window size")
603
+ return self
604
+
605
+
606
+ class Invitation(DomainModel):
607
+ protocol_version: Literal[1] = PROTOCOL_VERSION
608
+ run_id: RunId
609
+ initiator_public_key: PublicKey
610
+ bootstrap_uri: Annotated[str, StringConstraints(min_length=1, max_length=2048)]
611
+ draft_hash: Sha256
612
+ expected_participant_count: Annotated[
613
+ int,
614
+ Field(ge=MIN_PARTICIPANT_COUNT, le=MAX_PARTICIPANT_COUNT),
615
+ ] = MIN_PARTICIPANT_COUNT
616
+ enrollment_expires_at: datetime | None = None
617
+
618
+ @model_validator(mode="after")
619
+ def valid_expected_membership(self) -> Self:
620
+ validate_participant_count(self.expected_participant_count)
621
+ return self
622
+
623
+
624
+ class SealedManifest(DraftRunSpec):
625
+ # Defaults are a draft-authoring convenience; sealed authority is explicit.
626
+ # Pydantic permits required overrides of defaulted, keyword-only fields.
627
+ algorithm_id: AlgorithmId = Field( # pyright: ignore[reportGeneralTypeIssues]
628
+ ...
629
+ )
630
+ draft_hash: Sha256
631
+ participants: tuple[Participant, ...] = Field(
632
+ min_length=MIN_PARTICIPANT_COUNT,
633
+ max_length=MAX_PARTICIPANT_COUNT,
634
+ )
635
+ initial_checkpoint_hash: Sha256
636
+ tensor_schema: TensorSchema
637
+
638
+ @model_validator(mode="after")
639
+ def valid_membership(self) -> Self:
640
+ participant_count = len(self.participants)
641
+ validate_participant_count(participant_count)
642
+ keys = {participant.public_key for participant in self.participants}
643
+ indices = {participant.node_index for participant in self.participants}
644
+ if len(keys) != participant_count:
645
+ raise ValueError("participant public keys must be unique")
646
+ if indices != set(range(participant_count)):
647
+ raise ValueError(
648
+ "participant node indices must be exactly 0 through "
649
+ f"{participant_count - 1}"
650
+ )
651
+ if self.participant_count != participant_count:
652
+ raise ValueError(
653
+ "declared participant count must match sealed participant count"
654
+ )
655
+ return self
656
+
657
+
658
+ class SealedManifestExpectation(DomainModel):
659
+ """Machine-local preflight constraints for one formed manifest."""
660
+
661
+ draft_hash: Sha256
662
+ participants: tuple[Participant, ...] = Field(
663
+ min_length=MIN_PARTICIPANT_COUNT,
664
+ max_length=MAX_PARTICIPANT_COUNT,
665
+ )
666
+ initial_checkpoint_hash: Sha256
667
+ tensor_schema_hash: Sha256
668
+
669
+
670
+ class OpaqueArtifactMetadata(DomainModel):
671
+ name: Identifier
672
+ size_bytes: Annotated[int, Field(gt=0)]
673
+ sha256: Sha256
674
+ codec_id: Identifier
675
+ logical_schema_hash: Sha256
676
+ encoded_schema_hash: Sha256
677
+
678
+
679
+ class OpaqueUpdateBundleMetadata(DomainModel):
680
+ """Bundle metadata carried as the first artifact of every update."""
681
+
682
+ run_id: RunId
683
+ manifest_hash: Sha256
684
+ sender_public_key: PublicKey
685
+ algorithm_id: AlgorithmId
686
+ round_id: RoundId
687
+ artifacts: tuple[OpaqueArtifactMetadata, ...] = Field(min_length=1, max_length=16)
688
+
689
+ @model_validator(mode="after")
690
+ def unique_artifacts(self) -> Self:
691
+ names = [artifact.name for artifact in self.artifacts]
692
+ if len(names) != len(set(names)):
693
+ raise ValueError("artifact names must be unique")
694
+ return self
@@ -0,0 +1 @@
1
+ """Fixed run membership."""