paramesh 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. paramesh-1.1.0/ARCHITECTURE.md +180 -0
  2. paramesh-1.1.0/BENCHMARKS.md +238 -0
  3. paramesh-1.1.0/LICENSE +201 -0
  4. paramesh-1.1.0/MANIFEST.in +4 -0
  5. paramesh-1.1.0/PKG-INFO +574 -0
  6. paramesh-1.1.0/README.md +539 -0
  7. paramesh-1.1.0/ROADMAP.md +148 -0
  8. paramesh-1.1.0/SECURITY.md +111 -0
  9. paramesh-1.1.0/examples/compose-bounded.json +10 -0
  10. paramesh-1.1.0/examples/compose.json +9 -0
  11. paramesh-1.1.0/examples/external_benchmark.py +43 -0
  12. paramesh-1.1.0/examples/external_linear.py +31 -0
  13. paramesh-1.1.0/examples/external_model_families.py +32 -0
  14. paramesh-1.1.0/examples/local.json +9 -0
  15. paramesh-1.1.0/examples/recovery_demo.py +20 -0
  16. paramesh-1.1.0/examples/scaling_matrix.py +31 -0
  17. paramesh-1.1.0/proto/paramesh/rpc/training.proto +222 -0
  18. paramesh-1.1.0/pyproject.toml +74 -0
  19. paramesh-1.1.0/scripts/__init__.py +1 -0
  20. paramesh-1.1.0/scripts/create_dev_env.py +55 -0
  21. paramesh-1.1.0/scripts/generate_grafana.py +75 -0
  22. paramesh-1.1.0/scripts/generate_proto.py +39 -0
  23. paramesh-1.1.0/scripts/immutable_release.py +445 -0
  24. paramesh-1.1.0/scripts/quickstart.py +48 -0
  25. paramesh-1.1.0/scripts/standalone_launcher.py +13 -0
  26. paramesh-1.1.0/scripts/verify_compose.py +238 -0
  27. paramesh-1.1.0/scripts/verify_immutable_compose.py +81 -0
  28. paramesh-1.1.0/scripts/verify_operations.py +571 -0
  29. paramesh-1.1.0/scripts/windows_launcher.py +26 -0
  30. paramesh-1.1.0/setup.cfg +4 -0
  31. paramesh-1.1.0/src/paramesh/__init__.py +1 -0
  32. paramesh-1.1.0/src/paramesh/benchmarks.py +291 -0
  33. paramesh-1.1.0/src/paramesh/checkpoint.py +442 -0
  34. paramesh-1.1.0/src/paramesh/cli.py +388 -0
  35. paramesh-1.1.0/src/paramesh/config.py +35 -0
  36. paramesh-1.1.0/src/paramesh/coordinator.py +465 -0
  37. paramesh-1.1.0/src/paramesh/dashboard.py +214 -0
  38. paramesh-1.1.0/src/paramesh/demo.py +514 -0
  39. paramesh-1.1.0/src/paramesh/events.py +197 -0
  40. paramesh-1.1.0/src/paramesh/journal.py +79 -0
  41. paramesh-1.1.0/src/paramesh/local_launcher.py +85 -0
  42. paramesh-1.1.0/src/paramesh/monitor.py +391 -0
  43. paramesh-1.1.0/src/paramesh/operations.py +159 -0
  44. paramesh-1.1.0/src/paramesh/py.typed +0 -0
  45. paramesh-1.1.0/src/paramesh/recovery_benchmarks.py +151 -0
  46. paramesh-1.1.0/src/paramesh/rpc/__init__.py +1 -0
  47. paramesh-1.1.0/src/paramesh/rpc/training_pb2.py +106 -0
  48. paramesh-1.1.0/src/paramesh/rpc/training_pb2.pyi +370 -0
  49. paramesh-1.1.0/src/paramesh/rpc/training_pb2_grpc.py +771 -0
  50. paramesh-1.1.0/src/paramesh/scaling_benchmarks.py +280 -0
  51. paramesh-1.1.0/src/paramesh/security.py +158 -0
  52. paramesh-1.1.0/src/paramesh/services.py +210 -0
  53. paramesh-1.1.0/src/paramesh/shard.py +376 -0
  54. paramesh-1.1.0/src/paramesh/state.py +75 -0
  55. paramesh-1.1.0/src/paramesh/storage.py +197 -0
  56. paramesh-1.1.0/src/paramesh/telemetry.py +357 -0
  57. paramesh-1.1.0/src/paramesh/tensors.py +63 -0
  58. paramesh-1.1.0/src/paramesh/web/app.js +312 -0
  59. paramesh-1.1.0/src/paramesh/web/favicon.svg +1 -0
  60. paramesh-1.1.0/src/paramesh/web/index.html +65 -0
  61. paramesh-1.1.0/src/paramesh/web/style.css +160 -0
  62. paramesh-1.1.0/src/paramesh/worker.py +356 -0
  63. paramesh-1.1.0/src/paramesh/workload.py +166 -0
  64. paramesh-1.1.0/src/paramesh.egg-info/PKG-INFO +574 -0
  65. paramesh-1.1.0/src/paramesh.egg-info/SOURCES.txt +97 -0
  66. paramesh-1.1.0/src/paramesh.egg-info/dependency_links.txt +1 -0
  67. paramesh-1.1.0/src/paramesh.egg-info/entry_points.txt +3 -0
  68. paramesh-1.1.0/src/paramesh.egg-info/requires.txt +30 -0
  69. paramesh-1.1.0/src/paramesh.egg-info/top_level.txt +1 -0
  70. paramesh-1.1.0/tests/test_azure.py +253 -0
  71. paramesh-1.1.0/tests/test_benchmark_evidence.py +80 -0
  72. paramesh-1.1.0/tests/test_benchmarks.py +250 -0
  73. paramesh-1.1.0/tests/test_checkpoint.py +336 -0
  74. paramesh-1.1.0/tests/test_checkpoint_epochs.py +1012 -0
  75. paramesh-1.1.0/tests/test_coordinator.py +85 -0
  76. paramesh-1.1.0/tests/test_dashboard.py +72 -0
  77. paramesh-1.1.0/tests/test_environment_script.py +53 -0
  78. paramesh-1.1.0/tests/test_events.py +178 -0
  79. paramesh-1.1.0/tests/test_failures.py +102 -0
  80. paramesh-1.1.0/tests/test_fencing.py +61 -0
  81. paramesh-1.1.0/tests/test_immutable_release.py +103 -0
  82. paramesh-1.1.0/tests/test_journal.py +36 -0
  83. paramesh-1.1.0/tests/test_model_families.py +23 -0
  84. paramesh-1.1.0/tests/test_monitor.py +218 -0
  85. paramesh-1.1.0/tests/test_operations.py +101 -0
  86. paramesh-1.1.0/tests/test_quickstart.py +53 -0
  87. paramesh-1.1.0/tests/test_recovery_processes.py +21 -0
  88. paramesh-1.1.0/tests/test_rpc.py +62 -0
  89. paramesh-1.1.0/tests/test_s3.py +99 -0
  90. paramesh-1.1.0/tests/test_scaling_benchmarks.py +87 -0
  91. paramesh-1.1.0/tests/test_security.py +139 -0
  92. paramesh-1.1.0/tests/test_shard.py +115 -0
  93. paramesh-1.1.0/tests/test_shard_persistence.py +51 -0
  94. paramesh-1.1.0/tests/test_standalone_launcher.py +21 -0
  95. paramesh-1.1.0/tests/test_state.py +47 -0
  96. paramesh-1.1.0/tests/test_telemetry.py +183 -0
  97. paramesh-1.1.0/tests/test_tensors.py +58 -0
  98. paramesh-1.1.0/tests/test_training.py +86 -0
  99. paramesh-1.1.0/tests/test_worker.py +217 -0
@@ -0,0 +1,180 @@
1
+ # Architecture
2
+
3
+ ## Control plane
4
+
5
+ One coordinator serves one immutable run configuration. Configuration includes
6
+ the run ID, fixed worker membership, ordered shard endpoints, synchronization mode,
7
+ learning rate, aggregation rule and barrier timeout. Run creation is currently a
8
+ local JSON configuration operation, not a mutable remote API.
9
+
10
+ Workers register using protocol version 1 and an allowed worker ID. A random
11
+ session token identifies a control-plane lease. A live identity cannot be claimed
12
+ twice. The coordinator uses a monotonic clock: a node is initially `registered`,
13
+ then `healthy` after a heartbeat, `suspect` after 5 seconds and `unavailable` after
14
+ 15 seconds. Thresholds can be set when constructing the coordinator. A completed
15
+ worker remains `completed`, rather than appearing as a failed node later.
16
+
17
+ Expired workers may register a fresh control-plane session; old tokens cannot
18
+ send heartbeats. The coordinator durably reserves a monotonically increasing
19
+ generation, fences it on every shard, then issues the new session. An interrupted
20
+ partial fence never reuses a generation. Shard updates reject superseded generations.
21
+
22
+ Run configuration, generation counters and progress survive in checksummed SQLite
23
+ state. Restart invalidates prior leases and requires re-registration; it does not
24
+ pretend saved monotonic-clock timestamps are live leases. `global_step` is a
25
+ reported completed-step lower watermark across configured workers, not a global
26
+ atomic model version. Shard endpoints remain configured; dynamic registration
27
+ and coordinator replication are not implemented.
28
+
29
+ ## Data plane and ownership
30
+
31
+ Workers talk directly to shards over gRPC. Parameter ownership is:
32
+
33
+ ```text
34
+ int.from_bytes(sha256(parameter_name.encode()).digest()[:8], "big") % shard_count
35
+ ```
36
+
37
+ Shard index is the endpoint's position in the immutable configuration. Python's
38
+ randomized built-in hash is never used. Renaming a parameter or changing the
39
+ shard count can move ownership; live resharding is unsupported. Whole tensors
40
+ are assigned to shards, so load can be uneven and shards can be empty for very
41
+ small models. The two-shard MLP test asserts both shards own parameters.
42
+
43
+ The worker model performs forward and backward passes with PyTorch. Each shard
44
+ stores float32 NumPy weights and applies SGD itself. Plain SGD has no momentum
45
+ or Adam state; weights, version and replay receipts are authoritative. Supporting
46
+ a stateful optimizer will require including its state in checkpoints.
47
+
48
+ ## Protocol and validation
49
+
50
+ Source: `proto/paramesh/rpc/training.proto`; package: `paramesh.v1`. Python bindings
51
+ and message `.pyi` files are generated and checked into source control. CI compares
52
+ fresh generation with committed bytes. New compatible fields can be added without
53
+ reusing existing field numbers. The wire version is early and not a stable public
54
+ API commitment yet.
55
+
56
+ Tensors carry a name, shape, explicit `float32-le` dtype and little-endian bytes.
57
+ The decoder allows at most eight dimensions, 4 MiB per tensor, 1,024 tensors and
58
+ 8 MiB of tensor data per collection. RPC messages themselves are capped at 8 MiB,
59
+ including protobuf overhead. Scalars are allowed; zero-sized tensors are not.
60
+ Neither network endpoints nor model loading accept arbitrary Python object files.
61
+
62
+ Each update validates run identity, configured worker membership, exact parameter
63
+ names and shapes, dtype, payload length, finite values, update sequence and base
64
+ version. Decoding happens before entering the optimizer critical section. A
65
+ condition lock serializes commits. Proposed new weights are computed separately
66
+ and checked for overflow before replacing any authoritative weight.
67
+
68
+ ## Versions, sequence numbers and retries
69
+
70
+ Each shard has its own monotonically increasing version, starting at zero.
71
+ Every accepted optimizer step increments it once, even for an empty shard. A
72
+ worker's update sequence starts at zero and increases by one per local iteration.
73
+ These are distinct counters: async shard versions count every worker's updates.
74
+
75
+ The shard remembers the latest accepted sequence, SHA-256 request fingerprint,
76
+ resulting version and staleness for each configured worker. An identical retry
77
+ returns that receipt without applying SGD again. A changed retry or sequence gap
78
+ is rejected. Memory is bounded by membership and one in-flight barrier. Older
79
+ receipts are not retained. Weights, SGD configuration, versions, receipts and fences
80
+ are committed together to SQLite WAL with `synchronous=FULL` before acknowledgement.
81
+ This supports retry after a process restart when the volume survives; it is not
82
+ an exactly-once delivery claim after storage loss.
83
+
84
+ Workers use explicit RPC deadlines and retry transient unavailable/deadline errors
85
+ at most three times with short backoff. They reuse the original request bytes;
86
+ they do not recompute a gradient after an ambiguous transport failure. Calls use
87
+ wait-for-ready within their deadlines so reconnection can outlast a transient
88
+ failure rather than exhausting retries immediately. A test
89
+ simulates losing the response after commit and verifies one update, not two.
90
+
91
+ ## Synchronization
92
+
93
+ ### Synchronous
94
+
95
+ Every worker submits one gradient per version. The shard accumulates a fixed
96
+ membership barrier, computes mean or sum and commits one SGD update. Float64
97
+ accumulation reduces summation overflow before conversion to float32 weights.
98
+ Mean gives equal weight to each worker, which is equivalent to the demo's local
99
+ reference because each worker has the same number of examples.
100
+
101
+ A shared deadline takes the earliest participant deadline, capped by the configured
102
+ barrier timeout. Late final arrivals cannot commit merely because an expired
103
+ waiting thread has not resumed yet. A timeout clears that shard's pending contributions and wakes waiting calls.
104
+ Membership is not silently reduced. The server's RPC thread limit exceeds the
105
+ configured worker count, preventing waiting participants from starving the final
106
+ arrival. No distributed commit ties separate shard barriers together.
107
+
108
+ ### Asynchronous
109
+
110
+ Each valid gradient commits immediately. Staleness is `current_shard_version -
111
+ base_version` and is returned to the worker. Ordering depends on process and
112
+ network scheduling; async training is not expected to match synchronous SGD.
113
+
114
+ ### Bounded lag
115
+
116
+ An update whose shard-local staleness exceeds `max_staleness` receives
117
+ `FAILED_PRECONDITION`. The worker pulls fresh weights and recomputes gradients
118
+ for contributions that have not yet committed in its current iteration. Already
119
+ accepted shards are not submitted again with changed contents. Refreshes are
120
+ bounded to 100 attempts; exhaustion fails the worker visibly.
121
+
122
+ Consequently a worker iteration can combine contributions computed from different
123
+ model snapshots. The bound is enforced at each shard's acceptance point. There is
124
+ no single global version, atomic multi-shard pull or classic SSP worker-clock bound.
125
+
126
+ ## Failure and admission boundaries
127
+
128
+ Missing workers abort local synchronous barriers; membership is not silently
129
+ reduced. A worker journals complete gradient requests before sending them. After
130
+ restart it reads each shard's progress, skips completed contributions and finishes
131
+ the original pending step under its replacement generation. Fingerprints exclude
132
+ only generation metadata, so replacing a worker cannot change an already committed
133
+ gradient. Fresh bounded-mode recomputation only changes uncommitted contributions.
134
+
135
+ Shard restart reloads the last durable weights and receipts before serving traffic.
136
+ Tests actually terminate worker/shard processes and compare the completed sync run
137
+ with local SGD. State files have an OS-owned single-writer lock released on exit
138
+ or process death, preventing overlapping writers from corrupting worker journals.
139
+ Whole-run offline checkpoints preserve coordinator metadata, shard snapshots and
140
+ worker journals together. Export holds all source database leases; restore validates
141
+ every object before publishing fresh state. A partial multi-shard step retains its
142
+ original gradient bytes. Local and S3-compatible immutable stores publish manifests
143
+ last; an interrupted upload is not restorable. See [the recovery design](docs/recovery-design.md)
144
+ and [operator commands](docs/checkpoints.md). Live peers must never be mixed with
145
+ an older restored shard. Topology remapping and online coordinated cuts are unsupported.
146
+
147
+ gRPC admission is bounded to `2 * worker_count + 4` concurrent calls per shard;
148
+ excess requests receive resource exhaustion. Workers issue one gradient push at
149
+ a time. These limits do not replace deployment resource limits or provide a full
150
+ cluster-wide backpressure scheme. No unbounded training request queue is added.
151
+
152
+ ## Deployment and evidence
153
+
154
+ The native demo spawns separate processes with OS-assigned loopback ports. It
155
+ waits for readiness, starts workers, collects final parameters and compares sync
156
+ weights against concatenated-data PyTorch SGD. Its control pipes are trusted,
157
+ local parent-child IPC, never the training transport.
158
+
159
+ Compose defines separate containers with an internal network, dropped capabilities,
160
+ read-only filesystems, five persistent volumes and no public training ports. Local
161
+ container verification checks actual training, role enforcement, clean shutdown
162
+ and state-preserving recreation. A large dependency download was completed in
163
+ validated ranges and matched PyTorch's published SHA-256; the local bootstrap
164
+ mirror did not change the installed package. Normal Git-backed builds use the
165
+ default official CPU index. No Coolify, k3s, AWS or Azure deployment has been exercised.
166
+
167
+ The optional SeaweedFS storage profile has a private network and a persistent volume.
168
+ The maintenance helper receives storage credentials and can reach external endpoints;
169
+ training containers receive neither storage credentials nor a public route.
170
+ The optional operations profile adds a separate lifecycle event plane and telemetry
171
+ backends. NATS carries bounded allowlisted events, never gradient traffic. Broker
172
+ failure does not participate in registration, fencing, SGD or durable receipts.
173
+ Prometheus scrapes process/RPC metrics; OTel propagates gRPC trace context to Tempo.
174
+ The dashboard persists up to 200 validated events and mirrors them to Loki.
175
+ Its sampler uses read-only RPCs and fixed backend queries; its browser receives
176
+ neither data-plane nor administrative credentials. UI and Grafana publish loopback
177
+ ports through a management network, while internal networks keep training and
178
+ operations services private. See [operations](docs/operations.md) for definitions
179
+ and failure limits. Azure Blob checkpoints are verified against Azurite; online
180
+ checkpoint coordination and hosted validation remain future work.
@@ -0,0 +1,238 @@
1
+ # Correctness evidence and benchmark methodology
2
+
3
+ The `demo` command is a CPU correctness run, not a performance benchmark.
4
+ It runs real training in independent processes and exports a JSON report:
5
+
6
+ ```powershell
7
+ paramesh demo --mode sync --workers 2 --shards 2 --steps 20 --output artifacts/sync.json
8
+ paramesh demo --mode async --workers 2 --shards 2 --steps 20 --output artifacts/async.json
9
+ paramesh demo --mode bounded --workers 2 --shards 2 --steps 20 --output artifacts/bounded.json
10
+ ```
11
+
12
+ ## Numerical reference
13
+
14
+ ### Captured CPU evidence
15
+
16
+ These are actual 20-step runs at commit
17
+ `fca471cdfaa0e12bd8dcf212323b7013685e895d` with a clean working tree, recorded
18
+ on November 1, 2026 using Windows AMD64, Python 3.12.10 and PyTorch 2.15.0+cpu.
19
+ Each run used two workers and two nonempty shards; both workers completed.
20
+
21
+ | Mode | Initial MSE | Final MSE | Final shard versions | Max absolute error vs local SGD | Raw artifact |
22
+ | --- | --- | --- | --- | --- | --- |
23
+ | Sync | 14.232687 | 0.733824 | 20, 20 | 5.96e-8 | [JSON](docs/evidence/sync-cpu.json) |
24
+ | Async | 14.232687 | 0.538783 | 40, 40 | Not equivalent | [JSON](docs/evidence/async-cpu.json) |
25
+ | Bounded | 14.232687 | 0.508454 | 40, 40 | Not equivalent | [JSON](docs/evidence/bounded-cpu.json) |
26
+
27
+ These final losses are single-run observations, not a convergence ranking. Async
28
+ and bounded runs have twice as many shard SGD updates as sync for the same local
29
+ worker iteration count. Their artifacts were collected concurrently, so wall
30
+ times must not be used for a performance comparison. No CPU/RAM inventory or
31
+ steady-state benchmark was collected for these correctness runs.
32
+
33
+ The sync reference uses the same seeded MLP initialization, concatenates the same
34
+ 32-example worker datasets and applies PyTorch SGD for the same number of steps.
35
+ Worker mean-reduced MSE gradients are averaged by each shard. Since worker batches
36
+ are equal-sized, that matches the reference objective. Tests require maximum
37
+ absolute parameter difference below `1e-6` and loss reduction.
38
+
39
+ Async and bounded runs assert completion, expected shard update counts and loss
40
+ reduction for this workload. They are not compared numerically to sync because
41
+ their update order and model snapshots differ. This is not a general convergence
42
+ guarantee for arbitrary models or learning rates.
43
+
44
+ ## Report fields and limitations
45
+
46
+ Reports include worker/shard counts, process count, version counters, ownership,
47
+ initial/final full-dataset loss, worker-local pre-update losses, refresh counts,
48
+ reference error, Python/PyTorch/OS/architecture, timestamp and Git commit/dirty state.
49
+ The workload description records model dimensions, seeds, batch size, learning
50
+ rate, optimizer, aggregation, CPU device and thread count. Unknown Git provenance
51
+ is labeled `unknown`, such as when using a wheel outside a checkout.
52
+
53
+ `worker_wall_seconds_including_startup` begins before workers are spawned and ends
54
+ when their results arrive. It includes imports, startup, training and final heartbeat;
55
+ it excludes shard/coordinator startup, cleanup and the local reference. It is not
56
+ steady-state throughput or an RPC latency metric. No speedup, scaling efficiency,
57
+ compression gain or cloud performance is claimed. Recovery timings below use a
58
+ separate deliberately paced fault scenario.
59
+
60
+ ## Captured process recovery
61
+
62
+ Clean source `f92103a92071a339253fcad5203642151af650ad`, Windows AMD64,
63
+ Python 3.12.10/PyTorch 2.15.0+cpu, November 1, 2026. Two workers/two shards,
64
+ 20 synchronous steps, 0.1-second deliberate per-step delay. Test leases are
65
+ 1.5 seconds suspect / 3 seconds unavailable, not the default 5 / 15 seconds.
66
+
67
+ | Actual terminated process | Detection | Time from termination to later progress | Final shard versions | Raw report |
68
+ | --- | --- | --- | --- | --- |
69
+ | Worker 0 | 3.000 s heartbeat lease | 7.047 s | 20, 20 | [worker recovery](docs/evidence/worker-recovery.json) |
70
+ | Shard 0 | 0.562 s observer RPC failure | 4.828 s | 20, 20 | [shard recovery](docs/evidence/shard-recovery.json) |
71
+
72
+ Both runs finish with maximum parameter error `5.96e-8` against uninterrupted local
73
+ SGD. The worker replacement advances generation 1 to 2 and resumes from durable
74
+ step 7. These single-run timings include replacement process startup/imports;
75
+ they are not recovery SLAs or steady-state benchmarks. These particular runs did
76
+ not replay an in-flight contribution. A separate checkpoint/RPC integration test
77
+ forces partial delivery and verifies one replay without duplicate application.
78
+ Uncommitted computation lost before journaling is not instrumented.
79
+
80
+ ## Controlled uncompressed framework
81
+
82
+ ```powershell
83
+ uv sync --locked --all-extras --python 3.12
84
+ .\.venv\Scripts\Activate.ps1
85
+ paramesh benchmark --output artifacts/baseline --warmup-steps 20 --steps 200 --repetitions 3
86
+ ```
87
+
88
+ Use a fresh output directory: existing results are never overwritten. Each trial
89
+ creates separate authenticated coordinator/shard/worker processes and fresh durable
90
+ state. Workers finish warm-up, wait until every peer is ready, then begin a measured
91
+ session in the same already imported processes. The session includes channel setup,
92
+ replacement registration/fencing and final heartbeat. Its throughput is explicitly
93
+ **warm-process session throughput**, not pure steady-state compute throughput.
94
+ The original startup-inclusive demo duration remains separate.
95
+
96
+ Modes run sequentially with their order rotated between repetitions. The workload
97
+ remains the seeded CPU 4→8→1 MLP and repeated 32-example batch per worker, with one
98
+ Torch thread per process, float32 transport, durable SQLite commits and no deliberate
99
+ step delay. Bounded mode uses the existing shard-local lag limit of 1; the complete
100
+ RunConfig and warm-up session durations are preserved in each raw trial.
101
+ Trace/network exporters are disabled; bounded client RPC recording is
102
+ enabled consistently across modes. Instrumentation overhead is included and not
103
+ calibrated away. This is a small-model protocol/storage-overhead experiment.
104
+
105
+ | Artifact | Meaning |
106
+ | --- | --- |
107
+ | `trial-NNN-mode.json` | Complete actual trial, losses, versions, provenance and raw client RPC samples |
108
+ | `rpc.csv` | One observed client attempt: method/status, elapsed seconds and protobuf request/response bytes |
109
+ | `trials.csv` | Measured window, processed examples, latency percentiles, payload totals, loss and shard-commit deltas |
110
+ | `manifest.json` | Configuration, host CPU/RAM/clock, lock hash, actual trial order and completion/failure status |
111
+ | `overview.png` / `.svg` | Recorded trial dots, median and observed range; no confidence-interval claim |
112
+
113
+ Use `--hidden-width 256` to repeat the same seeded workload with a larger MLP and
114
+ measure substantially larger raw float32 gradient messages. The default width 8
115
+ preserves the original tiny baseline. Width is recorded in the manifest and every
116
+ trial; it changes the model computation and payload together, so larger-width
117
+ throughput is not a pure transport measurement. Keep correctness/reference checks
118
+ enabled and compare actual RPC payload bytes. This loopback benchmark does not model
119
+ cross-host bandwidth or establish that compression will help. Large widths can
120
+ destabilize this fixed-learning-rate workload; failed numerical checks invalidate
121
+ the trial rather than producing performance evidence.
122
+
123
+ ### External workloads
124
+
125
+ The same benchmark API accepts a `TrainingWorkload` with top-level, spawn-safe
126
+ model and batch factories. The bundled CLI remains the deterministic MLP default;
127
+ custom workloads can use the runnable example:
128
+
129
+ ```powershell
130
+ python examples/external_benchmark.py --output artifacts/external-linear
131
+ ```
132
+
133
+ This records the external model, raw RPC observations, mode-specific results,
134
+ source metadata and numerical checks in the same artifact schema. Keep the model,
135
+ dataset, optimizer and update budget identical when comparing modes; the existing
136
+ warnings about different commit semantics still apply.
137
+
138
+ Completed manifests can be shown in the authenticated dashboard Evidence page by setting
139
+ `PARAMESH_EXTERNAL_ARTIFACTS_DIR` to the artifact directory before starting the operations
140
+ Compose profile (for example, `python scripts/quickstart.py --no-build`).
141
+ Rows are labeled `external workload` and remain read-only post-run evidence; the dashboard does
142
+ not claim live monitoring of a separate external process harness.
143
+
144
+ Each worker records start/end with the same host's system-wide monotonic performance
145
+ clock. The aggregate window spans the earliest measured start through the latest
146
+ end, excluding result serialization and model evaluation. Processed examples equal
147
+ the verified measured worker iterations times batch size; they are not unique data.
148
+ Client latency includes the interceptor/application path and server waits, including
149
+ synchronous barriers and durable commits. P95 uses the successful observed calls in
150
+ each trial; failures remain in the raw CSV and payload totals. Bytes exclude HTTP/2,
151
+ TLS and NIC overhead. Captured records contain sizes/statuses, never tensor values
152
+ or authentication metadata. Capture overflow fails the benchmark result.
153
+
154
+ Equal worker iterations cause different SGD commit counts in sync versus async or
155
+ bounded mode. Compare convergence and update budgets alongside throughput; do not
156
+ turn a mode's rate into a claim of equal-quality training speedup. Three repetitions
157
+ describe observed variability and do not establish statistical significance.
158
+
159
+ The host is an ordinary workstation with background applications and the retained
160
+ operations stack running. CPU/memory/clock metadata make that context inspectable,
161
+ but do not establish resource isolation or multi-machine network performance.
162
+ Completed trial artifacts survive a later failure/interruption; a completed run's
163
+ numerical-failure diagnostics are written before validation raises. A mid-trial
164
+ termination may yield only failure status, rather than a complete raw result.
165
+ Only successful complete output may support a measurement claim. CI's small test runs verify framework behavior,
166
+ not a performance baseline. [The design contract](docs/benchmark-design.md) records
167
+ the boundaries; [Python clock semantics](https://docs.python.org/3.12/library/time.html)
168
+ and [Matplotlib export](https://matplotlib.org/stable/api/_as_gen/matplotlib.figure.Figure.savefig.html)
169
+ support the timing and chart implementation.
170
+
171
+ ### ParamMesh versus a local PyTorch reference
172
+
173
+ The scaling-matrix helper measures the same seeded workload through the existing
174
+ ParamMesh process harness and a single-process PyTorch SGD reference:
175
+
176
+ ```powershell
177
+ python examples/scaling_matrix.py `
178
+ --output artifacts/scaling-matrix `
179
+ --workers 1 2 4 `
180
+ --repetitions 3 `
181
+ --warmup-steps 20 `
182
+ --steps 200
183
+ ```
184
+
185
+ Each worker count gets raw ParamMesh and reference JSON, a combined `trials.csv`,
186
+ an environment/lock manifest, and a chart. The reference uses the same model
187
+ factory, worker datasets, optimizer, steps and CPU thread limit; the comparison
188
+ therefore describes protocol overhead and scaling behavior for that workload, not
189
+ model quality or a universal speedup. Add `8` only when the host has enough
190
+ isolated CPU capacity. Failures are preserved in the manifest and invalidate a
191
+ performance claim rather than being treated as missing samples.
192
+
193
+ The clean-source one-repetition smoke capture is preserved in
194
+ [`docs/evidence/benchmarks/scaling-matrix-local-2026-11-07/`](docs/evidence/benchmarks/scaling-matrix-local-2026-11-07/README.md).
195
+ The first three-repetition 1/2/4-worker capture is preserved in
196
+ [`docs/evidence/benchmarks/scaling-matrix-local-2026-11-07-three-repetition/`](docs/evidence/benchmarks/scaling-matrix-local-2026-11-07-three-repetition/README.md).
197
+ Both retain raw output with the same metadata and checksum rules.
198
+
199
+ ## Remaining performance work
200
+
201
+ ### Repeated local fault trials
202
+
203
+ `paramesh recovery-benchmark --output artifacts/recovery-trials --repetitions 3`
204
+ repeats the existing real 20-step synchronous worker-kill and shard-kill scenarios
205
+ in fresh processes and durable state. It alternates fault order, retains each raw
206
+ JSON run, writes a manifest with hardware/lock/provenance and `trials.csv`, and plots
207
+ individual observations with median and observed range. It refuses an existing output
208
+ directory and keeps completed reports if a later trial fails. Every recorded success
209
+ requires both shards at version 20, completed workers, a positive detection-to-progress
210
+ interval and maximum error below `1e-6` against uninterrupted local SGD.
211
+
212
+ The worker detection observation is the coordinator's 3-second unavailable lease;
213
+ the shard observation is a failed 0.5-second RPC probe. They are different detection
214
+ mechanisms, so compare each to its own behavior. The run includes a deliberate
215
+ 0.1-second per-step delay and process replacement/import time. Termination-to-progress
216
+ is not full job recovery or an SLA. The harness does not instrument uncommitted
217
+ computation lost before journaling; it cannot claim zero lost work. CI's single
218
+ repetition checks the CLI and dependencies, not a statistical baseline.
219
+
220
+ The [recorded local uncompressed baseline](docs/evidence/benchmarks/uncompressed-local-2026-11-02/README.md)
221
+ preserves nine clean-source trials, the exact dependency lock, raw RPC observations,
222
+ checksums and generated charts. Median warm-process rates were 4,119 / 4,562 / 4,999
223
+ processed examples/s for sync / async / bounded respectively. Different update
224
+ budgets prevent treating those rates as equal-quality speedups. The 196-byte model
225
+ gradient provides no evidence of a network-bandwidth bottleneck, so this dataset
226
+ does not justify adding compression.
227
+
228
+ The operations milestone exposes real protobuf bytes, RPC duration histograms,
229
+ worker steps/examples, pre-update loss and accepted staleness. Its deliberately
230
+ paced synthetic dashboards are operational evidence, not comparative performance
231
+ benchmarks. [The stack report](docs/evidence/operations-stack.json) verifies actual
232
+ backend integration at 100 steps; it records its development source identity.
233
+ Native worker/shard fault reports above remain the numerical correctness reference.
234
+
235
+ Broader model/payload sizes, resource isolation, strong/weak scaling, multi-host
236
+ network conditions and compression experiments require separate controlled runs.
237
+ Compression must be justified by measured payload/bottleneck evidence. Failure
238
+ timings require real termination and verified restoration, not simulated statuses.
paramesh-1.1.0/LICENSE ADDED
@@ -0,0 +1,201 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
@@ -0,0 +1,4 @@
1
+ recursive-include proto *.proto
2
+ recursive-include scripts *.py
3
+ recursive-include examples *.json *.py
4
+ include ARCHITECTURE.md BENCHMARKS.md SECURITY.md ROADMAP.md