paramesh 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- paramesh-1.1.0/ARCHITECTURE.md +180 -0
- paramesh-1.1.0/BENCHMARKS.md +238 -0
- paramesh-1.1.0/LICENSE +201 -0
- paramesh-1.1.0/MANIFEST.in +4 -0
- paramesh-1.1.0/PKG-INFO +574 -0
- paramesh-1.1.0/README.md +539 -0
- paramesh-1.1.0/ROADMAP.md +148 -0
- paramesh-1.1.0/SECURITY.md +111 -0
- paramesh-1.1.0/examples/compose-bounded.json +10 -0
- paramesh-1.1.0/examples/compose.json +9 -0
- paramesh-1.1.0/examples/external_benchmark.py +43 -0
- paramesh-1.1.0/examples/external_linear.py +31 -0
- paramesh-1.1.0/examples/external_model_families.py +32 -0
- paramesh-1.1.0/examples/local.json +9 -0
- paramesh-1.1.0/examples/recovery_demo.py +20 -0
- paramesh-1.1.0/examples/scaling_matrix.py +31 -0
- paramesh-1.1.0/proto/paramesh/rpc/training.proto +222 -0
- paramesh-1.1.0/pyproject.toml +74 -0
- paramesh-1.1.0/scripts/__init__.py +1 -0
- paramesh-1.1.0/scripts/create_dev_env.py +55 -0
- paramesh-1.1.0/scripts/generate_grafana.py +75 -0
- paramesh-1.1.0/scripts/generate_proto.py +39 -0
- paramesh-1.1.0/scripts/immutable_release.py +445 -0
- paramesh-1.1.0/scripts/quickstart.py +48 -0
- paramesh-1.1.0/scripts/standalone_launcher.py +13 -0
- paramesh-1.1.0/scripts/verify_compose.py +238 -0
- paramesh-1.1.0/scripts/verify_immutable_compose.py +81 -0
- paramesh-1.1.0/scripts/verify_operations.py +571 -0
- paramesh-1.1.0/scripts/windows_launcher.py +26 -0
- paramesh-1.1.0/setup.cfg +4 -0
- paramesh-1.1.0/src/paramesh/__init__.py +1 -0
- paramesh-1.1.0/src/paramesh/benchmarks.py +291 -0
- paramesh-1.1.0/src/paramesh/checkpoint.py +442 -0
- paramesh-1.1.0/src/paramesh/cli.py +388 -0
- paramesh-1.1.0/src/paramesh/config.py +35 -0
- paramesh-1.1.0/src/paramesh/coordinator.py +465 -0
- paramesh-1.1.0/src/paramesh/dashboard.py +214 -0
- paramesh-1.1.0/src/paramesh/demo.py +514 -0
- paramesh-1.1.0/src/paramesh/events.py +197 -0
- paramesh-1.1.0/src/paramesh/journal.py +79 -0
- paramesh-1.1.0/src/paramesh/local_launcher.py +85 -0
- paramesh-1.1.0/src/paramesh/monitor.py +391 -0
- paramesh-1.1.0/src/paramesh/operations.py +159 -0
- paramesh-1.1.0/src/paramesh/py.typed +0 -0
- paramesh-1.1.0/src/paramesh/recovery_benchmarks.py +151 -0
- paramesh-1.1.0/src/paramesh/rpc/__init__.py +1 -0
- paramesh-1.1.0/src/paramesh/rpc/training_pb2.py +106 -0
- paramesh-1.1.0/src/paramesh/rpc/training_pb2.pyi +370 -0
- paramesh-1.1.0/src/paramesh/rpc/training_pb2_grpc.py +771 -0
- paramesh-1.1.0/src/paramesh/scaling_benchmarks.py +280 -0
- paramesh-1.1.0/src/paramesh/security.py +158 -0
- paramesh-1.1.0/src/paramesh/services.py +210 -0
- paramesh-1.1.0/src/paramesh/shard.py +376 -0
- paramesh-1.1.0/src/paramesh/state.py +75 -0
- paramesh-1.1.0/src/paramesh/storage.py +197 -0
- paramesh-1.1.0/src/paramesh/telemetry.py +357 -0
- paramesh-1.1.0/src/paramesh/tensors.py +63 -0
- paramesh-1.1.0/src/paramesh/web/app.js +312 -0
- paramesh-1.1.0/src/paramesh/web/favicon.svg +1 -0
- paramesh-1.1.0/src/paramesh/web/index.html +65 -0
- paramesh-1.1.0/src/paramesh/web/style.css +160 -0
- paramesh-1.1.0/src/paramesh/worker.py +356 -0
- paramesh-1.1.0/src/paramesh/workload.py +166 -0
- paramesh-1.1.0/src/paramesh.egg-info/PKG-INFO +574 -0
- paramesh-1.1.0/src/paramesh.egg-info/SOURCES.txt +97 -0
- paramesh-1.1.0/src/paramesh.egg-info/dependency_links.txt +1 -0
- paramesh-1.1.0/src/paramesh.egg-info/entry_points.txt +3 -0
- paramesh-1.1.0/src/paramesh.egg-info/requires.txt +30 -0
- paramesh-1.1.0/src/paramesh.egg-info/top_level.txt +1 -0
- paramesh-1.1.0/tests/test_azure.py +253 -0
- paramesh-1.1.0/tests/test_benchmark_evidence.py +80 -0
- paramesh-1.1.0/tests/test_benchmarks.py +250 -0
- paramesh-1.1.0/tests/test_checkpoint.py +336 -0
- paramesh-1.1.0/tests/test_checkpoint_epochs.py +1012 -0
- paramesh-1.1.0/tests/test_coordinator.py +85 -0
- paramesh-1.1.0/tests/test_dashboard.py +72 -0
- paramesh-1.1.0/tests/test_environment_script.py +53 -0
- paramesh-1.1.0/tests/test_events.py +178 -0
- paramesh-1.1.0/tests/test_failures.py +102 -0
- paramesh-1.1.0/tests/test_fencing.py +61 -0
- paramesh-1.1.0/tests/test_immutable_release.py +103 -0
- paramesh-1.1.0/tests/test_journal.py +36 -0
- paramesh-1.1.0/tests/test_model_families.py +23 -0
- paramesh-1.1.0/tests/test_monitor.py +218 -0
- paramesh-1.1.0/tests/test_operations.py +101 -0
- paramesh-1.1.0/tests/test_quickstart.py +53 -0
- paramesh-1.1.0/tests/test_recovery_processes.py +21 -0
- paramesh-1.1.0/tests/test_rpc.py +62 -0
- paramesh-1.1.0/tests/test_s3.py +99 -0
- paramesh-1.1.0/tests/test_scaling_benchmarks.py +87 -0
- paramesh-1.1.0/tests/test_security.py +139 -0
- paramesh-1.1.0/tests/test_shard.py +115 -0
- paramesh-1.1.0/tests/test_shard_persistence.py +51 -0
- paramesh-1.1.0/tests/test_standalone_launcher.py +21 -0
- paramesh-1.1.0/tests/test_state.py +47 -0
- paramesh-1.1.0/tests/test_telemetry.py +183 -0
- paramesh-1.1.0/tests/test_tensors.py +58 -0
- paramesh-1.1.0/tests/test_training.py +86 -0
- paramesh-1.1.0/tests/test_worker.py +217 -0
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
## Control plane
|
|
4
|
+
|
|
5
|
+
One coordinator serves one immutable run configuration. Configuration includes
|
|
6
|
+
the run ID, fixed worker membership, ordered shard endpoints, synchronization mode,
|
|
7
|
+
learning rate, aggregation rule and barrier timeout. Run creation is currently a
|
|
8
|
+
local JSON configuration operation, not a mutable remote API.
|
|
9
|
+
|
|
10
|
+
Workers register using protocol version 1 and an allowed worker ID. A random
|
|
11
|
+
session token identifies a control-plane lease. A live identity cannot be claimed
|
|
12
|
+
twice. The coordinator uses a monotonic clock: a node is initially `registered`,
|
|
13
|
+
then `healthy` after a heartbeat, `suspect` after 5 seconds and `unavailable` after
|
|
14
|
+
15 seconds. Thresholds can be set when constructing the coordinator. A completed
|
|
15
|
+
worker remains `completed`, rather than appearing as a failed node later.
|
|
16
|
+
|
|
17
|
+
Expired workers may register a fresh control-plane session; old tokens cannot
|
|
18
|
+
send heartbeats. The coordinator durably reserves a monotonically increasing
|
|
19
|
+
generation, fences it on every shard, then issues the new session. An interrupted
|
|
20
|
+
partial fence never reuses a generation. Shard updates reject superseded generations.
|
|
21
|
+
|
|
22
|
+
Run configuration, generation counters and progress survive in checksummed SQLite
|
|
23
|
+
state. Restart invalidates prior leases and requires re-registration; it does not
|
|
24
|
+
pretend saved monotonic-clock timestamps are live leases. `global_step` is a
|
|
25
|
+
reported completed-step lower watermark across configured workers, not a global
|
|
26
|
+
atomic model version. Shard endpoints remain configured; dynamic registration
|
|
27
|
+
and coordinator replication are not implemented.
|
|
28
|
+
|
|
29
|
+
## Data plane and ownership
|
|
30
|
+
|
|
31
|
+
Workers talk directly to shards over gRPC. Parameter ownership is:
|
|
32
|
+
|
|
33
|
+
```text
|
|
34
|
+
int.from_bytes(sha256(parameter_name.encode()).digest()[:8], "big") % shard_count
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Shard index is the endpoint's position in the immutable configuration. Python's
|
|
38
|
+
randomized built-in hash is never used. Renaming a parameter or changing the
|
|
39
|
+
shard count can move ownership; live resharding is unsupported. Whole tensors
|
|
40
|
+
are assigned to shards, so load can be uneven and shards can be empty for very
|
|
41
|
+
small models. The two-shard MLP test asserts both shards own parameters.
|
|
42
|
+
|
|
43
|
+
The worker model performs forward and backward passes with PyTorch. Each shard
|
|
44
|
+
stores float32 NumPy weights and applies SGD itself. Plain SGD has no momentum
|
|
45
|
+
or Adam state; weights, version and replay receipts are authoritative. Supporting
|
|
46
|
+
a stateful optimizer will require including its state in checkpoints.
|
|
47
|
+
|
|
48
|
+
## Protocol and validation
|
|
49
|
+
|
|
50
|
+
Source: `proto/paramesh/rpc/training.proto`; package: `paramesh.v1`. Python bindings
|
|
51
|
+
and message `.pyi` files are generated and checked into source control. CI compares
|
|
52
|
+
fresh generation with committed bytes. New compatible fields can be added without
|
|
53
|
+
reusing existing field numbers. The wire version is early and not a stable public
|
|
54
|
+
API commitment yet.
|
|
55
|
+
|
|
56
|
+
Tensors carry a name, shape, explicit `float32-le` dtype and little-endian bytes.
|
|
57
|
+
The decoder allows at most eight dimensions, 4 MiB per tensor, 1,024 tensors and
|
|
58
|
+
8 MiB of tensor data per collection. RPC messages themselves are capped at 8 MiB,
|
|
59
|
+
including protobuf overhead. Scalars are allowed; zero-sized tensors are not.
|
|
60
|
+
Neither network endpoints nor model loading accept arbitrary Python object files.
|
|
61
|
+
|
|
62
|
+
Each update validates run identity, configured worker membership, exact parameter
|
|
63
|
+
names and shapes, dtype, payload length, finite values, update sequence and base
|
|
64
|
+
version. Decoding happens before entering the optimizer critical section. A
|
|
65
|
+
condition lock serializes commits. Proposed new weights are computed separately
|
|
66
|
+
and checked for overflow before replacing any authoritative weight.
|
|
67
|
+
|
|
68
|
+
## Versions, sequence numbers and retries
|
|
69
|
+
|
|
70
|
+
Each shard has its own monotonically increasing version, starting at zero.
|
|
71
|
+
Every accepted optimizer step increments it once, even for an empty shard. A
|
|
72
|
+
worker's update sequence starts at zero and increases by one per local iteration.
|
|
73
|
+
These are distinct counters: async shard versions count every worker's updates.
|
|
74
|
+
|
|
75
|
+
The shard remembers the latest accepted sequence, SHA-256 request fingerprint,
|
|
76
|
+
resulting version and staleness for each configured worker. An identical retry
|
|
77
|
+
returns that receipt without applying SGD again. A changed retry or sequence gap
|
|
78
|
+
is rejected. Memory is bounded by membership and one in-flight barrier. Older
|
|
79
|
+
receipts are not retained. Weights, SGD configuration, versions, receipts and fences
|
|
80
|
+
are committed together to SQLite WAL with `synchronous=FULL` before acknowledgement.
|
|
81
|
+
This supports retry after a process restart when the volume survives; it is not
|
|
82
|
+
an exactly-once delivery claim after storage loss.
|
|
83
|
+
|
|
84
|
+
Workers use explicit RPC deadlines and retry transient unavailable/deadline errors
|
|
85
|
+
at most three times with short backoff. They reuse the original request bytes;
|
|
86
|
+
they do not recompute a gradient after an ambiguous transport failure. Calls use
|
|
87
|
+
wait-for-ready within their deadlines so reconnection can outlast a transient
|
|
88
|
+
failure rather than exhausting retries immediately. A test
|
|
89
|
+
simulates losing the response after commit and verifies one update, not two.
|
|
90
|
+
|
|
91
|
+
## Synchronization
|
|
92
|
+
|
|
93
|
+
### Synchronous
|
|
94
|
+
|
|
95
|
+
Every worker submits one gradient per version. The shard accumulates a fixed
|
|
96
|
+
membership barrier, computes mean or sum and commits one SGD update. Float64
|
|
97
|
+
accumulation reduces summation overflow before conversion to float32 weights.
|
|
98
|
+
Mean gives equal weight to each worker, which is equivalent to the demo's local
|
|
99
|
+
reference because each worker has the same number of examples.
|
|
100
|
+
|
|
101
|
+
A shared deadline takes the earliest participant deadline, capped by the configured
|
|
102
|
+
barrier timeout. Late final arrivals cannot commit merely because an expired
|
|
103
|
+
waiting thread has not resumed yet. A timeout clears that shard's pending contributions and wakes waiting calls.
|
|
104
|
+
Membership is not silently reduced. The server's RPC thread limit exceeds the
|
|
105
|
+
configured worker count, preventing waiting participants from starving the final
|
|
106
|
+
arrival. No distributed commit ties separate shard barriers together.
|
|
107
|
+
|
|
108
|
+
### Asynchronous
|
|
109
|
+
|
|
110
|
+
Each valid gradient commits immediately. Staleness is `current_shard_version -
|
|
111
|
+
base_version` and is returned to the worker. Ordering depends on process and
|
|
112
|
+
network scheduling; async training is not expected to match synchronous SGD.
|
|
113
|
+
|
|
114
|
+
### Bounded lag
|
|
115
|
+
|
|
116
|
+
An update whose shard-local staleness exceeds `max_staleness` receives
|
|
117
|
+
`FAILED_PRECONDITION`. The worker pulls fresh weights and recomputes gradients
|
|
118
|
+
for contributions that have not yet committed in its current iteration. Already
|
|
119
|
+
accepted shards are not submitted again with changed contents. Refreshes are
|
|
120
|
+
bounded to 100 attempts; exhaustion fails the worker visibly.
|
|
121
|
+
|
|
122
|
+
Consequently a worker iteration can combine contributions computed from different
|
|
123
|
+
model snapshots. The bound is enforced at each shard's acceptance point. There is
|
|
124
|
+
no single global version, atomic multi-shard pull or classic SSP worker-clock bound.
|
|
125
|
+
|
|
126
|
+
## Failure and admission boundaries
|
|
127
|
+
|
|
128
|
+
Missing workers abort local synchronous barriers; membership is not silently
|
|
129
|
+
reduced. A worker journals complete gradient requests before sending them. After
|
|
130
|
+
restart it reads each shard's progress, skips completed contributions and finishes
|
|
131
|
+
the original pending step under its replacement generation. Fingerprints exclude
|
|
132
|
+
only generation metadata, so replacing a worker cannot change an already committed
|
|
133
|
+
gradient. Fresh bounded-mode recomputation only changes uncommitted contributions.
|
|
134
|
+
|
|
135
|
+
Shard restart reloads the last durable weights and receipts before serving traffic.
|
|
136
|
+
Tests actually terminate worker/shard processes and compare the completed sync run
|
|
137
|
+
with local SGD. State files have an OS-owned single-writer lock released on exit
|
|
138
|
+
or process death, preventing overlapping writers from corrupting worker journals.
|
|
139
|
+
Whole-run offline checkpoints preserve coordinator metadata, shard snapshots and
|
|
140
|
+
worker journals together. Export holds all source database leases; restore validates
|
|
141
|
+
every object before publishing fresh state. A partial multi-shard step retains its
|
|
142
|
+
original gradient bytes. Local and S3-compatible immutable stores publish manifests
|
|
143
|
+
last; an interrupted upload is not restorable. See [the recovery design](docs/recovery-design.md)
|
|
144
|
+
and [operator commands](docs/checkpoints.md). Live peers must never be mixed with
|
|
145
|
+
an older restored shard. Topology remapping and online coordinated cuts are unsupported.
|
|
146
|
+
|
|
147
|
+
gRPC admission is bounded to `2 * worker_count + 4` concurrent calls per shard;
|
|
148
|
+
excess requests receive resource exhaustion. Workers issue one gradient push at
|
|
149
|
+
a time. These limits do not replace deployment resource limits or provide a full
|
|
150
|
+
cluster-wide backpressure scheme. No unbounded training request queue is added.
|
|
151
|
+
|
|
152
|
+
## Deployment and evidence
|
|
153
|
+
|
|
154
|
+
The native demo spawns separate processes with OS-assigned loopback ports. It
|
|
155
|
+
waits for readiness, starts workers, collects final parameters and compares sync
|
|
156
|
+
weights against concatenated-data PyTorch SGD. Its control pipes are trusted,
|
|
157
|
+
local parent-child IPC, never the training transport.
|
|
158
|
+
|
|
159
|
+
Compose defines separate containers with an internal network, dropped capabilities,
|
|
160
|
+
read-only filesystems, five persistent volumes and no public training ports. Local
|
|
161
|
+
container verification checks actual training, role enforcement, clean shutdown
|
|
162
|
+
and state-preserving recreation. A large dependency download was completed in
|
|
163
|
+
validated ranges and matched PyTorch's published SHA-256; the local bootstrap
|
|
164
|
+
mirror did not change the installed package. Normal Git-backed builds use the
|
|
165
|
+
default official CPU index. No Coolify, k3s, AWS or Azure deployment has been exercised.
|
|
166
|
+
|
|
167
|
+
The optional SeaweedFS storage profile has a private network and a persistent volume.
|
|
168
|
+
The maintenance helper receives storage credentials and can reach external endpoints;
|
|
169
|
+
training containers receive neither storage credentials nor a public route.
|
|
170
|
+
The optional operations profile adds a separate lifecycle event plane and telemetry
|
|
171
|
+
backends. NATS carries bounded allowlisted events, never gradient traffic. Broker
|
|
172
|
+
failure does not participate in registration, fencing, SGD or durable receipts.
|
|
173
|
+
Prometheus scrapes process/RPC metrics; OTel propagates gRPC trace context to Tempo.
|
|
174
|
+
The dashboard persists up to 200 validated events and mirrors them to Loki.
|
|
175
|
+
Its sampler uses read-only RPCs and fixed backend queries; its browser receives
|
|
176
|
+
neither data-plane nor administrative credentials. UI and Grafana publish loopback
|
|
177
|
+
ports through a management network, while internal networks keep training and
|
|
178
|
+
operations services private. See [operations](docs/operations.md) for definitions
|
|
179
|
+
and failure limits. Azure Blob checkpoints are verified against Azurite; online
|
|
180
|
+
checkpoint coordination and hosted validation remain future work.
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
# Correctness evidence and benchmark methodology
|
|
2
|
+
|
|
3
|
+
The `demo` command is a CPU correctness run, not a performance benchmark.
|
|
4
|
+
It runs real training in independent processes and exports a JSON report:
|
|
5
|
+
|
|
6
|
+
```powershell
|
|
7
|
+
paramesh demo --mode sync --workers 2 --shards 2 --steps 20 --output artifacts/sync.json
|
|
8
|
+
paramesh demo --mode async --workers 2 --shards 2 --steps 20 --output artifacts/async.json
|
|
9
|
+
paramesh demo --mode bounded --workers 2 --shards 2 --steps 20 --output artifacts/bounded.json
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Numerical reference
|
|
13
|
+
|
|
14
|
+
### Captured CPU evidence
|
|
15
|
+
|
|
16
|
+
These are actual 20-step runs at commit
|
|
17
|
+
`fca471cdfaa0e12bd8dcf212323b7013685e895d` with a clean working tree, recorded
|
|
18
|
+
on November 1, 2026 using Windows AMD64, Python 3.12.10 and PyTorch 2.15.0+cpu.
|
|
19
|
+
Each run used two workers and two nonempty shards; both workers completed.
|
|
20
|
+
|
|
21
|
+
| Mode | Initial MSE | Final MSE | Final shard versions | Max absolute error vs local SGD | Raw artifact |
|
|
22
|
+
| --- | --- | --- | --- | --- | --- |
|
|
23
|
+
| Sync | 14.232687 | 0.733824 | 20, 20 | 5.96e-8 | [JSON](docs/evidence/sync-cpu.json) |
|
|
24
|
+
| Async | 14.232687 | 0.538783 | 40, 40 | Not equivalent | [JSON](docs/evidence/async-cpu.json) |
|
|
25
|
+
| Bounded | 14.232687 | 0.508454 | 40, 40 | Not equivalent | [JSON](docs/evidence/bounded-cpu.json) |
|
|
26
|
+
|
|
27
|
+
These final losses are single-run observations, not a convergence ranking. Async
|
|
28
|
+
and bounded runs have twice as many shard SGD updates as sync for the same local
|
|
29
|
+
worker iteration count. Their artifacts were collected concurrently, so wall
|
|
30
|
+
times must not be used for a performance comparison. No CPU/RAM inventory or
|
|
31
|
+
steady-state benchmark was collected for these correctness runs.
|
|
32
|
+
|
|
33
|
+
The sync reference uses the same seeded MLP initialization, concatenates the same
|
|
34
|
+
32-example worker datasets and applies PyTorch SGD for the same number of steps.
|
|
35
|
+
Worker mean-reduced MSE gradients are averaged by each shard. Since worker batches
|
|
36
|
+
are equal-sized, that matches the reference objective. Tests require maximum
|
|
37
|
+
absolute parameter difference below `1e-6` and loss reduction.
|
|
38
|
+
|
|
39
|
+
Async and bounded runs assert completion, expected shard update counts and loss
|
|
40
|
+
reduction for this workload. They are not compared numerically to sync because
|
|
41
|
+
their update order and model snapshots differ. This is not a general convergence
|
|
42
|
+
guarantee for arbitrary models or learning rates.
|
|
43
|
+
|
|
44
|
+
## Report fields and limitations
|
|
45
|
+
|
|
46
|
+
Reports include worker/shard counts, process count, version counters, ownership,
|
|
47
|
+
initial/final full-dataset loss, worker-local pre-update losses, refresh counts,
|
|
48
|
+
reference error, Python/PyTorch/OS/architecture, timestamp and Git commit/dirty state.
|
|
49
|
+
The workload description records model dimensions, seeds, batch size, learning
|
|
50
|
+
rate, optimizer, aggregation, CPU device and thread count. Unknown Git provenance
|
|
51
|
+
is labeled `unknown`, such as when using a wheel outside a checkout.
|
|
52
|
+
|
|
53
|
+
`worker_wall_seconds_including_startup` begins before workers are spawned and ends
|
|
54
|
+
when their results arrive. It includes imports, startup, training and final heartbeat;
|
|
55
|
+
it excludes shard/coordinator startup, cleanup and the local reference. It is not
|
|
56
|
+
steady-state throughput or an RPC latency metric. No speedup, scaling efficiency,
|
|
57
|
+
compression gain or cloud performance is claimed. Recovery timings below use a
|
|
58
|
+
separate deliberately paced fault scenario.
|
|
59
|
+
|
|
60
|
+
## Captured process recovery
|
|
61
|
+
|
|
62
|
+
Clean source `f92103a92071a339253fcad5203642151af650ad`, Windows AMD64,
|
|
63
|
+
Python 3.12.10/PyTorch 2.15.0+cpu, November 1, 2026. Two workers/two shards,
|
|
64
|
+
20 synchronous steps, 0.1-second deliberate per-step delay. Test leases are
|
|
65
|
+
1.5 seconds suspect / 3 seconds unavailable, not the default 5 / 15 seconds.
|
|
66
|
+
|
|
67
|
+
| Actual terminated process | Detection | Time from termination to later progress | Final shard versions | Raw report |
|
|
68
|
+
| --- | --- | --- | --- | --- |
|
|
69
|
+
| Worker 0 | 3.000 s heartbeat lease | 7.047 s | 20, 20 | [worker recovery](docs/evidence/worker-recovery.json) |
|
|
70
|
+
| Shard 0 | 0.562 s observer RPC failure | 4.828 s | 20, 20 | [shard recovery](docs/evidence/shard-recovery.json) |
|
|
71
|
+
|
|
72
|
+
Both runs finish with maximum parameter error `5.96e-8` against uninterrupted local
|
|
73
|
+
SGD. The worker replacement advances generation 1 to 2 and resumes from durable
|
|
74
|
+
step 7. These single-run timings include replacement process startup/imports;
|
|
75
|
+
they are not recovery SLAs or steady-state benchmarks. These particular runs did
|
|
76
|
+
not replay an in-flight contribution. A separate checkpoint/RPC integration test
|
|
77
|
+
forces partial delivery and verifies one replay without duplicate application.
|
|
78
|
+
Uncommitted computation lost before journaling is not instrumented.
|
|
79
|
+
|
|
80
|
+
## Controlled uncompressed framework
|
|
81
|
+
|
|
82
|
+
```powershell
|
|
83
|
+
uv sync --locked --all-extras --python 3.12
|
|
84
|
+
.\.venv\Scripts\Activate.ps1
|
|
85
|
+
paramesh benchmark --output artifacts/baseline --warmup-steps 20 --steps 200 --repetitions 3
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Use a fresh output directory: existing results are never overwritten. Each trial
|
|
89
|
+
creates separate authenticated coordinator/shard/worker processes and fresh durable
|
|
90
|
+
state. Workers finish warm-up, wait until every peer is ready, then begin a measured
|
|
91
|
+
session in the same already imported processes. The session includes channel setup,
|
|
92
|
+
replacement registration/fencing and final heartbeat. Its throughput is explicitly
|
|
93
|
+
**warm-process session throughput**, not pure steady-state compute throughput.
|
|
94
|
+
The original startup-inclusive demo duration remains separate.
|
|
95
|
+
|
|
96
|
+
Modes run sequentially with their order rotated between repetitions. The workload
|
|
97
|
+
remains the seeded CPU 4→8→1 MLP and repeated 32-example batch per worker, with one
|
|
98
|
+
Torch thread per process, float32 transport, durable SQLite commits and no deliberate
|
|
99
|
+
step delay. Bounded mode uses the existing shard-local lag limit of 1; the complete
|
|
100
|
+
RunConfig and warm-up session durations are preserved in each raw trial.
|
|
101
|
+
Trace/network exporters are disabled; bounded client RPC recording is
|
|
102
|
+
enabled consistently across modes. Instrumentation overhead is included and not
|
|
103
|
+
calibrated away. This is a small-model protocol/storage-overhead experiment.
|
|
104
|
+
|
|
105
|
+
| Artifact | Meaning |
|
|
106
|
+
| --- | --- |
|
|
107
|
+
| `trial-NNN-mode.json` | Complete actual trial, losses, versions, provenance and raw client RPC samples |
|
|
108
|
+
| `rpc.csv` | One observed client attempt: method/status, elapsed seconds and protobuf request/response bytes |
|
|
109
|
+
| `trials.csv` | Measured window, processed examples, latency percentiles, payload totals, loss and shard-commit deltas |
|
|
110
|
+
| `manifest.json` | Configuration, host CPU/RAM/clock, lock hash, actual trial order and completion/failure status |
|
|
111
|
+
| `overview.png` / `.svg` | Recorded trial dots, median and observed range; no confidence-interval claim |
|
|
112
|
+
|
|
113
|
+
Use `--hidden-width 256` to repeat the same seeded workload with a larger MLP and
|
|
114
|
+
measure substantially larger raw float32 gradient messages. The default width 8
|
|
115
|
+
preserves the original tiny baseline. Width is recorded in the manifest and every
|
|
116
|
+
trial; it changes the model computation and payload together, so larger-width
|
|
117
|
+
throughput is not a pure transport measurement. Keep correctness/reference checks
|
|
118
|
+
enabled and compare actual RPC payload bytes. This loopback benchmark does not model
|
|
119
|
+
cross-host bandwidth or establish that compression will help. Large widths can
|
|
120
|
+
destabilize this fixed-learning-rate workload; failed numerical checks invalidate
|
|
121
|
+
the trial rather than producing performance evidence.
|
|
122
|
+
|
|
123
|
+
### External workloads
|
|
124
|
+
|
|
125
|
+
The same benchmark API accepts a `TrainingWorkload` with top-level, spawn-safe
|
|
126
|
+
model and batch factories. The bundled CLI remains the deterministic MLP default;
|
|
127
|
+
custom workloads can use the runnable example:
|
|
128
|
+
|
|
129
|
+
```powershell
|
|
130
|
+
python examples/external_benchmark.py --output artifacts/external-linear
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
This records the external model, raw RPC observations, mode-specific results,
|
|
134
|
+
source metadata and numerical checks in the same artifact schema. Keep the model,
|
|
135
|
+
dataset, optimizer and update budget identical when comparing modes; the existing
|
|
136
|
+
warnings about different commit semantics still apply.
|
|
137
|
+
|
|
138
|
+
Completed manifests can be shown in the authenticated dashboard Evidence page by setting
|
|
139
|
+
`PARAMESH_EXTERNAL_ARTIFACTS_DIR` to the artifact directory before starting the operations
|
|
140
|
+
Compose profile (for example, `python scripts/quickstart.py --no-build`).
|
|
141
|
+
Rows are labeled `external workload` and remain read-only post-run evidence; the dashboard does
|
|
142
|
+
not claim live monitoring of a separate external process harness.
|
|
143
|
+
|
|
144
|
+
Each worker records start/end with the same host's system-wide monotonic performance
|
|
145
|
+
clock. The aggregate window spans the earliest measured start through the latest
|
|
146
|
+
end, excluding result serialization and model evaluation. Processed examples equal
|
|
147
|
+
the verified measured worker iterations times batch size; they are not unique data.
|
|
148
|
+
Client latency includes the interceptor/application path and server waits, including
|
|
149
|
+
synchronous barriers and durable commits. P95 uses the successful observed calls in
|
|
150
|
+
each trial; failures remain in the raw CSV and payload totals. Bytes exclude HTTP/2,
|
|
151
|
+
TLS and NIC overhead. Captured records contain sizes/statuses, never tensor values
|
|
152
|
+
or authentication metadata. Capture overflow fails the benchmark result.
|
|
153
|
+
|
|
154
|
+
Equal worker iterations cause different SGD commit counts in sync versus async or
|
|
155
|
+
bounded mode. Compare convergence and update budgets alongside throughput; do not
|
|
156
|
+
turn a mode's rate into a claim of equal-quality training speedup. Three repetitions
|
|
157
|
+
describe observed variability and do not establish statistical significance.
|
|
158
|
+
|
|
159
|
+
The host is an ordinary workstation with background applications and the retained
|
|
160
|
+
operations stack running. CPU/memory/clock metadata make that context inspectable,
|
|
161
|
+
but do not establish resource isolation or multi-machine network performance.
|
|
162
|
+
Completed trial artifacts survive a later failure/interruption; a completed run's
|
|
163
|
+
numerical-failure diagnostics are written before validation raises. A mid-trial
|
|
164
|
+
termination may yield only failure status, rather than a complete raw result.
|
|
165
|
+
Only successful complete output may support a measurement claim. CI's small test runs verify framework behavior,
|
|
166
|
+
not a performance baseline. [The design contract](docs/benchmark-design.md) records
|
|
167
|
+
the boundaries; [Python clock semantics](https://docs.python.org/3.12/library/time.html)
|
|
168
|
+
and [Matplotlib export](https://matplotlib.org/stable/api/_as_gen/matplotlib.figure.Figure.savefig.html)
|
|
169
|
+
support the timing and chart implementation.
|
|
170
|
+
|
|
171
|
+
### ParamMesh versus a local PyTorch reference
|
|
172
|
+
|
|
173
|
+
The scaling-matrix helper measures the same seeded workload through the existing
|
|
174
|
+
ParamMesh process harness and a single-process PyTorch SGD reference:
|
|
175
|
+
|
|
176
|
+
```powershell
|
|
177
|
+
python examples/scaling_matrix.py `
|
|
178
|
+
--output artifacts/scaling-matrix `
|
|
179
|
+
--workers 1 2 4 `
|
|
180
|
+
--repetitions 3 `
|
|
181
|
+
--warmup-steps 20 `
|
|
182
|
+
--steps 200
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
Each worker count gets raw ParamMesh and reference JSON, a combined `trials.csv`,
|
|
186
|
+
an environment/lock manifest, and a chart. The reference uses the same model
|
|
187
|
+
factory, worker datasets, optimizer, steps and CPU thread limit; the comparison
|
|
188
|
+
therefore describes protocol overhead and scaling behavior for that workload, not
|
|
189
|
+
model quality or a universal speedup. Add `8` only when the host has enough
|
|
190
|
+
isolated CPU capacity. Failures are preserved in the manifest and invalidate a
|
|
191
|
+
performance claim rather than being treated as missing samples.
|
|
192
|
+
|
|
193
|
+
The clean-source one-repetition smoke capture is preserved in
|
|
194
|
+
[`docs/evidence/benchmarks/scaling-matrix-local-2026-11-07/`](docs/evidence/benchmarks/scaling-matrix-local-2026-11-07/README.md).
|
|
195
|
+
The first three-repetition 1/2/4-worker capture is preserved in
|
|
196
|
+
[`docs/evidence/benchmarks/scaling-matrix-local-2026-11-07-three-repetition/`](docs/evidence/benchmarks/scaling-matrix-local-2026-11-07-three-repetition/README.md).
|
|
197
|
+
Both retain raw output with the same metadata and checksum rules.
|
|
198
|
+
|
|
199
|
+
## Remaining performance work
|
|
200
|
+
|
|
201
|
+
### Repeated local fault trials
|
|
202
|
+
|
|
203
|
+
`paramesh recovery-benchmark --output artifacts/recovery-trials --repetitions 3`
|
|
204
|
+
repeats the existing real 20-step synchronous worker-kill and shard-kill scenarios
|
|
205
|
+
in fresh processes and durable state. It alternates fault order, retains each raw
|
|
206
|
+
JSON run, writes a manifest with hardware/lock/provenance and `trials.csv`, and plots
|
|
207
|
+
individual observations with median and observed range. It refuses an existing output
|
|
208
|
+
directory and keeps completed reports if a later trial fails. Every recorded success
|
|
209
|
+
requires both shards at version 20, completed workers, a positive detection-to-progress
|
|
210
|
+
interval and maximum error below `1e-6` against uninterrupted local SGD.
|
|
211
|
+
|
|
212
|
+
The worker detection observation is the coordinator's 3-second unavailable lease;
|
|
213
|
+
the shard observation is a failed 0.5-second RPC probe. They are different detection
|
|
214
|
+
mechanisms, so compare each to its own behavior. The run includes a deliberate
|
|
215
|
+
0.1-second per-step delay and process replacement/import time. Termination-to-progress
|
|
216
|
+
is not full job recovery or an SLA. The harness does not instrument uncommitted
|
|
217
|
+
computation lost before journaling; it cannot claim zero lost work. CI's single
|
|
218
|
+
repetition checks the CLI and dependencies, not a statistical baseline.
|
|
219
|
+
|
|
220
|
+
The [recorded local uncompressed baseline](docs/evidence/benchmarks/uncompressed-local-2026-11-02/README.md)
|
|
221
|
+
preserves nine clean-source trials, the exact dependency lock, raw RPC observations,
|
|
222
|
+
checksums and generated charts. Median warm-process rates were 4,119 / 4,562 / 4,999
|
|
223
|
+
processed examples/s for sync / async / bounded respectively. Different update
|
|
224
|
+
budgets prevent treating those rates as equal-quality speedups. The 196-byte model
|
|
225
|
+
gradient provides no evidence of a network-bandwidth bottleneck, so this dataset
|
|
226
|
+
does not justify adding compression.
|
|
227
|
+
|
|
228
|
+
The operations milestone exposes real protobuf bytes, RPC duration histograms,
|
|
229
|
+
worker steps/examples, pre-update loss and accepted staleness. Its deliberately
|
|
230
|
+
paced synthetic dashboards are operational evidence, not comparative performance
|
|
231
|
+
benchmarks. [The stack report](docs/evidence/operations-stack.json) verifies actual
|
|
232
|
+
backend integration at 100 steps; it records its development source identity.
|
|
233
|
+
Native worker/shard fault reports above remain the numerical correctness reference.
|
|
234
|
+
|
|
235
|
+
Broader model/payload sizes, resource isolation, strong/weak scaling, multi-host
|
|
236
|
+
network conditions and compression experiments require separate controlled runs.
|
|
237
|
+
Compression must be justified by measured payload/bottleneck evidence. Failure
|
|
238
|
+
timings require real termination and verified restoration, not simulated statuses.
|
paramesh-1.1.0/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|