optical-circuit-intent 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- optical_circuit_intent-1.0.1/LICENSE +21 -0
- optical_circuit_intent-1.0.1/PKG-INFO +241 -0
- optical_circuit_intent-1.0.1/README.md +219 -0
- optical_circuit_intent-1.0.1/pyproject.toml +46 -0
- optical_circuit_intent-1.0.1/setup.cfg +4 -0
- optical_circuit_intent-1.0.1/src/ocintent/__init__.py +108 -0
- optical_circuit_intent-1.0.1/src/ocintent/checkpoint.py +350 -0
- optical_circuit_intent-1.0.1/src/ocintent/cli.py +408 -0
- optical_circuit_intent-1.0.1/src/ocintent/drift.py +332 -0
- optical_circuit_intent-1.0.1/src/ocintent/intent.py +294 -0
- optical_circuit_intent-1.0.1/src/ocintent/ledger.py +338 -0
- optical_circuit_intent-1.0.1/src/ocintent/legality.py +381 -0
- optical_circuit_intent-1.0.1/src/ocintent/radix.py +281 -0
- optical_circuit_intent-1.0.1/src/optical_circuit_intent.egg-info/PKG-INFO +241 -0
- optical_circuit_intent-1.0.1/src/optical_circuit_intent.egg-info/SOURCES.txt +19 -0
- optical_circuit_intent-1.0.1/src/optical_circuit_intent.egg-info/dependency_links.txt +1 -0
- optical_circuit_intent-1.0.1/src/optical_circuit_intent.egg-info/entry_points.txt +2 -0
- optical_circuit_intent-1.0.1/src/optical_circuit_intent.egg-info/requires.txt +3 -0
- optical_circuit_intent-1.0.1/src/optical_circuit_intent.egg-info/top_level.txt +1 -0
- optical_circuit_intent-1.0.1/tests/test_mutations.py +291 -0
- optical_circuit_intent-1.0.1/tests/test_ocintent.py +826 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Margaret Nanyonga
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: optical-circuit-intent
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: What to ask an optical plant for, when it is legal to ask, and what the answer costs
|
|
5
|
+
Author: Margaret Nanyonga
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/dimaggi-ai/optical-circuit-intent
|
|
8
|
+
Project-URL: Research, https://dimaggi-ai.github.io/research
|
|
9
|
+
Keywords: optical circuit switching,distributed training,capacity planning,data center interconnect,checkpointing
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Intended Audience :: System Administrators
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering
|
|
15
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest==8.3.4; extra == "dev"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# optical-circuit-intent
|
|
24
|
+
|
|
25
|
+
**What to ask an optical plant for, when it is legal to ask, and what the answer
|
|
26
|
+
costs.**
|
|
27
|
+
|
|
28
|
+
A training job that crosses a data hall boundary sits on a circuit somebody has
|
|
29
|
+
to provision, retune, and eventually admit is not what the inventory says it is.
|
|
30
|
+
This repository models six parts of that problem, in the unit the rest of the
|
|
31
|
+
series uses — accelerator-hours — so the numbers can be added up.
|
|
32
|
+
|
|
33
|
+
It is a companion to [span-contract](https://github.com/dimaggi-ai/span-contract),
|
|
34
|
+
which decides whether a job may span at all. This one is about the circuit
|
|
35
|
+
underneath that decision.
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
pip install optical-circuit-intent
|
|
39
|
+
ocintent ladder # the retune legality ladder, at the reference rhythm
|
|
40
|
+
ocintent checkpoint # what each checkpoint strategy actually costs
|
|
41
|
+
ocintent disagree # where the two scheduling objectives part company
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Three findings
|
|
47
|
+
|
|
48
|
+
### 1. Two defensible retune objectives disagree over a 765-second band
|
|
49
|
+
|
|
50
|
+
A circuit retune has to wait for a boundary in the job's rhythm. Which boundary
|
|
51
|
+
you pick depends on what you are minimising, and there are two reasonable
|
|
52
|
+
answers:
|
|
53
|
+
|
|
54
|
+
- **`cheapest_legal`** minimises accelerator-hours lost. Waiting costs nothing,
|
|
55
|
+
so it waits for a boundary where the retune is invisible.
|
|
56
|
+
- **`soonest_legal`** minimises total delay. Stalling costs time, so it takes
|
|
57
|
+
the first boundary where the retune is merely expensive.
|
|
58
|
+
|
|
59
|
+
Both are defensible. At the reference rhythm — 16,384 accelerators, a 2.4 s
|
|
60
|
+
step, a checkpoint every 250 steps — they pick different boundaries for every
|
|
61
|
+
retune between **2.09 s and 767.09 s**:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
retune of 60 s against a 16,384-accelerator job
|
|
65
|
+
|
|
66
|
+
boundary legality quiet wait stall accel-h
|
|
67
|
+
mid-collective stall 0.00 0.0 60.00 273.1
|
|
68
|
+
between-steps stall 2.09 1.2 57.91 263.6
|
|
69
|
+
between-checkpoints invisible 122.09 300.0 0.00 0.0
|
|
70
|
+
between-epochs invisible 167.09 4800.0 0.00 0.0
|
|
71
|
+
between-jobs invisible inf 43200.0 0.00 0.0
|
|
72
|
+
|
|
73
|
+
cheapest_legal between-checkpoints (0.0 accel-h, 300 s of delay)
|
|
74
|
+
soonest_legal between-steps (263.6 accel-h, 59 s of delay)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
The sharp part: **whether checkpoints cross the stitch changes that wait by
|
|
78
|
+
144×.** If they stay local, the cheap option waits 300 s for the next
|
|
79
|
+
checkpoint. If they replicate over the same circuit being retuned, no checkpoint
|
|
80
|
+
boundary is quiet, the cheap option falls through to the next *job* boundary,
|
|
81
|
+
and the wait becomes 43,200 s. A storage decision, made by a different team,
|
|
82
|
+
sets the cost of a network operation.
|
|
83
|
+
|
|
84
|
+
**Who this does not apply to.** A MEMS optical circuit switch reconfigures in
|
|
85
|
+
tens of milliseconds, which is below the lower edge of every band measured here.
|
|
86
|
+
For an intra-campus OCS the two objectives agree and the ladder is academic. The
|
|
87
|
+
disagreement is a *slow*-reconfiguration problem: ROADM provisioning, a metro
|
|
88
|
+
turn-up, a hand patch. That is the honest scope.
|
|
89
|
+
|
|
90
|
+
Reproduce: `make experiments`, or `python experiments/retune_disagreement.py`.
|
|
91
|
+
|
|
92
|
+
### 2. The cheapest durable checkpoint strategy changes twice inside the range of real plants
|
|
93
|
+
|
|
94
|
+
The stop tax — how long the job is halted — is visible in any training curve.
|
|
95
|
+
The contention tax — replication traffic stealing bandwidth from the next
|
|
96
|
+
collective — is invisible unless someone measures it. Counting both, the
|
|
97
|
+
cheapest strategy that survives losing a hall is not fixed:
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
stitch write-local async-replicate sync-replicate stage-through-object
|
|
101
|
+
100G 10.45 22.01* 35.90 35.90
|
|
102
|
+
200G 10.45 16.93* 21.88 21.88
|
|
103
|
+
400G 10.45 13.69 12.28* 12.28
|
|
104
|
+
800G 10.45 12.07 10.45 6.54*
|
|
105
|
+
1600G 10.45 11.26 10.45 3.38*
|
|
106
|
+
3200G 10.45 10.85 10.45 1.72*
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Percent of wall clock lost to checkpointing; `*` marks the cheapest strategy
|
|
110
|
+
that survives losing the hall it wrote in. `write-local` is never marked — it is
|
|
111
|
+
cheapest everywhere and durable nowhere.
|
|
112
|
+
|
|
113
|
+
At 400G the strategy with the **shortest checkpoint window** (async replicate,
|
|
114
|
+
70 s) is not the cheapest (sync replicate). The difference is entirely in
|
|
115
|
+
bandwidth stolen from collectives, which no training curve will show you.
|
|
116
|
+
|
|
117
|
+
### 3. A switch four-fifths empty can be unable to admit anything
|
|
118
|
+
|
|
119
|
+
Optical ports belong to trunks, and trunks lead somewhere specific. Free ports on
|
|
120
|
+
the wrong trunk are **stranded**: they exist, they are unallocated, and they
|
|
121
|
+
cannot serve the demand waiting for them.
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
6/32 ports used, 26 free
|
|
125
|
+
free on the trunk the demand wants: 2
|
|
126
|
+
free on the trunk it does not: 24
|
|
127
|
+
stranded: 26 (100% of free capacity)
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Over 400 randomly generated switches and demands, the median switch has **83% of
|
|
131
|
+
its free ports stranded**, and 36% have every free port stranded. A capacity
|
|
132
|
+
report that counts free ports is counting the wrong thing.
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## The other three models
|
|
137
|
+
|
|
138
|
+
**Intent** (`ocintent.intent`) — a verb, two endpoints, a bandwidth, a hold time,
|
|
139
|
+
compiled into generic operations. No vendor session is opened; plans are
|
|
140
|
+
returned, not executed. A failover always verifies the replacement before
|
|
141
|
+
tearing the old path down, because the reverse order turns a degraded circuit
|
|
142
|
+
into no circuit.
|
|
143
|
+
|
|
144
|
+
**Drift** (`ocintent.drift`) — the campus bug: a YAML that says 800G at 8 µs when
|
|
145
|
+
the path is 40 µs with a dirty connector. Hashes the declared circuit, compares
|
|
146
|
+
it to the measured one, and refuses to compare fields at all when the plant and
|
|
147
|
+
the inventory disagree about *which* circuit this is. Forecasts when a
|
|
148
|
+
slowly-worsening path crosses an error-rate target, using the published
|
|
149
|
+
`0.5·erfc(Q/√2)` relation — steep enough that 2 dB of lost margin moves the error
|
|
150
|
+
rate by six orders of magnitude, which is why a gentle insertion-loss trend is
|
|
151
|
+
the only warning you get.
|
|
152
|
+
|
|
153
|
+
**Ledger** (`ocintent.ledger`) — all of the above, aged and totalled in the shape
|
|
154
|
+
of an accounts-receivable schedule, because that is a format people already know
|
|
155
|
+
how to read:
|
|
156
|
+
|
|
157
|
+
```
|
|
158
|
+
outstanding capacity debt: 620,292 accelerator-hours
|
|
159
|
+
open entries: 4 of 4
|
|
160
|
+
|
|
161
|
+
by age
|
|
162
|
+
0-7d 0 0.0%
|
|
163
|
+
7-30d 57,014 9.2%
|
|
164
|
+
30-90d 2,945 0.5%
|
|
165
|
+
90d+ 560,333 90.3%
|
|
166
|
+
|
|
167
|
+
fix these first (by daily rate, with payback)
|
|
168
|
+
drift-stitch-ab-1 5,898/day 95 d old pays back in 2.1 d
|
|
169
|
+
ckpt-b 4,247/day 8 d old pays back in 1.9 d
|
|
170
|
+
stranded-hall-a 1,152/day 20 d old pays back in 0.8 d
|
|
171
|
+
retune-ab 0/day 40 d old never pays back
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Ranked by *daily rate*, not accrued total: a large old entry that has stopped
|
|
175
|
+
bleeding is a worse use of a maintenance window than a small new one that has
|
|
176
|
+
not. The ledger quotes no currency — that needs a rate only the plant owner has,
|
|
177
|
+
and one invented here would travel downstream looking like a measurement.
|
|
178
|
+
|
|
179
|
+
---
|
|
180
|
+
|
|
181
|
+
## What this does not do
|
|
182
|
+
|
|
183
|
+
The validation registry prints its declined list *before* its results, every
|
|
184
|
+
run. The short version:
|
|
185
|
+
|
|
186
|
+
- **No measured plant.** Every number is a model output. Nothing has been
|
|
187
|
+
compared against a real optical switch, ROADM, or metro span.
|
|
188
|
+
- **Two calibrated points, one relation.** Both anchors pin `0.5·erfc(Q/√2)` at
|
|
189
|
+
different places. A systematic error in that relation leaves both green.
|
|
190
|
+
- **Connector contamination is invisible to the forecast.** It is an event, not a
|
|
191
|
+
trend, and it is the most common cause of real insertion-loss faults. A green
|
|
192
|
+
forecast is not a statement that a path is healthy.
|
|
193
|
+
- **No queueing model.** Contention shares a circuit linearly between a
|
|
194
|
+
replication and a collective, which is a simplification at every width quoted.
|
|
195
|
+
|
|
196
|
+
Run `make validate` for the other seven.
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
## Reproducing
|
|
201
|
+
|
|
202
|
+
```
|
|
203
|
+
make venv # pinned virtual environment, Python 3.12
|
|
204
|
+
make test # 137 tests, including 17 mutation tests
|
|
205
|
+
make validate # 25 registry points, and the 11 things it declines to check
|
|
206
|
+
make examples # 17 example inputs reach their documented results
|
|
207
|
+
make experiments # the three figures above
|
|
208
|
+
make smoke-test # everything except experiments, under a minute
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
Every validation point is one of three kinds. **Calibrated** points are pinned
|
|
212
|
+
to a published figure (there are two). **Emergent** points are orderings nothing
|
|
213
|
+
was tuned to produce (eleven). **Sanity** points check this repository's own
|
|
214
|
+
structure and are worth nothing as evidence about optical plants (twelve) — they
|
|
215
|
+
carry no citation, and the code refuses to let them carry one.
|
|
216
|
+
|
|
217
|
+
The mutation tests break real machinery and assert the *exact* set of points that
|
|
218
|
+
turns red. One of them asserts the registry does **not** notice a hundred-fold
|
|
219
|
+
error in the fibre thermal coefficient, because it genuinely cannot: that value
|
|
220
|
+
is an input, and closing the gap needs a measurement from a real span.
|
|
221
|
+
|
|
222
|
+
## Install
|
|
223
|
+
|
|
224
|
+
```
|
|
225
|
+
pip install optical-circuit-intent
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
No dependencies outside the standard library.
|
|
229
|
+
|
|
230
|
+
## Documents
|
|
231
|
+
|
|
232
|
+
- [`docs/the-models.md`](docs/the-models.md) — what each of the six is for
|
|
233
|
+
- [`docs/integration.md`](docs/integration.md) — wiring this to a scheduler
|
|
234
|
+
- [`DECISIONS.md`](DECISIONS.md) — twelve choices, and what each cost
|
|
235
|
+
- [`ASSUMPTIONS.md`](ASSUMPTIONS.md) — what is taken on faith
|
|
236
|
+
- [`SOURCES.md`](SOURCES.md) — the published figures the calibrated points use
|
|
237
|
+
- [`STATUS.md`](STATUS.md) — what is done, what is not, what would change it
|
|
238
|
+
|
|
239
|
+
Part of the [DIMAGGI usable-capacity series](https://dimaggi-ai.github.io/research).
|
|
240
|
+
|
|
241
|
+
MIT licensed. Copyright (c) 2026 Margaret Nanyonga.
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
# optical-circuit-intent
|
|
2
|
+
|
|
3
|
+
**What to ask an optical plant for, when it is legal to ask, and what the answer
|
|
4
|
+
costs.**
|
|
5
|
+
|
|
6
|
+
A training job that crosses a data hall boundary sits on a circuit somebody has
|
|
7
|
+
to provision, retune, and eventually admit is not what the inventory says it is.
|
|
8
|
+
This repository models six parts of that problem, in the unit the rest of the
|
|
9
|
+
series uses — accelerator-hours — so the numbers can be added up.
|
|
10
|
+
|
|
11
|
+
It is a companion to [span-contract](https://github.com/dimaggi-ai/span-contract),
|
|
12
|
+
which decides whether a job may span at all. This one is about the circuit
|
|
13
|
+
underneath that decision.
|
|
14
|
+
|
|
15
|
+
```
|
|
16
|
+
pip install optical-circuit-intent
|
|
17
|
+
ocintent ladder # the retune legality ladder, at the reference rhythm
|
|
18
|
+
ocintent checkpoint # what each checkpoint strategy actually costs
|
|
19
|
+
ocintent disagree # where the two scheduling objectives part company
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## Three findings
|
|
25
|
+
|
|
26
|
+
### 1. Two defensible retune objectives disagree over a 765-second band
|
|
27
|
+
|
|
28
|
+
A circuit retune has to wait for a boundary in the job's rhythm. Which boundary
|
|
29
|
+
you pick depends on what you are minimising, and there are two reasonable
|
|
30
|
+
answers:
|
|
31
|
+
|
|
32
|
+
- **`cheapest_legal`** minimises accelerator-hours lost. Waiting costs nothing,
|
|
33
|
+
so it waits for a boundary where the retune is invisible.
|
|
34
|
+
- **`soonest_legal`** minimises total delay. Stalling costs time, so it takes
|
|
35
|
+
the first boundary where the retune is merely expensive.
|
|
36
|
+
|
|
37
|
+
Both are defensible. At the reference rhythm — 16,384 accelerators, a 2.4 s
|
|
38
|
+
step, a checkpoint every 250 steps — they pick different boundaries for every
|
|
39
|
+
retune between **2.09 s and 767.09 s**:
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
retune of 60 s against a 16,384-accelerator job
|
|
43
|
+
|
|
44
|
+
boundary legality quiet wait stall accel-h
|
|
45
|
+
mid-collective stall 0.00 0.0 60.00 273.1
|
|
46
|
+
between-steps stall 2.09 1.2 57.91 263.6
|
|
47
|
+
between-checkpoints invisible 122.09 300.0 0.00 0.0
|
|
48
|
+
between-epochs invisible 167.09 4800.0 0.00 0.0
|
|
49
|
+
between-jobs invisible inf 43200.0 0.00 0.0
|
|
50
|
+
|
|
51
|
+
cheapest_legal between-checkpoints (0.0 accel-h, 300 s of delay)
|
|
52
|
+
soonest_legal between-steps (263.6 accel-h, 59 s of delay)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The sharp part: **whether checkpoints cross the stitch changes that wait by
|
|
56
|
+
144×.** If they stay local, the cheap option waits 300 s for the next
|
|
57
|
+
checkpoint. If they replicate over the same circuit being retuned, no checkpoint
|
|
58
|
+
boundary is quiet, the cheap option falls through to the next *job* boundary,
|
|
59
|
+
and the wait becomes 43,200 s. A storage decision, made by a different team,
|
|
60
|
+
sets the cost of a network operation.
|
|
61
|
+
|
|
62
|
+
**Who this does not apply to.** A MEMS optical circuit switch reconfigures in
|
|
63
|
+
tens of milliseconds, which is below the lower edge of every band measured here.
|
|
64
|
+
For an intra-campus OCS the two objectives agree and the ladder is academic. The
|
|
65
|
+
disagreement is a *slow*-reconfiguration problem: ROADM provisioning, a metro
|
|
66
|
+
turn-up, a hand patch. That is the honest scope.
|
|
67
|
+
|
|
68
|
+
Reproduce: `make experiments`, or `python experiments/retune_disagreement.py`.
|
|
69
|
+
|
|
70
|
+
### 2. The cheapest durable checkpoint strategy changes twice inside the range of real plants
|
|
71
|
+
|
|
72
|
+
The stop tax — how long the job is halted — is visible in any training curve.
|
|
73
|
+
The contention tax — replication traffic stealing bandwidth from the next
|
|
74
|
+
collective — is invisible unless someone measures it. Counting both, the
|
|
75
|
+
cheapest strategy that survives losing a hall is not fixed:
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
stitch write-local async-replicate sync-replicate stage-through-object
|
|
79
|
+
100G 10.45 22.01* 35.90 35.90
|
|
80
|
+
200G 10.45 16.93* 21.88 21.88
|
|
81
|
+
400G 10.45 13.69 12.28* 12.28
|
|
82
|
+
800G 10.45 12.07 10.45 6.54*
|
|
83
|
+
1600G 10.45 11.26 10.45 3.38*
|
|
84
|
+
3200G 10.45 10.85 10.45 1.72*
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Percent of wall clock lost to checkpointing; `*` marks the cheapest strategy
|
|
88
|
+
that survives losing the hall it wrote in. `write-local` is never marked — it is
|
|
89
|
+
cheapest everywhere and durable nowhere.
|
|
90
|
+
|
|
91
|
+
At 400G the strategy with the **shortest checkpoint window** (async replicate,
|
|
92
|
+
70 s) is not the cheapest (sync replicate). The difference is entirely in
|
|
93
|
+
bandwidth stolen from collectives, which no training curve will show you.
|
|
94
|
+
|
|
95
|
+
### 3. A switch four-fifths empty can be unable to admit anything
|
|
96
|
+
|
|
97
|
+
Optical ports belong to trunks, and trunks lead somewhere specific. Free ports on
|
|
98
|
+
the wrong trunk are **stranded**: they exist, they are unallocated, and they
|
|
99
|
+
cannot serve the demand waiting for them.
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
6/32 ports used, 26 free
|
|
103
|
+
free on the trunk the demand wants: 2
|
|
104
|
+
free on the trunk it does not: 24
|
|
105
|
+
stranded: 26 (100% of free capacity)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Over 400 randomly generated switches and demands, the median switch has **83% of
|
|
109
|
+
its free ports stranded**, and 36% have every free port stranded. A capacity
|
|
110
|
+
report that counts free ports is counting the wrong thing.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## The other three models
|
|
115
|
+
|
|
116
|
+
**Intent** (`ocintent.intent`) — a verb, two endpoints, a bandwidth, a hold time,
|
|
117
|
+
compiled into generic operations. No vendor session is opened; plans are
|
|
118
|
+
returned, not executed. A failover always verifies the replacement before
|
|
119
|
+
tearing the old path down, because the reverse order turns a degraded circuit
|
|
120
|
+
into no circuit.
|
|
121
|
+
|
|
122
|
+
**Drift** (`ocintent.drift`) — the campus bug: a YAML that says 800G at 8 µs when
|
|
123
|
+
the path is 40 µs with a dirty connector. Hashes the declared circuit, compares
|
|
124
|
+
it to the measured one, and refuses to compare fields at all when the plant and
|
|
125
|
+
the inventory disagree about *which* circuit this is. Forecasts when a
|
|
126
|
+
slowly-worsening path crosses an error-rate target, using the published
|
|
127
|
+
`0.5·erfc(Q/√2)` relation — steep enough that 2 dB of lost margin moves the error
|
|
128
|
+
rate by six orders of magnitude, which is why a gentle insertion-loss trend is
|
|
129
|
+
the only warning you get.
|
|
130
|
+
|
|
131
|
+
**Ledger** (`ocintent.ledger`) — all of the above, aged and totalled in the shape
|
|
132
|
+
of an accounts-receivable schedule, because that is a format people already know
|
|
133
|
+
how to read:
|
|
134
|
+
|
|
135
|
+
```
|
|
136
|
+
outstanding capacity debt: 620,292 accelerator-hours
|
|
137
|
+
open entries: 4 of 4
|
|
138
|
+
|
|
139
|
+
by age
|
|
140
|
+
0-7d 0 0.0%
|
|
141
|
+
7-30d 57,014 9.2%
|
|
142
|
+
30-90d 2,945 0.5%
|
|
143
|
+
90d+ 560,333 90.3%
|
|
144
|
+
|
|
145
|
+
fix these first (by daily rate, with payback)
|
|
146
|
+
drift-stitch-ab-1 5,898/day 95 d old pays back in 2.1 d
|
|
147
|
+
ckpt-b 4,247/day 8 d old pays back in 1.9 d
|
|
148
|
+
stranded-hall-a 1,152/day 20 d old pays back in 0.8 d
|
|
149
|
+
retune-ab 0/day 40 d old never pays back
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Ranked by *daily rate*, not accrued total: a large old entry that has stopped
|
|
153
|
+
bleeding is a worse use of a maintenance window than a small new one that has
|
|
154
|
+
not. The ledger quotes no currency — that needs a rate only the plant owner has,
|
|
155
|
+
and one invented here would travel downstream looking like a measurement.
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
## What this does not do
|
|
160
|
+
|
|
161
|
+
The validation registry prints its declined list *before* its results, every
|
|
162
|
+
run. The short version:
|
|
163
|
+
|
|
164
|
+
- **No measured plant.** Every number is a model output. Nothing has been
|
|
165
|
+
compared against a real optical switch, ROADM, or metro span.
|
|
166
|
+
- **Two calibrated points, one relation.** Both anchors pin `0.5·erfc(Q/√2)` at
|
|
167
|
+
different places. A systematic error in that relation leaves both green.
|
|
168
|
+
- **Connector contamination is invisible to the forecast.** It is an event, not a
|
|
169
|
+
trend, and it is the most common cause of real insertion-loss faults. A green
|
|
170
|
+
forecast is not a statement that a path is healthy.
|
|
171
|
+
- **No queueing model.** Contention shares a circuit linearly between a
|
|
172
|
+
replication and a collective, which is a simplification at every width quoted.
|
|
173
|
+
|
|
174
|
+
Run `make validate` for the other seven.
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## Reproducing
|
|
179
|
+
|
|
180
|
+
```
|
|
181
|
+
make venv # pinned virtual environment, Python 3.12
|
|
182
|
+
make test # 137 tests, including 17 mutation tests
|
|
183
|
+
make validate # 25 registry points, and the 11 things it declines to check
|
|
184
|
+
make examples # 17 example inputs reach their documented results
|
|
185
|
+
make experiments # the three figures above
|
|
186
|
+
make smoke-test # everything except experiments, under a minute
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Every validation point is one of three kinds. **Calibrated** points are pinned
|
|
190
|
+
to a published figure (there are two). **Emergent** points are orderings nothing
|
|
191
|
+
was tuned to produce (eleven). **Sanity** points check this repository's own
|
|
192
|
+
structure and are worth nothing as evidence about optical plants (twelve) — they
|
|
193
|
+
carry no citation, and the code refuses to let them carry one.
|
|
194
|
+
|
|
195
|
+
The mutation tests break real machinery and assert the *exact* set of points that
|
|
196
|
+
turns red. One of them asserts the registry does **not** notice a hundred-fold
|
|
197
|
+
error in the fibre thermal coefficient, because it genuinely cannot: that value
|
|
198
|
+
is an input, and closing the gap needs a measurement from a real span.
|
|
199
|
+
|
|
200
|
+
## Install
|
|
201
|
+
|
|
202
|
+
```
|
|
203
|
+
pip install optical-circuit-intent
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
No dependencies outside the standard library.
|
|
207
|
+
|
|
208
|
+
## Documents
|
|
209
|
+
|
|
210
|
+
- [`docs/the-models.md`](docs/the-models.md) — what each of the six is for
|
|
211
|
+
- [`docs/integration.md`](docs/integration.md) — wiring this to a scheduler
|
|
212
|
+
- [`DECISIONS.md`](DECISIONS.md) — twelve choices, and what each cost
|
|
213
|
+
- [`ASSUMPTIONS.md`](ASSUMPTIONS.md) — what is taken on faith
|
|
214
|
+
- [`SOURCES.md`](SOURCES.md) — the published figures the calibrated points use
|
|
215
|
+
- [`STATUS.md`](STATUS.md) — what is done, what is not, what would change it
|
|
216
|
+
|
|
217
|
+
Part of the [DIMAGGI usable-capacity series](https://dimaggi-ai.github.io/research).
|
|
218
|
+
|
|
219
|
+
MIT licensed. Copyright (c) 2026 Margaret Nanyonga.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "optical-circuit-intent"
|
|
7
|
+
version = "1.0.1"
|
|
8
|
+
description = "What to ask an optical plant for, when it is legal to ask, and what the answer costs"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Margaret Nanyonga" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"optical circuit switching",
|
|
15
|
+
"distributed training",
|
|
16
|
+
"capacity planning",
|
|
17
|
+
"data center interconnect",
|
|
18
|
+
"checkpointing",
|
|
19
|
+
]
|
|
20
|
+
classifiers = [
|
|
21
|
+
"Intended Audience :: Science/Research",
|
|
22
|
+
"Intended Audience :: System Administrators",
|
|
23
|
+
"License :: OSI Approved :: MIT License",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Topic :: Scientific/Engineering",
|
|
26
|
+
"Topic :: System :: Distributed Computing",
|
|
27
|
+
]
|
|
28
|
+
dependencies = []
|
|
29
|
+
|
|
30
|
+
[project.optional-dependencies]
|
|
31
|
+
dev = ["pytest==8.3.4"]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/dimaggi-ai/optical-circuit-intent"
|
|
35
|
+
Research = "https://dimaggi-ai.github.io/research"
|
|
36
|
+
|
|
37
|
+
[project.scripts]
|
|
38
|
+
ocintent = "ocintent.cli:main"
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.packages.find]
|
|
41
|
+
where = ["src"]
|
|
42
|
+
|
|
43
|
+
[tool.pytest.ini_options]
|
|
44
|
+
testpaths = ["tests"]
|
|
45
|
+
pythonpath = ["src", "validation"]
|
|
46
|
+
addopts = "-q"
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""What to ask an optical plant for, when it is legal to ask, and what it costs.
|
|
2
|
+
|
|
3
|
+
Six models, each usable on its own:
|
|
4
|
+
|
|
5
|
+
``intent``
|
|
6
|
+
A verb, two endpoints, a bandwidth, a boundary. Compiles to a plan of
|
|
7
|
+
generic operations. No vendor session is opened; plans are returned, not
|
|
8
|
+
executed (DECISIONS.md D2).
|
|
9
|
+
``legality``
|
|
10
|
+
When a retune is invisible, when it stalls the job, and when it kills it.
|
|
11
|
+
Two defensible objectives that disagree over a wide band of retune times.
|
|
12
|
+
``radix``
|
|
13
|
+
Ports are not interchangeable across trunks. Free ports that cannot serve
|
|
14
|
+
pending demand are stranded, and stranded ports are the fragmentation
|
|
15
|
+
number that matters.
|
|
16
|
+
``checkpoint``
|
|
17
|
+
The stop tax you can see in a training curve, and the contention tax you
|
|
18
|
+
cannot see unless you measure it.
|
|
19
|
+
``drift``
|
|
20
|
+
The declared circuit against the measured one, and a forecast of when a
|
|
21
|
+
path that is fine now stops being fine.
|
|
22
|
+
``ledger``
|
|
23
|
+
All of the above, aged and totalled, in the shape finance already reads.
|
|
24
|
+
|
|
25
|
+
``drift.compare`` and ``checkpoint.compare`` are both named ``compare`` in
|
|
26
|
+
their own modules, which is right there and wrong here, so this namespace
|
|
27
|
+
exports them as :func:`compare_circuit` and :func:`compare_strategies`.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
from . import checkpoint, drift, intent, ledger, legality, radix
|
|
33
|
+
from .checkpoint import (
|
|
34
|
+
CheckpointPlan,
|
|
35
|
+
StallCause,
|
|
36
|
+
StallEvidence,
|
|
37
|
+
Strategy,
|
|
38
|
+
Tax,
|
|
39
|
+
cheapest_durable,
|
|
40
|
+
classify_stall,
|
|
41
|
+
)
|
|
42
|
+
from .checkpoint import compare as compare_strategies
|
|
43
|
+
from .checkpoint import tax
|
|
44
|
+
from .drift import (
|
|
45
|
+
DeclaredCircuit,
|
|
46
|
+
DriftForecast,
|
|
47
|
+
DriftReport,
|
|
48
|
+
DriftVerdict,
|
|
49
|
+
MeasuredCircuit,
|
|
50
|
+
THERMAL_DELAY_PS_PER_KM_K,
|
|
51
|
+
ber_from_margin_db,
|
|
52
|
+
ber_from_q,
|
|
53
|
+
forecast,
|
|
54
|
+
q_from_margin_db,
|
|
55
|
+
thermal_rtt_swing_us,
|
|
56
|
+
)
|
|
57
|
+
from .drift import compare as compare_circuit
|
|
58
|
+
from .intent import Boundary, Endpoint, Intent, Operation, Plan, Verb, compile_intent
|
|
59
|
+
from .ledger import AGING_BUCKETS, Cause, DebtEntry, Ledger, debt_from_drift, debt_from_stranded_ports
|
|
60
|
+
from .legality import (
|
|
61
|
+
JobRhythm,
|
|
62
|
+
Legality,
|
|
63
|
+
RetuneCost,
|
|
64
|
+
assess,
|
|
65
|
+
cheapest_legal,
|
|
66
|
+
disagreement_intervals,
|
|
67
|
+
disagreement_width_s,
|
|
68
|
+
ladder,
|
|
69
|
+
objectives_disagree,
|
|
70
|
+
soonest_legal,
|
|
71
|
+
)
|
|
72
|
+
from .radix import (
|
|
73
|
+
Allocation,
|
|
74
|
+
OpticalSwitch,
|
|
75
|
+
Preemption,
|
|
76
|
+
RadixExhausted,
|
|
77
|
+
Request,
|
|
78
|
+
Trunk,
|
|
79
|
+
preemption_plan,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
__version__ = "1.0.1"
|
|
83
|
+
|
|
84
|
+
__all__ = [
|
|
85
|
+
"__version__",
|
|
86
|
+
# modules
|
|
87
|
+
"checkpoint", "drift", "intent", "ledger", "legality", "radix",
|
|
88
|
+
# intent
|
|
89
|
+
"Boundary", "Endpoint", "Intent", "Operation", "Plan", "Verb", "compile_intent",
|
|
90
|
+
# legality
|
|
91
|
+
"JobRhythm", "Legality", "RetuneCost", "assess", "cheapest_legal", "ladder",
|
|
92
|
+
"soonest_legal", "objectives_disagree", "disagreement_intervals",
|
|
93
|
+
"disagreement_width_s",
|
|
94
|
+
# radix
|
|
95
|
+
"Allocation", "OpticalSwitch", "Preemption", "RadixExhausted", "Request", "Trunk",
|
|
96
|
+
"preemption_plan",
|
|
97
|
+
# checkpoint
|
|
98
|
+
"CheckpointPlan", "Strategy", "Tax", "tax", "compare_strategies",
|
|
99
|
+
"cheapest_durable", "StallCause", "StallEvidence", "classify_stall",
|
|
100
|
+
# drift
|
|
101
|
+
"DeclaredCircuit", "MeasuredCircuit", "DriftReport", "DriftVerdict",
|
|
102
|
+
"DriftForecast", "compare_circuit", "forecast", "ber_from_q",
|
|
103
|
+
"q_from_margin_db", "ber_from_margin_db", "thermal_rtt_swing_us",
|
|
104
|
+
"THERMAL_DELAY_PS_PER_KM_K",
|
|
105
|
+
# ledger
|
|
106
|
+
"Cause", "DebtEntry", "Ledger", "AGING_BUCKETS", "debt_from_drift",
|
|
107
|
+
"debt_from_stranded_ports",
|
|
108
|
+
]
|