spectrax-lib 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. spectrax_lib-0.0.1/PKG-INFO +437 -0
  2. spectrax_lib-0.0.1/README.md +396 -0
  3. spectrax_lib-0.0.1/pyproject.toml +151 -0
  4. spectrax_lib-0.0.1/spectrax/__init__.py +204 -0
  5. spectrax_lib-0.0.1/spectrax/_version.py +20 -0
  6. spectrax_lib-0.0.1/spectrax/api.py +357 -0
  7. spectrax_lib-0.0.1/spectrax/contrib/__init__.py +23 -0
  8. spectrax_lib-0.0.1/spectrax/contrib/optimizer.py +377 -0
  9. spectrax_lib-0.0.1/spectrax/core/__init__.py +57 -0
  10. spectrax_lib-0.0.1/spectrax/core/_typing.py +144 -0
  11. spectrax_lib-0.0.1/spectrax/core/_weakcache.py +56 -0
  12. spectrax_lib-0.0.1/spectrax/core/containers.py +336 -0
  13. spectrax_lib-0.0.1/spectrax/core/context.py +187 -0
  14. spectrax_lib-0.0.1/spectrax/core/errors.py +89 -0
  15. spectrax_lib-0.0.1/spectrax/core/graph.py +858 -0
  16. spectrax_lib-0.0.1/spectrax/core/lazy_init.py +63 -0
  17. spectrax_lib-0.0.1/spectrax/core/module.py +975 -0
  18. spectrax_lib-0.0.1/spectrax/core/paths.py +128 -0
  19. spectrax_lib-0.0.1/spectrax/core/policy.py +114 -0
  20. spectrax_lib-0.0.1/spectrax/core/registry.py +102 -0
  21. spectrax_lib-0.0.1/spectrax/core/selector.py +553 -0
  22. spectrax_lib-0.0.1/spectrax/core/sharding.py +156 -0
  23. spectrax_lib-0.0.1/spectrax/core/stage_assignment.py +109 -0
  24. spectrax_lib-0.0.1/spectrax/core/state.py +361 -0
  25. spectrax_lib-0.0.1/spectrax/core/static.py +85 -0
  26. spectrax_lib-0.0.1/spectrax/core/variable.py +898 -0
  27. spectrax_lib-0.0.1/spectrax/functional/__init__.py +84 -0
  28. spectrax_lib-0.0.1/spectrax/functional/activation.py +132 -0
  29. spectrax_lib-0.0.1/spectrax/functional/attention.py +81 -0
  30. spectrax_lib-0.0.1/spectrax/functional/conv.py +129 -0
  31. spectrax_lib-0.0.1/spectrax/functional/dropout.py +56 -0
  32. spectrax_lib-0.0.1/spectrax/functional/linear.py +74 -0
  33. spectrax_lib-0.0.1/spectrax/functional/norm.py +83 -0
  34. spectrax_lib-0.0.1/spectrax/functional/pool.py +106 -0
  35. spectrax_lib-0.0.1/spectrax/functional/util.py +43 -0
  36. spectrax_lib-0.0.1/spectrax/hooks/__init__.py +30 -0
  37. spectrax_lib-0.0.1/spectrax/hooks/forward.py +48 -0
  38. spectrax_lib-0.0.1/spectrax/hooks/variable.py +60 -0
  39. spectrax_lib-0.0.1/spectrax/init/__init__.py +41 -0
  40. spectrax_lib-0.0.1/spectrax/init/constant.py +47 -0
  41. spectrax_lib-0.0.1/spectrax/init/kaiming.py +75 -0
  42. spectrax_lib-0.0.1/spectrax/init/normal.py +49 -0
  43. spectrax_lib-0.0.1/spectrax/init/orthogonal.py +54 -0
  44. spectrax_lib-0.0.1/spectrax/init/uniform.py +31 -0
  45. spectrax_lib-0.0.1/spectrax/init/xavier.py +82 -0
  46. spectrax_lib-0.0.1/spectrax/inspect/__init__.py +35 -0
  47. spectrax_lib-0.0.1/spectrax/inspect/counting.py +31 -0
  48. spectrax_lib-0.0.1/spectrax/inspect/display.py +37 -0
  49. spectrax_lib-0.0.1/spectrax/inspect/repr.py +107 -0
  50. spectrax_lib-0.0.1/spectrax/inspect/summary.py +83 -0
  51. spectrax_lib-0.0.1/spectrax/inspect/tabulate.py +142 -0
  52. spectrax_lib-0.0.1/spectrax/inspect/tree.py +47 -0
  53. spectrax_lib-0.0.1/spectrax/lint.py +66 -0
  54. spectrax_lib-0.0.1/spectrax/nn/__init__.py +125 -0
  55. spectrax_lib-0.0.1/spectrax/nn/activation.py +95 -0
  56. spectrax_lib-0.0.1/spectrax/nn/attention.py +277 -0
  57. spectrax_lib-0.0.1/spectrax/nn/conv.py +306 -0
  58. spectrax_lib-0.0.1/spectrax/nn/dense.py +202 -0
  59. spectrax_lib-0.0.1/spectrax/nn/dropout.py +97 -0
  60. spectrax_lib-0.0.1/spectrax/nn/embed.py +101 -0
  61. spectrax_lib-0.0.1/spectrax/nn/fp8.py +724 -0
  62. spectrax_lib-0.0.1/spectrax/nn/identity.py +36 -0
  63. spectrax_lib-0.0.1/spectrax/nn/linear.py +188 -0
  64. spectrax_lib-0.0.1/spectrax/nn/lora.py +356 -0
  65. spectrax_lib-0.0.1/spectrax/nn/mlp.py +112 -0
  66. spectrax_lib-0.0.1/spectrax/nn/norm.py +358 -0
  67. spectrax_lib-0.0.1/spectrax/nn/pipeline_sequential.py +141 -0
  68. spectrax_lib-0.0.1/spectrax/nn/pool.py +170 -0
  69. spectrax_lib-0.0.1/spectrax/nn/recurrent.py +556 -0
  70. spectrax_lib-0.0.1/spectrax/py.typed +0 -0
  71. spectrax_lib-0.0.1/spectrax/rng/__init__.py +26 -0
  72. spectrax_lib-0.0.1/spectrax/rng/rngs.py +258 -0
  73. spectrax_lib-0.0.1/spectrax/rng/seed.py +81 -0
  74. spectrax_lib-0.0.1/spectrax/runtime/__init__.py +81 -0
  75. spectrax_lib-0.0.1/spectrax/runtime/mpmd/__init__.py +95 -0
  76. spectrax_lib-0.0.1/spectrax/runtime/mpmd/compiler.py +334 -0
  77. spectrax_lib-0.0.1/spectrax/runtime/mpmd/markers.py +447 -0
  78. spectrax_lib-0.0.1/spectrax/runtime/mpmd/per_rank.py +341 -0
  79. spectrax_lib-0.0.1/spectrax/runtime/mpmd/pscan_compiler.py +977 -0
  80. spectrax_lib-0.0.1/spectrax/runtime/mpmd/runtime.py +3064 -0
  81. spectrax_lib-0.0.1/spectrax/runtime/mpmd/treduce.py +483 -0
  82. spectrax_lib-0.0.1/spectrax/runtime/primitives/__init__.py +25 -0
  83. spectrax_lib-0.0.1/spectrax/runtime/primitives/boundary.py +77 -0
  84. spectrax_lib-0.0.1/spectrax/runtime/primitives/split.py +272 -0
  85. spectrax_lib-0.0.1/spectrax/runtime/schedules/__init__.py +46 -0
  86. spectrax_lib-0.0.1/spectrax/runtime/schedules/base.py +213 -0
  87. spectrax_lib-0.0.1/spectrax/runtime/schedules/dualpipe.py +260 -0
  88. spectrax_lib-0.0.1/spectrax/runtime/schedules/fusion.py +123 -0
  89. spectrax_lib-0.0.1/spectrax/runtime/schedules/gpipe.py +86 -0
  90. spectrax_lib-0.0.1/spectrax/runtime/schedules/interleaved.py +315 -0
  91. spectrax_lib-0.0.1/spectrax/runtime/schedules/one_f_one_b.py +181 -0
  92. spectrax_lib-0.0.1/spectrax/runtime/schedules/zero_bubble.py +115 -0
  93. spectrax_lib-0.0.1/spectrax/runtime/spmd/__init__.py +33 -0
  94. spectrax_lib-0.0.1/spectrax/runtime/spmd/api.py +998 -0
  95. spectrax_lib-0.0.1/spectrax/runtime/spmd/hybrid.py +360 -0
  96. spectrax_lib-0.0.1/spectrax/runtime/spmd/runtime.py +351 -0
  97. spectrax_lib-0.0.1/spectrax/runtime/spmd/shard_map.py +554 -0
  98. spectrax_lib-0.0.1/spectrax/runtime/types/__init__.py +29 -0
  99. spectrax_lib-0.0.1/spectrax/runtime/types/array.py +332 -0
  100. spectrax_lib-0.0.1/spectrax/runtime/types/mesh.py +229 -0
  101. spectrax_lib-0.0.1/spectrax/runtime/types/stage.py +44 -0
  102. spectrax_lib-0.0.1/spectrax/sharding/__init__.py +56 -0
  103. spectrax_lib-0.0.1/spectrax/sharding/logical.py +59 -0
  104. spectrax_lib-0.0.1/spectrax/sharding/mesh.py +706 -0
  105. spectrax_lib-0.0.1/spectrax/sharding/partition.py +184 -0
  106. spectrax_lib-0.0.1/spectrax/transforms/__init__.py +55 -0
  107. spectrax_lib-0.0.1/spectrax/transforms/control_flow.py +393 -0
  108. spectrax_lib-0.0.1/spectrax/transforms/eval_shape.py +47 -0
  109. spectrax_lib-0.0.1/spectrax/transforms/grad.py +683 -0
  110. spectrax_lib-0.0.1/spectrax/transforms/jit.py +343 -0
  111. spectrax_lib-0.0.1/spectrax/transforms/remat.py +254 -0
  112. spectrax_lib-0.0.1/spectrax/transforms/rng_axes.py +117 -0
  113. spectrax_lib-0.0.1/spectrax/transforms/scan.py +210 -0
  114. spectrax_lib-0.0.1/spectrax/transforms/split_merge.py +739 -0
  115. spectrax_lib-0.0.1/spectrax/transforms/vmap.py +157 -0
  116. spectrax_lib-0.0.1/spectrax/typing.py +54 -0
@@ -0,0 +1,437 @@
1
+ Metadata-Version: 2.3
2
+ Name: spectrax-lib
3
+ Version: 0.0.1
4
+ Summary: SpectraX: a JAX-only neural-network library with a PyTorch-shaped eager surface and Flax-NNX-style graph/state underneath.
5
+ Keywords: Deep Learning,Machine Learning,Neural Networks,JAX,XLA
6
+ Author: Erfan Zare Chavoshi
7
+ Author-email: Erfan Zare Chavoshi <Erfanzare810@gmail.com>
8
+ License: Apache-2.0
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
12
+ Classifier: License :: OSI Approved :: Apache Software License
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Requires-Dist: jax>=0.9.2
17
+ Requires-Dist: jaxlib>=0.9.2
18
+ Requires-Dist: numpy>=1.26
19
+ Requires-Dist: treescope>=0.1.7
20
+ Requires-Dist: optax>=0.2.8 ; extra == 'contrib'
21
+ Requires-Dist: jax[cuda13]>=0.9.2 ; extra == 'cuda'
22
+ Requires-Dist: ruff>=0.6 ; extra == 'dev'
23
+ Requires-Dist: pytest>=7 ; extra == 'dev'
24
+ Requires-Dist: sphinx>=7 ; extra == 'docs'
25
+ Requires-Dist: sphinx-book-theme>=1.1 ; extra == 'docs'
26
+ Requires-Dist: sphinx-design>=0.6 ; extra == 'docs'
27
+ Requires-Dist: sphinx-autodoc-typehints>=2 ; extra == 'docs'
28
+ Requires-Dist: myst-nb>=1 ; extra == 'docs'
29
+ Requires-Dist: pytest>=7 ; extra == 'test'
30
+ Requires-Dist: jax[tpu]>=0.9.2 ; extra == 'tpu'
31
+ Requires-Python: >=3.11, <3.14
32
+ Project-URL: Homepage, https://github.com/erfanzar/spectrax
33
+ Project-URL: Repository, https://github.com/erfanzar/spectrax
34
+ Provides-Extra: contrib
35
+ Provides-Extra: cuda
36
+ Provides-Extra: dev
37
+ Provides-Extra: docs
38
+ Provides-Extra: test
39
+ Provides-Extra: tpu
40
+ Description-Content-Type: text/markdown
41
+
42
+ # SpectraX
43
+
44
+ **True MPMD pipeline parallelism for JAX — with an eager module API.**
45
+
46
+ [**Quick Start**](#quick-start) |
47
+ [**Installation**](#installation) |
48
+ [**MPMD Runtime**](#mpmd-runtime) |
49
+ [**Benchmark**](#benchmark) |
50
+ [**Examples**](#examples) |
51
+ [**Docs**](docs/)
52
+
53
+ ---
54
+
55
+ SpectraX is a JAX-native neural network library built around **true MPMD
56
+ pipeline parallelism**. Each physical rank compiles and runs its own XLA
57
+ program — no shared `shard_map` HLO, no SPMD-same-shape constraint.
58
+ Heterogeneous stages (embed → blocks → head), multiple schedules
59
+ (GPipe, 1F1B, ZeroBubble, Interleaved, DualPipe), and a unified
60
+ `spx.run()` entry point that dispatches to SPMD or MPMD from the same
61
+ training script.
62
+
63
+ The module API is eager and debuggable — subclass `Module`, override
64
+ `forward`, call `model(x)` — but every `Module` is a JAX pytree, so
65
+ `jax.jit`, `jax.grad`, and `jax.tree.map` work directly.
66
+
67
+ ---
68
+
69
+ ## Why SpectraX?
70
+
71
+ | Capability | SpectraX | JAX + manual | other JAX frameworks |
72
+ | ------------------------ | --------------------------------------- | ------------------- | -------------------------- |
73
+ | **True MPMD** | built-in (`sxcall`, `sxjit`) | hand-rolled | SPMD-only |
74
+ | **Heterogeneous stages** | native (different class/shape per rank) | fragile | not supported |
75
+ | **Pipeline schedules** | 9 schedules (GPipe→DualPipeV) | hand-rolled | limited |
76
+ | **Unified runtime** | `spx.run(mesh)` → SPMD or MPMD | separate code paths | separate APIs |
77
+ | **Eager modules** | `model(x)` + pytree-native | functional only | functional or ref-tracking |
78
+ | **Dispatch overhead** | ~150 µs | ~N/A | ~300–2000 µs |
79
+
80
+ *Dispatch overhead measured on a tiny 2-layer CPU transformer. See [benchmark](#benchmark).*
81
+
82
+ ---
83
+
84
+ ## Quick Start
85
+
86
+ ```bash
87
+ pip install spectrax-lib
88
+ ```
89
+
90
+ ### Single-device eager training
91
+
92
+ ```python
93
+ import jax.numpy as jnp
94
+ import spectrax as spx
95
+ from spectrax import nn
96
+
97
+ class MLP(spx.Module):
98
+ def __init__(self, d, h, o, *, rngs):
99
+ super().__init__()
100
+ self.fc1 = nn.Linear(d, h, rngs=rngs)
101
+ self.fc2 = nn.Linear(h, o, rngs=rngs)
102
+
103
+ def forward(self, x):
104
+ return self.fc2(nn.gelu(self.fc1(x)))
105
+
106
+ model = MLP(16, 64, 4, rngs=spx.Rngs(0))
107
+
108
+ @spx.jit
109
+ @spx.value_and_grad
110
+ def loss_fn(m, x, y):
111
+ return ((m(x) - y) ** 2).mean()
112
+
113
+ loss, grads = loss_fn(model, jnp.ones((8, 16)), jnp.zeros((8, 4)))
114
+ ```
115
+
116
+ ### Marker-based MPMD — split a function into per-rank programs
117
+
118
+ ```python
119
+ from spectrax.runtime.mpmd import sxjit, sxstage_iter
120
+ from spectrax.pipeline import Std1F1B
121
+
122
+ # Define a multi-stage forward with explicit stage boundaries
123
+ @sxjit(schedule=Std1F1B(microbatches=8))
124
+ def pipeline_fwd(model, x):
125
+ x = model.embed(x) # stage 0
126
+ x = sxstage_iter(x) # boundary — split here
127
+ x = model.blocks[0](x) # stage 1
128
+ x = sxstage_iter(x) # boundary — split here
129
+ x = model.blocks[1](x) # stage 2
130
+ x = sxstage_iter(x) # boundary — split here
131
+ return model.head(x) # stage 3
132
+
133
+ # sxjit traces the function, splits the jaxpr at sxstage_iter markers,
134
+ # and compiles one XLA executable per stage/rank. Each rank only sees
135
+ # its own sub-graph — true MPMD, not SPMD.
136
+ output = pipeline_fwd(model, x)
137
+ ```
138
+
139
+ ### MPMD pipeline training — one call, multiple devices
140
+
141
+ ```python
142
+ from spectrax.sharding import logical_axis_rules
143
+
144
+ # Create a 4-stage pipeline mesh
145
+ mesh = spx.create_mesh(axis_dims=(2, 1, -1, 1, 1, 1), mpmd_axis="pp")
146
+
147
+ with logical_axis_rules(FSDP_TP_RULES), mesh:
148
+ # MPMD: each rank gets its own compiled stage
149
+ loss, grads = spx.run(
150
+ model, inputs=x, targets=y,
151
+ mesh=mesh, mode="train", loss_fn=ce_loss, microbatches=8,
152
+ )
153
+
154
+ # Drop mpmd_axis → same code runs under pure SPMD pjit
155
+ # Same model, same script, different mesh — no code changes.
156
+ ```
157
+
158
+ ### Deferred initialization — infer shapes at runtime
159
+
160
+ ```python
161
+ model = nn.Sequential(
162
+ nn.Linear(None, 256, rngs=rngs), # in_features inferred from first call
163
+ nn.ReLU(),
164
+ nn.Linear(256, 10, rngs=rngs),
165
+ )
166
+ y = model(jnp.zeros((8, 128))) # weight shapes resolved here
167
+ ```
168
+
169
+ ---
170
+
171
+ ## MPMD Runtime
172
+
173
+ SpectraX implements **true MPMD**: each physical rank compiles and executes
174
+ its own distinct JAX program. This is not SPMD-with-barriers — stages can
175
+ have different classes, different parameter shapes, and different I/O shapes.
176
+
177
+ ### `spx.run` — unified entry point
178
+
179
+ `spx.run` routes to SPMD (`pjit`) or MPMD (`sxcall`) based on the mesh:
180
+
181
+ ```python
182
+ spx.run(
183
+ model,
184
+ inputs=x, # microbatched along leading axis
185
+ targets=y,
186
+ mesh=mesh, # SpxMesh — mpmd_axis decides the path
187
+ mode="train", # "forward" | "train"
188
+ loss_fn=ce_loss,
189
+ microbatches=8,
190
+ )
191
+ ```
192
+
193
+ | Mesh type | What happens |
194
+ | ---------------- | -------------------------------------------------------- |
195
+ | `mpmd_axis=None` | Pure SPMD — `pjit` with FSDP/TP via logical axis rules |
196
+ | `mpmd_axis="pp"` | True MPMD — auto-split into stages, per-rank compilation |
197
+
198
+ ### Lower-level primitives
199
+
200
+ For full control, drop below `spx.run`:
201
+
202
+ | Primitive | Purpose |
203
+ | ----------------------------- | -------------------------------------------------------------------------------------------------------- |
204
+ | `sxcall` | Execute a `PipelineSequential` under a schedule — Python dispatch loop over pre-built per-rank callables |
205
+ | `sxjit` | Decorator: trace a function, split at `sxstage_iter` markers, compile one XLA executable per stage/rank |
206
+ | `sxgrad` / `sxvalue_and_grad` | Schedule-faithful gradients of an `sxjit` function |
207
+ | `treduce` | Schedule-driven microbatch reduction primitive — binds a body + schedule into the traced jaxpr |
208
+ | `sxstage_iter` | Marker primitive for stage boundaries inside `sxjit` |
209
+
210
+ ### Supported schedules
211
+
212
+ | Schedule | Type | Key trait |
213
+ | --------------- | ------- | ------------------------------------------------------------------------- |
214
+ | `GPipe` | Flat | All forwards, then all backwards. Simple, high memory. |
215
+ | `Std1F1B` | Flat | Standard 1-forward-1-backward. Peak memory `O(n_stages)`. |
216
+ | `ZeroBubbleH1` | Flat | Splits BWD into input-grad + weight-grad; weight-grad fills bubble slots. |
217
+ | `InterleavedH1` | Virtual | Each rank owns `v` non-contiguous stages. Bubble shrinks by `v`. |
218
+ | `DualPipeV` | Virtual | V-shaped bidirectional pipeline (DeepSeek-style). |
219
+ | `KimiK2` | Virtual | Interleaved with extra warmup (Moonshot K2 design). |
220
+
221
+ ### Heterogeneous stages
222
+
223
+ No same-shape constraint. Stages can be completely different:
224
+
225
+ ```python
226
+ model = PipelineSequential(
227
+ EmbedStage(vocab, d, rngs=rngs), # (B, S) int → (B, S, d)
228
+ BlockStage(d, rngs=rngs), # (B, S, d) → (B, S, d)
229
+ BlockStage(d, rngs=rngs), # (B, S, d) → (B, S, d)
230
+ HeadStage(d, vocab, rngs=rngs), # (B, S, d) → (B, S, vocab)
231
+ )
232
+ ```
233
+
234
+ Auto-splitting is available for homogeneous stacks: `spx.run` detects
235
+ `model.blocks: ModuleList` and slices it evenly across pipeline stages.
236
+
237
+ ---
238
+
239
+ ## Installation
240
+
241
+ ```bash
242
+ pip install spectrax-lib
243
+
244
+ # Optional extras
245
+ pip install "spectrax-lib[contrib]" # optax integration
246
+ pip install "spectrax-lib[cuda]" # CUDA jaxlib
247
+ pip install "spectrax-lib[tpu]" # TPU jaxlib
248
+ ```
249
+
250
+ From source:
251
+
252
+ ```bash
253
+ uv sync --extra dev --extra test --extra contrib
254
+ ```
255
+
256
+ Requires Python 3.11+ and JAX >= 0.9.2.
257
+
258
+ ---
259
+
260
+ ## Features
261
+
262
+ ### MPMD Pipeline Parallelism
263
+
264
+ True multi-program multi-data execution with per-rank compilation,
265
+ schedule-faithful dispatch, and heterogeneous stage support.
266
+
267
+ ### Module-aware JAX transforms
268
+
269
+ | Transform | What it does |
270
+ | -------------------------------------------------------------- | ------------------------------------------------------ |
271
+ | `spx.jit` | `mutable=` selector declares writable collections |
272
+ | `spx.grad` / `spx.value_and_grad` | `wrt=` selector picks the differentiated subset |
273
+ | `spx.vmap` | module states passed with `in_axes=None` automatically |
274
+ | `spx.scan` / `spx.remat_scan` | module-aware loops, optionally checkpointed |
275
+ | `spx.remat` | gradient checkpointing |
276
+ | `spx.cond` / `spx.switch` / `spx.while_loop` / `spx.fori_loop` | control flow over module state |
277
+
278
+ ### Selector DSL
279
+
280
+ One composable predicate for every "subset of the model" API:
281
+
282
+ ```python
283
+ spx.grad(loss, wrt="params") # by collection name
284
+ spx.grad(loss, wrt=nn.LoraParameter) # by Variable class
285
+ spx.grad(loss, wrt=spx.path_contains("attn")) # by path glob
286
+
287
+ sel = spx.select().at_instances_of(nn.Linear).of_type(spx.Parameter) - spx.path_contains("head")
288
+ trainable, frozen = sel.partition_state(model, state)
289
+ ```
290
+
291
+ ### Sharding & SPMD
292
+
293
+ Annotate variables with logical axis names and let the mesh decide:
294
+
295
+ ```python
296
+ w = spx.Parameter(
297
+ jnp.zeros((256, 256)),
298
+ sharding=spx.Sharding(("data", "model")),
299
+ axis_names=("in", "out"),
300
+ )
301
+ ```
302
+
303
+ ### LoRA Fine-Tuning
304
+
305
+ ```python
306
+ base = nn.Linear(768, 768, rngs=spx.Rngs(0))
307
+ model = nn.wrap_lora(base, rank=8, alpha=16, rngs=spx.Rngs(1))
308
+
309
+ @spx.jit
310
+ @spx.grad(wrt="lora")
311
+ def step(m, x, y):
312
+ return ((m(x) - y) ** 2).mean()
313
+ ```
314
+
315
+ ### FP8 Training
316
+
317
+ Delayed-scaling per-tensor FP8 with rolling amax history:
318
+
319
+ ```python
320
+ @spx.jit(mutable="fp8_meta")
321
+ def step(m, x, y):
322
+ def loss(m, x, y):
323
+ return ((m(x) - y) ** 2).mean()
324
+ return spx.grad(loss)(m, x, y)
325
+ ```
326
+
327
+ ### Explicit graph / state seam
328
+
329
+ ```python
330
+ gdef, state = spx.export(model) # immutable GraphDef + State dict
331
+ model2 = spx.bind(gdef, state) # reconstruct, skips __init__
332
+ spx.update(model, state) # in-place state patch
333
+ clone = spx.clone(model) # deep-copy via export+bind
334
+ ```
335
+
336
+ ### Inspection
337
+
338
+ ```python
339
+ spx.inspect.summary(model, jnp.zeros((1, 128)))
340
+ spx.inspect.count_params(model)
341
+ spx.inspect.count_bytes(model)
342
+ spx.inspect.tabulate(model, example_input)
343
+ ```
344
+
345
+ ---
346
+
347
+ ## Examples
348
+
349
+ See [`examples/`](examples/) for runnable scripts:
350
+
351
+ | Folder | Topic |
352
+ | --------------------------------------------------------------- | ------------------------------------------------------------------- |
353
+ | [`01_basics/`](examples/01_basics/) | Modules, training loops, export/bind, optimizers |
354
+ | [`02_implementation_guide/`](examples/02_implementation_guide/) | Llama 3, Qwen 2, GPT-2, ViT, custom transformer |
355
+ | [`03_transformations/`](examples/03_transformations/) | jit, grad, vmap, remat, scan, fori_loop |
356
+ | [`04_surgery/`](examples/04_surgery/) | Selectors, LoRA injection, FP8, freezing, swapping |
357
+ | [`05_shardings/`](examples/05_shardings/) | FSDP, TP, hybrid sharding, logical axis rules |
358
+ | [`06_spmd_scheduled/`](examples/06_spmd_scheduled/) | Pipeline runtime with all schedules |
359
+ | [`07_mpmd/`](examples/07_mpmd/) | Real MPMD pipeline via `spx.run` — train, forward, decode, 3-D mesh |
360
+
361
+ ```bash
362
+ python -m examples.01_basics.02_training_loop
363
+ python -m examples.02_implementation_guide.01_llama3
364
+ python -m examples.07_mpmd.01_train_homogeneous
365
+ ```
366
+
367
+ ---
368
+
369
+ ## Design
370
+
371
+ 1. **True MPMD first** — `sxcall` compiles one XLA program per physical rank.
372
+ Stages can differ in class, shape, and parameters. No SPMD-same-shape
373
+ constraint.
374
+ 2. **Unified runtime** — `spx.run` dispatches on the mesh. Same model, same
375
+ script; change the mesh and you change the parallelism strategy.
376
+ 3. **Schedule-faithful execution** — the dispatch loop walks the schedule grid
377
+ exactly as planned. No hidden reordering, no implicit fusing.
378
+ 4. **Modules are JAX pytrees** — flatten/unflatten through `export`/`bind`.
379
+ `jax.jit`, `jax.tree.map`, `jax.value_and_grad` work directly.
380
+ 5. **State lives in `Variable` cells** — `Parameter`, `Buffer`, custom
381
+ subclasses. Each tags its collection (`params`, `buffers`, `lora`,
382
+ `fp8_meta`, ...).
383
+ 6. **One filter DSL everywhere** — `Selector` serves `grad(wrt=...)`,
384
+ `jit(mutable=...)`, `Optimizer(wrt=...)`, `freeze(...)`,
385
+ `iter_variables(select=...)`.
386
+
387
+ ---
388
+
389
+ ## Benchmark
390
+
391
+ ```bash
392
+ python -m benchmarks.bench --cases all --device cpu
393
+ python -m benchmarks.bench --cases large --device tpu # 1.21B transformer
394
+ python -m benchmarks.llama_transforms_3way --preset small --device tpu --plots
395
+ ```
396
+
397
+ The CPU dispatch benchmark writes `benchmarks/results/latest.{json,md}` with
398
+ per-case `spectrax_ms / nnx_ms` ratios. On a tiny CPU dispatch-bound benchmark
399
+ (2-layer / d=64 / batch-2 transformer), SpectraX runs at **1.83×** the speed of
400
+ flax.nnx; on d=48 it hits **2.0×**. On compute-bound workloads (TPU 8B) the
401
+ Python gap shrinks but stays positive.
402
+
403
+ The plots below are from the **TPU small** three-way Llama transform benchmark,
404
+ which compares raw JAX, Flax NNX, and SpectraX across the transform surface used
405
+ by real training code: `jit`, `grad`, `value_and_grad`, `jvp`, `vjp`, `vmap`,
406
+ control-flow primitives, `scan`, and `remat`. These are runtime ratios against
407
+ raw JAX, so **1.0 means parity**, **lower is faster**, and **higher is slower**.
408
+
409
+ ![TPU small runtime ratio vs raw JAX](benchmarks/benchmarks-plots/spectrax_tpu_small_speedup_vs_jax.png)
410
+
411
+ Compile latency is tracked separately because first-call XLA costs can
412
+ dominate short benchmark cases and tell a different story from steady-state
413
+ runtime.
414
+
415
+ ![TPU small compile time](benchmarks/benchmarks-plots/spectrax_tpu_small_compile_time.png)
416
+
417
+ ---
418
+
419
+ ## Testing
420
+
421
+ ```bash
422
+ pytest tests/ -q
423
+ pytest tests/test_conformance.py
424
+ ```
425
+
426
+ ---
427
+
428
+ ## Status
429
+
430
+ `v0.0.1` — alpha. API may still move; pin the version if you depend on
431
+ behavioural stability.
432
+
433
+ ---
434
+
435
+ ## License
436
+
437
+ Apache-2.0.