spectrax-lib 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spectrax_lib-0.0.1/PKG-INFO +437 -0
- spectrax_lib-0.0.1/README.md +396 -0
- spectrax_lib-0.0.1/pyproject.toml +151 -0
- spectrax_lib-0.0.1/spectrax/__init__.py +204 -0
- spectrax_lib-0.0.1/spectrax/_version.py +20 -0
- spectrax_lib-0.0.1/spectrax/api.py +357 -0
- spectrax_lib-0.0.1/spectrax/contrib/__init__.py +23 -0
- spectrax_lib-0.0.1/spectrax/contrib/optimizer.py +377 -0
- spectrax_lib-0.0.1/spectrax/core/__init__.py +57 -0
- spectrax_lib-0.0.1/spectrax/core/_typing.py +144 -0
- spectrax_lib-0.0.1/spectrax/core/_weakcache.py +56 -0
- spectrax_lib-0.0.1/spectrax/core/containers.py +336 -0
- spectrax_lib-0.0.1/spectrax/core/context.py +187 -0
- spectrax_lib-0.0.1/spectrax/core/errors.py +89 -0
- spectrax_lib-0.0.1/spectrax/core/graph.py +858 -0
- spectrax_lib-0.0.1/spectrax/core/lazy_init.py +63 -0
- spectrax_lib-0.0.1/spectrax/core/module.py +975 -0
- spectrax_lib-0.0.1/spectrax/core/paths.py +128 -0
- spectrax_lib-0.0.1/spectrax/core/policy.py +114 -0
- spectrax_lib-0.0.1/spectrax/core/registry.py +102 -0
- spectrax_lib-0.0.1/spectrax/core/selector.py +553 -0
- spectrax_lib-0.0.1/spectrax/core/sharding.py +156 -0
- spectrax_lib-0.0.1/spectrax/core/stage_assignment.py +109 -0
- spectrax_lib-0.0.1/spectrax/core/state.py +361 -0
- spectrax_lib-0.0.1/spectrax/core/static.py +85 -0
- spectrax_lib-0.0.1/spectrax/core/variable.py +898 -0
- spectrax_lib-0.0.1/spectrax/functional/__init__.py +84 -0
- spectrax_lib-0.0.1/spectrax/functional/activation.py +132 -0
- spectrax_lib-0.0.1/spectrax/functional/attention.py +81 -0
- spectrax_lib-0.0.1/spectrax/functional/conv.py +129 -0
- spectrax_lib-0.0.1/spectrax/functional/dropout.py +56 -0
- spectrax_lib-0.0.1/spectrax/functional/linear.py +74 -0
- spectrax_lib-0.0.1/spectrax/functional/norm.py +83 -0
- spectrax_lib-0.0.1/spectrax/functional/pool.py +106 -0
- spectrax_lib-0.0.1/spectrax/functional/util.py +43 -0
- spectrax_lib-0.0.1/spectrax/hooks/__init__.py +30 -0
- spectrax_lib-0.0.1/spectrax/hooks/forward.py +48 -0
- spectrax_lib-0.0.1/spectrax/hooks/variable.py +60 -0
- spectrax_lib-0.0.1/spectrax/init/__init__.py +41 -0
- spectrax_lib-0.0.1/spectrax/init/constant.py +47 -0
- spectrax_lib-0.0.1/spectrax/init/kaiming.py +75 -0
- spectrax_lib-0.0.1/spectrax/init/normal.py +49 -0
- spectrax_lib-0.0.1/spectrax/init/orthogonal.py +54 -0
- spectrax_lib-0.0.1/spectrax/init/uniform.py +31 -0
- spectrax_lib-0.0.1/spectrax/init/xavier.py +82 -0
- spectrax_lib-0.0.1/spectrax/inspect/__init__.py +35 -0
- spectrax_lib-0.0.1/spectrax/inspect/counting.py +31 -0
- spectrax_lib-0.0.1/spectrax/inspect/display.py +37 -0
- spectrax_lib-0.0.1/spectrax/inspect/repr.py +107 -0
- spectrax_lib-0.0.1/spectrax/inspect/summary.py +83 -0
- spectrax_lib-0.0.1/spectrax/inspect/tabulate.py +142 -0
- spectrax_lib-0.0.1/spectrax/inspect/tree.py +47 -0
- spectrax_lib-0.0.1/spectrax/lint.py +66 -0
- spectrax_lib-0.0.1/spectrax/nn/__init__.py +125 -0
- spectrax_lib-0.0.1/spectrax/nn/activation.py +95 -0
- spectrax_lib-0.0.1/spectrax/nn/attention.py +277 -0
- spectrax_lib-0.0.1/spectrax/nn/conv.py +306 -0
- spectrax_lib-0.0.1/spectrax/nn/dense.py +202 -0
- spectrax_lib-0.0.1/spectrax/nn/dropout.py +97 -0
- spectrax_lib-0.0.1/spectrax/nn/embed.py +101 -0
- spectrax_lib-0.0.1/spectrax/nn/fp8.py +724 -0
- spectrax_lib-0.0.1/spectrax/nn/identity.py +36 -0
- spectrax_lib-0.0.1/spectrax/nn/linear.py +188 -0
- spectrax_lib-0.0.1/spectrax/nn/lora.py +356 -0
- spectrax_lib-0.0.1/spectrax/nn/mlp.py +112 -0
- spectrax_lib-0.0.1/spectrax/nn/norm.py +358 -0
- spectrax_lib-0.0.1/spectrax/nn/pipeline_sequential.py +141 -0
- spectrax_lib-0.0.1/spectrax/nn/pool.py +170 -0
- spectrax_lib-0.0.1/spectrax/nn/recurrent.py +556 -0
- spectrax_lib-0.0.1/spectrax/py.typed +0 -0
- spectrax_lib-0.0.1/spectrax/rng/__init__.py +26 -0
- spectrax_lib-0.0.1/spectrax/rng/rngs.py +258 -0
- spectrax_lib-0.0.1/spectrax/rng/seed.py +81 -0
- spectrax_lib-0.0.1/spectrax/runtime/__init__.py +81 -0
- spectrax_lib-0.0.1/spectrax/runtime/mpmd/__init__.py +95 -0
- spectrax_lib-0.0.1/spectrax/runtime/mpmd/compiler.py +334 -0
- spectrax_lib-0.0.1/spectrax/runtime/mpmd/markers.py +447 -0
- spectrax_lib-0.0.1/spectrax/runtime/mpmd/per_rank.py +341 -0
- spectrax_lib-0.0.1/spectrax/runtime/mpmd/pscan_compiler.py +977 -0
- spectrax_lib-0.0.1/spectrax/runtime/mpmd/runtime.py +3064 -0
- spectrax_lib-0.0.1/spectrax/runtime/mpmd/treduce.py +483 -0
- spectrax_lib-0.0.1/spectrax/runtime/primitives/__init__.py +25 -0
- spectrax_lib-0.0.1/spectrax/runtime/primitives/boundary.py +77 -0
- spectrax_lib-0.0.1/spectrax/runtime/primitives/split.py +272 -0
- spectrax_lib-0.0.1/spectrax/runtime/schedules/__init__.py +46 -0
- spectrax_lib-0.0.1/spectrax/runtime/schedules/base.py +213 -0
- spectrax_lib-0.0.1/spectrax/runtime/schedules/dualpipe.py +260 -0
- spectrax_lib-0.0.1/spectrax/runtime/schedules/fusion.py +123 -0
- spectrax_lib-0.0.1/spectrax/runtime/schedules/gpipe.py +86 -0
- spectrax_lib-0.0.1/spectrax/runtime/schedules/interleaved.py +315 -0
- spectrax_lib-0.0.1/spectrax/runtime/schedules/one_f_one_b.py +181 -0
- spectrax_lib-0.0.1/spectrax/runtime/schedules/zero_bubble.py +115 -0
- spectrax_lib-0.0.1/spectrax/runtime/spmd/__init__.py +33 -0
- spectrax_lib-0.0.1/spectrax/runtime/spmd/api.py +998 -0
- spectrax_lib-0.0.1/spectrax/runtime/spmd/hybrid.py +360 -0
- spectrax_lib-0.0.1/spectrax/runtime/spmd/runtime.py +351 -0
- spectrax_lib-0.0.1/spectrax/runtime/spmd/shard_map.py +554 -0
- spectrax_lib-0.0.1/spectrax/runtime/types/__init__.py +29 -0
- spectrax_lib-0.0.1/spectrax/runtime/types/array.py +332 -0
- spectrax_lib-0.0.1/spectrax/runtime/types/mesh.py +229 -0
- spectrax_lib-0.0.1/spectrax/runtime/types/stage.py +44 -0
- spectrax_lib-0.0.1/spectrax/sharding/__init__.py +56 -0
- spectrax_lib-0.0.1/spectrax/sharding/logical.py +59 -0
- spectrax_lib-0.0.1/spectrax/sharding/mesh.py +706 -0
- spectrax_lib-0.0.1/spectrax/sharding/partition.py +184 -0
- spectrax_lib-0.0.1/spectrax/transforms/__init__.py +55 -0
- spectrax_lib-0.0.1/spectrax/transforms/control_flow.py +393 -0
- spectrax_lib-0.0.1/spectrax/transforms/eval_shape.py +47 -0
- spectrax_lib-0.0.1/spectrax/transforms/grad.py +683 -0
- spectrax_lib-0.0.1/spectrax/transforms/jit.py +343 -0
- spectrax_lib-0.0.1/spectrax/transforms/remat.py +254 -0
- spectrax_lib-0.0.1/spectrax/transforms/rng_axes.py +117 -0
- spectrax_lib-0.0.1/spectrax/transforms/scan.py +210 -0
- spectrax_lib-0.0.1/spectrax/transforms/split_merge.py +739 -0
- spectrax_lib-0.0.1/spectrax/transforms/vmap.py +157 -0
- spectrax_lib-0.0.1/spectrax/typing.py +54 -0
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: spectrax-lib
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: SpectraX: a JAX-only neural-network library with a PyTorch-shaped eager surface and Flax-NNX-style graph/state underneath.
|
|
5
|
+
Keywords: Deep Learning,Machine Learning,Neural Networks,JAX,XLA
|
|
6
|
+
Author: Erfan Zare Chavoshi
|
|
7
|
+
Author-email: Erfan Zare Chavoshi <Erfanzare810@gmail.com>
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Requires-Dist: jax>=0.9.2
|
|
17
|
+
Requires-Dist: jaxlib>=0.9.2
|
|
18
|
+
Requires-Dist: numpy>=1.26
|
|
19
|
+
Requires-Dist: treescope>=0.1.7
|
|
20
|
+
Requires-Dist: optax>=0.2.8 ; extra == 'contrib'
|
|
21
|
+
Requires-Dist: jax[cuda13]>=0.9.2 ; extra == 'cuda'
|
|
22
|
+
Requires-Dist: ruff>=0.6 ; extra == 'dev'
|
|
23
|
+
Requires-Dist: pytest>=7 ; extra == 'dev'
|
|
24
|
+
Requires-Dist: sphinx>=7 ; extra == 'docs'
|
|
25
|
+
Requires-Dist: sphinx-book-theme>=1.1 ; extra == 'docs'
|
|
26
|
+
Requires-Dist: sphinx-design>=0.6 ; extra == 'docs'
|
|
27
|
+
Requires-Dist: sphinx-autodoc-typehints>=2 ; extra == 'docs'
|
|
28
|
+
Requires-Dist: myst-nb>=1 ; extra == 'docs'
|
|
29
|
+
Requires-Dist: pytest>=7 ; extra == 'test'
|
|
30
|
+
Requires-Dist: jax[tpu]>=0.9.2 ; extra == 'tpu'
|
|
31
|
+
Requires-Python: >=3.11, <3.14
|
|
32
|
+
Project-URL: Homepage, https://github.com/erfanzar/spectrax
|
|
33
|
+
Project-URL: Repository, https://github.com/erfanzar/spectrax
|
|
34
|
+
Provides-Extra: contrib
|
|
35
|
+
Provides-Extra: cuda
|
|
36
|
+
Provides-Extra: dev
|
|
37
|
+
Provides-Extra: docs
|
|
38
|
+
Provides-Extra: test
|
|
39
|
+
Provides-Extra: tpu
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# SpectraX
|
|
43
|
+
|
|
44
|
+
**True MPMD pipeline parallelism for JAX — with an eager module API.**
|
|
45
|
+
|
|
46
|
+
[**Quick Start**](#quick-start) |
|
|
47
|
+
[**Installation**](#installation) |
|
|
48
|
+
[**MPMD Runtime**](#mpmd-runtime) |
|
|
49
|
+
[**Benchmark**](#benchmark) |
|
|
50
|
+
[**Examples**](#examples) |
|
|
51
|
+
[**Docs**](docs/)
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
SpectraX is a JAX-native neural network library built around **true MPMD
|
|
56
|
+
pipeline parallelism**. Each physical rank compiles and runs its own XLA
|
|
57
|
+
program — no shared `shard_map` HLO, no SPMD-same-shape constraint.
|
|
58
|
+
Heterogeneous stages (embed → blocks → head), multiple schedules
|
|
59
|
+
(GPipe, 1F1B, ZeroBubble, Interleaved, DualPipe), and a unified
|
|
60
|
+
`spx.run()` entry point that dispatches to SPMD or MPMD from the same
|
|
61
|
+
training script.
|
|
62
|
+
|
|
63
|
+
The module API is eager and debuggable — subclass `Module`, override
|
|
64
|
+
`forward`, call `model(x)` — but every `Module` is a JAX pytree, so
|
|
65
|
+
`jax.jit`, `jax.grad`, and `jax.tree.map` work directly.
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## Why SpectraX?
|
|
70
|
+
|
|
71
|
+
| Capability | SpectraX | JAX + manual | other JAX frameworks |
|
|
72
|
+
| ------------------------ | --------------------------------------- | ------------------- | -------------------------- |
|
|
73
|
+
| **True MPMD** | built-in (`sxcall`, `sxjit`) | hand-rolled | SPMD-only |
|
|
74
|
+
| **Heterogeneous stages** | native (different class/shape per rank) | fragile | not supported |
|
|
75
|
+
| **Pipeline schedules** | 9 schedules (GPipe→DualPipeV) | hand-rolled | limited |
|
|
76
|
+
| **Unified runtime** | `spx.run(mesh)` → SPMD or MPMD | separate code paths | separate APIs |
|
|
77
|
+
| **Eager modules** | `model(x)` + pytree-native | functional only | functional or ref-tracking |
|
|
78
|
+
| **Dispatch overhead** | ~150 µs | ~N/A | ~300–2000 µs |
|
|
79
|
+
|
|
80
|
+
*Dispatch overhead measured on a tiny 2-layer CPU transformer. See [benchmark](#benchmark).*
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
## Quick Start
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install spectrax-lib
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Single-device eager training
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
import jax.numpy as jnp
|
|
94
|
+
import spectrax as spx
|
|
95
|
+
from spectrax import nn
|
|
96
|
+
|
|
97
|
+
class MLP(spx.Module):
|
|
98
|
+
def __init__(self, d, h, o, *, rngs):
|
|
99
|
+
super().__init__()
|
|
100
|
+
self.fc1 = nn.Linear(d, h, rngs=rngs)
|
|
101
|
+
self.fc2 = nn.Linear(h, o, rngs=rngs)
|
|
102
|
+
|
|
103
|
+
def forward(self, x):
|
|
104
|
+
return self.fc2(nn.gelu(self.fc1(x)))
|
|
105
|
+
|
|
106
|
+
model = MLP(16, 64, 4, rngs=spx.Rngs(0))
|
|
107
|
+
|
|
108
|
+
@spx.jit
|
|
109
|
+
@spx.value_and_grad
|
|
110
|
+
def loss_fn(m, x, y):
|
|
111
|
+
return ((m(x) - y) ** 2).mean()
|
|
112
|
+
|
|
113
|
+
loss, grads = loss_fn(model, jnp.ones((8, 16)), jnp.zeros((8, 4)))
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### Marker-based MPMD — split a function into per-rank programs
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from spectrax.runtime.mpmd import sxjit, sxstage_iter
|
|
120
|
+
from spectrax.pipeline import Std1F1B
|
|
121
|
+
|
|
122
|
+
# Define a multi-stage forward with explicit stage boundaries
|
|
123
|
+
@sxjit(schedule=Std1F1B(microbatches=8))
|
|
124
|
+
def pipeline_fwd(model, x):
|
|
125
|
+
x = model.embed(x) # stage 0
|
|
126
|
+
x = sxstage_iter(x) # boundary — split here
|
|
127
|
+
x = model.blocks[0](x) # stage 1
|
|
128
|
+
x = sxstage_iter(x) # boundary — split here
|
|
129
|
+
x = model.blocks[1](x) # stage 2
|
|
130
|
+
x = sxstage_iter(x) # boundary — split here
|
|
131
|
+
return model.head(x) # stage 3
|
|
132
|
+
|
|
133
|
+
# sxjit traces the function, splits the jaxpr at sxstage_iter markers,
|
|
134
|
+
# and compiles one XLA executable per stage/rank. Each rank only sees
|
|
135
|
+
# its own sub-graph — true MPMD, not SPMD.
|
|
136
|
+
output = pipeline_fwd(model, x)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
### MPMD pipeline training — one call, multiple devices
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
from spectrax.sharding import logical_axis_rules
|
|
143
|
+
|
|
144
|
+
# Create a 4-stage pipeline mesh
|
|
145
|
+
mesh = spx.create_mesh(axis_dims=(2, 1, -1, 1, 1, 1), mpmd_axis="pp")
|
|
146
|
+
|
|
147
|
+
with logical_axis_rules(FSDP_TP_RULES), mesh:
|
|
148
|
+
# MPMD: each rank gets its own compiled stage
|
|
149
|
+
loss, grads = spx.run(
|
|
150
|
+
model, inputs=x, targets=y,
|
|
151
|
+
mesh=mesh, mode="train", loss_fn=ce_loss, microbatches=8,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
# Drop mpmd_axis → same code runs under pure SPMD pjit
|
|
155
|
+
# Same model, same script, different mesh — no code changes.
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
### Deferred initialization — infer shapes at runtime
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
model = nn.Sequential(
|
|
162
|
+
nn.Linear(None, 256, rngs=rngs), # in_features inferred from first call
|
|
163
|
+
nn.ReLU(),
|
|
164
|
+
nn.Linear(256, 10, rngs=rngs),
|
|
165
|
+
)
|
|
166
|
+
y = model(jnp.zeros((8, 128))) # weight shapes resolved here
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## MPMD Runtime
|
|
172
|
+
|
|
173
|
+
SpectraX implements **true MPMD**: each physical rank compiles and executes
|
|
174
|
+
its own distinct JAX program. This is not SPMD-with-barriers — stages can
|
|
175
|
+
have different classes, different parameter shapes, and different I/O shapes.
|
|
176
|
+
|
|
177
|
+
### `spx.run` — unified entry point
|
|
178
|
+
|
|
179
|
+
`spx.run` routes to SPMD (`pjit`) or MPMD (`sxcall`) based on the mesh:
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
spx.run(
|
|
183
|
+
model,
|
|
184
|
+
inputs=x, # microbatched along leading axis
|
|
185
|
+
targets=y,
|
|
186
|
+
mesh=mesh, # SpxMesh — mpmd_axis decides the path
|
|
187
|
+
mode="train", # "forward" | "train"
|
|
188
|
+
loss_fn=ce_loss,
|
|
189
|
+
microbatches=8,
|
|
190
|
+
)
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
| Mesh type | What happens |
|
|
194
|
+
| ---------------- | -------------------------------------------------------- |
|
|
195
|
+
| `mpmd_axis=None` | Pure SPMD — `pjit` with FSDP/TP via logical axis rules |
|
|
196
|
+
| `mpmd_axis="pp"` | True MPMD — auto-split into stages, per-rank compilation |
|
|
197
|
+
|
|
198
|
+
### Lower-level primitives
|
|
199
|
+
|
|
200
|
+
For full control, drop below `spx.run`:
|
|
201
|
+
|
|
202
|
+
| Primitive | Purpose |
|
|
203
|
+
| ----------------------------- | -------------------------------------------------------------------------------------------------------- |
|
|
204
|
+
| `sxcall` | Execute a `PipelineSequential` under a schedule — Python dispatch loop over pre-built per-rank callables |
|
|
205
|
+
| `sxjit` | Decorator: trace a function, split at `sxstage_iter` markers, compile one XLA executable per stage/rank |
|
|
206
|
+
| `sxgrad` / `sxvalue_and_grad` | Schedule-faithful gradients of an `sxjit` function |
|
|
207
|
+
| `treduce` | Schedule-driven microbatch reduction primitive — binds a body + schedule into the traced jaxpr |
|
|
208
|
+
| `sxstage_iter` | Marker primitive for stage boundaries inside `sxjit` |
|
|
209
|
+
|
|
210
|
+
### Supported schedules
|
|
211
|
+
|
|
212
|
+
| Schedule | Type | Key trait |
|
|
213
|
+
| --------------- | ------- | ------------------------------------------------------------------------- |
|
|
214
|
+
| `GPipe` | Flat | All forwards, then all backwards. Simple, high memory. |
|
|
215
|
+
| `Std1F1B` | Flat | Standard 1-forward-1-backward. Peak memory `O(n_stages)`. |
|
|
216
|
+
| `ZeroBubbleH1` | Flat | Splits BWD into input-grad + weight-grad; weight-grad fills bubble slots. |
|
|
217
|
+
| `InterleavedH1` | Virtual | Each rank owns `v` non-contiguous stages. Bubble shrinks by `v`. |
|
|
218
|
+
| `DualPipeV` | Virtual | V-shaped bidirectional pipeline (DeepSeek-style). |
|
|
219
|
+
| `KimiK2` | Virtual | Interleaved with extra warmup (Moonshot K2 design). |
|
|
220
|
+
|
|
221
|
+
### Heterogeneous stages
|
|
222
|
+
|
|
223
|
+
No same-shape constraint. Stages can be completely different:
|
|
224
|
+
|
|
225
|
+
```python
|
|
226
|
+
model = PipelineSequential(
|
|
227
|
+
EmbedStage(vocab, d, rngs=rngs), # (B, S) int → (B, S, d)
|
|
228
|
+
BlockStage(d, rngs=rngs), # (B, S, d) → (B, S, d)
|
|
229
|
+
BlockStage(d, rngs=rngs), # (B, S, d) → (B, S, d)
|
|
230
|
+
HeadStage(d, vocab, rngs=rngs), # (B, S, d) → (B, S, vocab)
|
|
231
|
+
)
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
Auto-splitting is available for homogeneous stacks: `spx.run` detects
|
|
235
|
+
`model.blocks: ModuleList` and slices it evenly across pipeline stages.
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
## Installation
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
pip install spectrax-lib
|
|
243
|
+
|
|
244
|
+
# Optional extras
|
|
245
|
+
pip install "spectrax-lib[contrib]" # optax integration
|
|
246
|
+
pip install "spectrax-lib[cuda]" # CUDA jaxlib
|
|
247
|
+
pip install "spectrax-lib[tpu]" # TPU jaxlib
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
From source:
|
|
251
|
+
|
|
252
|
+
```bash
|
|
253
|
+
uv sync --extra dev --extra test --extra contrib
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
Requires Python 3.11+ and JAX >= 0.9.2.
|
|
257
|
+
|
|
258
|
+
---
|
|
259
|
+
|
|
260
|
+
## Features
|
|
261
|
+
|
|
262
|
+
### MPMD Pipeline Parallelism
|
|
263
|
+
|
|
264
|
+
True multi-program multi-data execution with per-rank compilation,
|
|
265
|
+
schedule-faithful dispatch, and heterogeneous stage support.
|
|
266
|
+
|
|
267
|
+
### Module-aware JAX transforms
|
|
268
|
+
|
|
269
|
+
| Transform | What it does |
|
|
270
|
+
| -------------------------------------------------------------- | ------------------------------------------------------ |
|
|
271
|
+
| `spx.jit` | `mutable=` selector declares writable collections |
|
|
272
|
+
| `spx.grad` / `spx.value_and_grad` | `wrt=` selector picks the differentiated subset |
|
|
273
|
+
| `spx.vmap` | module states passed with `in_axes=None` automatically |
|
|
274
|
+
| `spx.scan` / `spx.remat_scan` | module-aware loops, optionally checkpointed |
|
|
275
|
+
| `spx.remat` | gradient checkpointing |
|
|
276
|
+
| `spx.cond` / `spx.switch` / `spx.while_loop` / `spx.fori_loop` | control flow over module state |
|
|
277
|
+
|
|
278
|
+
### Selector DSL
|
|
279
|
+
|
|
280
|
+
One composable predicate for every "subset of the model" API:
|
|
281
|
+
|
|
282
|
+
```python
|
|
283
|
+
spx.grad(loss, wrt="params") # by collection name
|
|
284
|
+
spx.grad(loss, wrt=nn.LoraParameter) # by Variable class
|
|
285
|
+
spx.grad(loss, wrt=spx.path_contains("attn")) # by path glob
|
|
286
|
+
|
|
287
|
+
sel = spx.select().at_instances_of(nn.Linear).of_type(spx.Parameter) - spx.path_contains("head")
|
|
288
|
+
trainable, frozen = sel.partition_state(model, state)
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
### Sharding & SPMD
|
|
292
|
+
|
|
293
|
+
Annotate variables with logical axis names and let the mesh decide:
|
|
294
|
+
|
|
295
|
+
```python
|
|
296
|
+
w = spx.Parameter(
|
|
297
|
+
jnp.zeros((256, 256)),
|
|
298
|
+
sharding=spx.Sharding(("data", "model")),
|
|
299
|
+
axis_names=("in", "out"),
|
|
300
|
+
)
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
### LoRA Fine-Tuning
|
|
304
|
+
|
|
305
|
+
```python
|
|
306
|
+
base = nn.Linear(768, 768, rngs=spx.Rngs(0))
|
|
307
|
+
model = nn.wrap_lora(base, rank=8, alpha=16, rngs=spx.Rngs(1))
|
|
308
|
+
|
|
309
|
+
@spx.jit
|
|
310
|
+
@spx.grad(wrt="lora")
|
|
311
|
+
def step(m, x, y):
|
|
312
|
+
return ((m(x) - y) ** 2).mean()
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
### FP8 Training
|
|
316
|
+
|
|
317
|
+
Delayed-scaling per-tensor FP8 with rolling amax history:
|
|
318
|
+
|
|
319
|
+
```python
|
|
320
|
+
@spx.jit(mutable="fp8_meta")
|
|
321
|
+
def step(m, x, y):
|
|
322
|
+
def loss(m, x, y):
|
|
323
|
+
return ((m(x) - y) ** 2).mean()
|
|
324
|
+
return spx.grad(loss)(m, x, y)
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
### Explicit graph / state seam
|
|
328
|
+
|
|
329
|
+
```python
|
|
330
|
+
gdef, state = spx.export(model) # immutable GraphDef + State dict
|
|
331
|
+
model2 = spx.bind(gdef, state) # reconstruct, skips __init__
|
|
332
|
+
spx.update(model, state) # in-place state patch
|
|
333
|
+
clone = spx.clone(model) # deep-copy via export+bind
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
### Inspection
|
|
337
|
+
|
|
338
|
+
```python
|
|
339
|
+
spx.inspect.summary(model, jnp.zeros((1, 128)))
|
|
340
|
+
spx.inspect.count_params(model)
|
|
341
|
+
spx.inspect.count_bytes(model)
|
|
342
|
+
spx.inspect.tabulate(model, example_input)
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
---
|
|
346
|
+
|
|
347
|
+
## Examples
|
|
348
|
+
|
|
349
|
+
See [`examples/`](examples/) for runnable scripts:
|
|
350
|
+
|
|
351
|
+
| Folder | Topic |
|
|
352
|
+
| --------------------------------------------------------------- | ------------------------------------------------------------------- |
|
|
353
|
+
| [`01_basics/`](examples/01_basics/) | Modules, training loops, export/bind, optimizers |
|
|
354
|
+
| [`02_implementation_guide/`](examples/02_implementation_guide/) | Llama 3, Qwen 2, GPT-2, ViT, custom transformer |
|
|
355
|
+
| [`03_transformations/`](examples/03_transformations/) | jit, grad, vmap, remat, scan, fori_loop |
|
|
356
|
+
| [`04_surgery/`](examples/04_surgery/) | Selectors, LoRA injection, FP8, freezing, swapping |
|
|
357
|
+
| [`05_shardings/`](examples/05_shardings/) | FSDP, TP, hybrid sharding, logical axis rules |
|
|
358
|
+
| [`06_spmd_scheduled/`](examples/06_spmd_scheduled/) | Pipeline runtime with all schedules |
|
|
359
|
+
| [`07_mpmd/`](examples/07_mpmd/) | Real MPMD pipeline via `spx.run` — train, forward, decode, 3-D mesh |
|
|
360
|
+
|
|
361
|
+
```bash
|
|
362
|
+
python -m examples.01_basics.02_training_loop
|
|
363
|
+
python -m examples.02_implementation_guide.01_llama3
|
|
364
|
+
python -m examples.07_mpmd.01_train_homogeneous
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
---
|
|
368
|
+
|
|
369
|
+
## Design
|
|
370
|
+
|
|
371
|
+
1. **True MPMD first** — `sxcall` compiles one XLA program per physical rank.
|
|
372
|
+
Stages can differ in class, shape, and parameters. No SPMD-same-shape
|
|
373
|
+
constraint.
|
|
374
|
+
2. **Unified runtime** — `spx.run` dispatches on the mesh. Same model, same
|
|
375
|
+
script; change the mesh and you change the parallelism strategy.
|
|
376
|
+
3. **Schedule-faithful execution** — the dispatch loop walks the schedule grid
|
|
377
|
+
exactly as planned. No hidden reordering, no implicit fusing.
|
|
378
|
+
4. **Modules are JAX pytrees** — flatten/unflatten through `export`/`bind`.
|
|
379
|
+
`jax.jit`, `jax.tree.map`, `jax.value_and_grad` work directly.
|
|
380
|
+
5. **State lives in `Variable` cells** — `Parameter`, `Buffer`, custom
|
|
381
|
+
subclasses. Each tags its collection (`params`, `buffers`, `lora`,
|
|
382
|
+
`fp8_meta`, ...).
|
|
383
|
+
6. **One filter DSL everywhere** — `Selector` serves `grad(wrt=...)`,
|
|
384
|
+
`jit(mutable=...)`, `Optimizer(wrt=...)`, `freeze(...)`,
|
|
385
|
+
`iter_variables(select=...)`.
|
|
386
|
+
|
|
387
|
+
---
|
|
388
|
+
|
|
389
|
+
## Benchmark
|
|
390
|
+
|
|
391
|
+
```bash
|
|
392
|
+
python -m benchmarks.bench --cases all --device cpu
|
|
393
|
+
python -m benchmarks.bench --cases large --device tpu # 1.21B transformer
|
|
394
|
+
python -m benchmarks.llama_transforms_3way --preset small --device tpu --plots
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
The CPU dispatch benchmark writes `benchmarks/results/latest.{json,md}` with
|
|
398
|
+
per-case `spectrax_ms / nnx_ms` ratios. On a tiny CPU dispatch-bound benchmark
|
|
399
|
+
(2-layer / d=64 / batch-2 transformer), SpectraX runs at **1.83×** the speed of
|
|
400
|
+
flax.nnx; on d=48 it hits **2.0×**. On compute-bound workloads (TPU 8B) the
|
|
401
|
+
Python gap shrinks but stays positive.
|
|
402
|
+
|
|
403
|
+
The plots below are from the **TPU small** three-way Llama transform benchmark,
|
|
404
|
+
which compares raw JAX, Flax NNX, and SpectraX across the transform surface used
|
|
405
|
+
by real training code: `jit`, `grad`, `value_and_grad`, `jvp`, `vjp`, `vmap`,
|
|
406
|
+
control-flow primitives, `scan`, and `remat`. These are runtime ratios against
|
|
407
|
+
raw JAX, so **1.0 means parity**, **lower is faster**, and **higher is slower**.
|
|
408
|
+
|
|
409
|
+

|
|
410
|
+
|
|
411
|
+
Compile latency is tracked separately because first-call XLA costs can
|
|
412
|
+
dominate short benchmark cases and tell a different story from steady-state
|
|
413
|
+
runtime.
|
|
414
|
+
|
|
415
|
+

|
|
416
|
+
|
|
417
|
+
---
|
|
418
|
+
|
|
419
|
+
## Testing
|
|
420
|
+
|
|
421
|
+
```bash
|
|
422
|
+
pytest tests/ -q
|
|
423
|
+
pytest tests/test_conformance.py
|
|
424
|
+
```
|
|
425
|
+
|
|
426
|
+
---
|
|
427
|
+
|
|
428
|
+
## Status
|
|
429
|
+
|
|
430
|
+
`v0.0.1` — alpha. API may still move; pin the version if you depend on
|
|
431
|
+
behavioural stability.
|
|
432
|
+
|
|
433
|
+
---
|
|
434
|
+
|
|
435
|
+
## License
|
|
436
|
+
|
|
437
|
+
Apache-2.0.
|