spectrax-lib 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spectrax/__init__.py +204 -0
- spectrax/_version.py +20 -0
- spectrax/api.py +357 -0
- spectrax/contrib/__init__.py +23 -0
- spectrax/contrib/optimizer.py +377 -0
- spectrax/core/__init__.py +57 -0
- spectrax/core/_typing.py +144 -0
- spectrax/core/_weakcache.py +56 -0
- spectrax/core/containers.py +336 -0
- spectrax/core/context.py +187 -0
- spectrax/core/errors.py +89 -0
- spectrax/core/graph.py +858 -0
- spectrax/core/lazy_init.py +63 -0
- spectrax/core/module.py +975 -0
- spectrax/core/paths.py +128 -0
- spectrax/core/policy.py +114 -0
- spectrax/core/registry.py +102 -0
- spectrax/core/selector.py +553 -0
- spectrax/core/sharding.py +156 -0
- spectrax/core/stage_assignment.py +109 -0
- spectrax/core/state.py +361 -0
- spectrax/core/static.py +85 -0
- spectrax/core/variable.py +898 -0
- spectrax/functional/__init__.py +84 -0
- spectrax/functional/activation.py +132 -0
- spectrax/functional/attention.py +81 -0
- spectrax/functional/conv.py +129 -0
- spectrax/functional/dropout.py +56 -0
- spectrax/functional/linear.py +74 -0
- spectrax/functional/norm.py +83 -0
- spectrax/functional/pool.py +106 -0
- spectrax/functional/util.py +43 -0
- spectrax/hooks/__init__.py +30 -0
- spectrax/hooks/forward.py +48 -0
- spectrax/hooks/variable.py +60 -0
- spectrax/init/__init__.py +41 -0
- spectrax/init/constant.py +47 -0
- spectrax/init/kaiming.py +75 -0
- spectrax/init/normal.py +49 -0
- spectrax/init/orthogonal.py +54 -0
- spectrax/init/uniform.py +31 -0
- spectrax/init/xavier.py +82 -0
- spectrax/inspect/__init__.py +35 -0
- spectrax/inspect/counting.py +31 -0
- spectrax/inspect/display.py +37 -0
- spectrax/inspect/repr.py +107 -0
- spectrax/inspect/summary.py +83 -0
- spectrax/inspect/tabulate.py +142 -0
- spectrax/inspect/tree.py +47 -0
- spectrax/lint.py +66 -0
- spectrax/nn/__init__.py +125 -0
- spectrax/nn/activation.py +95 -0
- spectrax/nn/attention.py +277 -0
- spectrax/nn/conv.py +306 -0
- spectrax/nn/dense.py +202 -0
- spectrax/nn/dropout.py +97 -0
- spectrax/nn/embed.py +101 -0
- spectrax/nn/fp8.py +724 -0
- spectrax/nn/identity.py +36 -0
- spectrax/nn/linear.py +188 -0
- spectrax/nn/lora.py +356 -0
- spectrax/nn/mlp.py +112 -0
- spectrax/nn/norm.py +358 -0
- spectrax/nn/pipeline_sequential.py +141 -0
- spectrax/nn/pool.py +170 -0
- spectrax/nn/recurrent.py +556 -0
- spectrax/py.typed +0 -0
- spectrax/rng/__init__.py +26 -0
- spectrax/rng/rngs.py +258 -0
- spectrax/rng/seed.py +81 -0
- spectrax/runtime/__init__.py +81 -0
- spectrax/runtime/mpmd/__init__.py +95 -0
- spectrax/runtime/mpmd/compiler.py +334 -0
- spectrax/runtime/mpmd/markers.py +447 -0
- spectrax/runtime/mpmd/per_rank.py +341 -0
- spectrax/runtime/mpmd/pscan_compiler.py +977 -0
- spectrax/runtime/mpmd/runtime.py +3064 -0
- spectrax/runtime/mpmd/treduce.py +483 -0
- spectrax/runtime/primitives/__init__.py +25 -0
- spectrax/runtime/primitives/boundary.py +77 -0
- spectrax/runtime/primitives/split.py +272 -0
- spectrax/runtime/schedules/__init__.py +46 -0
- spectrax/runtime/schedules/base.py +213 -0
- spectrax/runtime/schedules/dualpipe.py +260 -0
- spectrax/runtime/schedules/fusion.py +123 -0
- spectrax/runtime/schedules/gpipe.py +86 -0
- spectrax/runtime/schedules/interleaved.py +315 -0
- spectrax/runtime/schedules/one_f_one_b.py +181 -0
- spectrax/runtime/schedules/zero_bubble.py +115 -0
- spectrax/runtime/spmd/__init__.py +33 -0
- spectrax/runtime/spmd/api.py +998 -0
- spectrax/runtime/spmd/hybrid.py +360 -0
- spectrax/runtime/spmd/runtime.py +351 -0
- spectrax/runtime/spmd/shard_map.py +554 -0
- spectrax/runtime/types/__init__.py +29 -0
- spectrax/runtime/types/array.py +332 -0
- spectrax/runtime/types/mesh.py +229 -0
- spectrax/runtime/types/stage.py +44 -0
- spectrax/sharding/__init__.py +56 -0
- spectrax/sharding/logical.py +59 -0
- spectrax/sharding/mesh.py +706 -0
- spectrax/sharding/partition.py +184 -0
- spectrax/transforms/__init__.py +55 -0
- spectrax/transforms/control_flow.py +393 -0
- spectrax/transforms/eval_shape.py +47 -0
- spectrax/transforms/grad.py +683 -0
- spectrax/transforms/jit.py +343 -0
- spectrax/transforms/remat.py +254 -0
- spectrax/transforms/rng_axes.py +117 -0
- spectrax/transforms/scan.py +210 -0
- spectrax/transforms/split_merge.py +739 -0
- spectrax/transforms/vmap.py +157 -0
- spectrax/typing.py +54 -0
- spectrax_lib-0.0.1.dist-info/METADATA +437 -0
- spectrax_lib-0.0.1.dist-info/RECORD +116 -0
- spectrax_lib-0.0.1.dist-info/WHEEL +4 -0
spectrax/__init__.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# Copyright 2026 Erfan Zare Chavoshi
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""spectrax — a JAX-only neural-network library.
|
|
15
|
+
|
|
16
|
+
Top-level public API. The library combines a PyTorch-shaped eager
|
|
17
|
+
surface (subclass :class:`Module`, override :meth:`~Module.forward`,
|
|
18
|
+
call ``model(x)``) with an explicit graph/state seam
|
|
19
|
+
(:func:`export`, :func:`bind`, :class:`GraphDef`, :class:`State`,
|
|
20
|
+
:class:`Selector`) and module-aware JAX transforms
|
|
21
|
+
(:func:`eval_shape`, :func:`jit`, :func:`grad`,
|
|
22
|
+
:func:`value_and_grad`, :func:`vmap`, :func:`scan`, :func:`remat`).
|
|
23
|
+
|
|
24
|
+
Submodules imported here for qualified access:
|
|
25
|
+
|
|
26
|
+
* :mod:`spectrax.nn` — neural-network layers.
|
|
27
|
+
* :mod:`spectrax.functional` — stateless tensor ops.
|
|
28
|
+
* :mod:`spectrax.init` — parameter initializers.
|
|
29
|
+
* :mod:`spectrax.hooks` — forward and variable hooks.
|
|
30
|
+
* :mod:`spectrax.inspect` — introspection helpers.
|
|
31
|
+
* :mod:`spectrax.typing` — public type aliases.
|
|
32
|
+
* :mod:`spectrax.sharding` — logical axis rules, :class:`SpxMesh`
|
|
33
|
+
construction, partition-spec helpers.
|
|
34
|
+
* :mod:`spectrax.runtime` — pipeline-parallel runtimes and schedules
|
|
35
|
+
(:class:`GPipe`, :class:`Std1F1B`, :class:`ZeroBubbleH1`, …),
|
|
36
|
+
:func:`sxcall` / :func:`sxjit`, :class:`MpMdMesh`,
|
|
37
|
+
:class:`PipelineSequential`.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from . import (
|
|
41
|
+
functional,
|
|
42
|
+
hooks,
|
|
43
|
+
init,
|
|
44
|
+
inspect,
|
|
45
|
+
nn,
|
|
46
|
+
runtime,
|
|
47
|
+
sharding,
|
|
48
|
+
typing,
|
|
49
|
+
)
|
|
50
|
+
from ._version import __version__
|
|
51
|
+
from .api import run
|
|
52
|
+
from .core import context
|
|
53
|
+
from .core.context import scope
|
|
54
|
+
from .core.errors import (
|
|
55
|
+
CyclicGraphError,
|
|
56
|
+
IllegalMutationError,
|
|
57
|
+
LazyInitUnderTransformError,
|
|
58
|
+
SelectorError,
|
|
59
|
+
SpecTraxError,
|
|
60
|
+
)
|
|
61
|
+
from .core.graph import (
|
|
62
|
+
GraphDef,
|
|
63
|
+
bind,
|
|
64
|
+
clone,
|
|
65
|
+
export,
|
|
66
|
+
find,
|
|
67
|
+
iter_modules,
|
|
68
|
+
iter_variables,
|
|
69
|
+
live_variables,
|
|
70
|
+
pop,
|
|
71
|
+
tree_state,
|
|
72
|
+
update,
|
|
73
|
+
)
|
|
74
|
+
from .core.lazy_init import lazy_init
|
|
75
|
+
from .core.module import Module, Opaque
|
|
76
|
+
from .core.policy import Policy
|
|
77
|
+
from .core.selector import (
|
|
78
|
+
Everything,
|
|
79
|
+
Nothing,
|
|
80
|
+
Selector,
|
|
81
|
+
all_of,
|
|
82
|
+
any_of,
|
|
83
|
+
as_selector,
|
|
84
|
+
not_,
|
|
85
|
+
of_type,
|
|
86
|
+
path_contains,
|
|
87
|
+
path_endswith,
|
|
88
|
+
path_startswith,
|
|
89
|
+
select,
|
|
90
|
+
)
|
|
91
|
+
from .core.sharding import AxisNames, Sharding
|
|
92
|
+
from .core.stage_assignment import assign_stage
|
|
93
|
+
from .core.state import State
|
|
94
|
+
from .core.static import Static
|
|
95
|
+
from .core.variable import (
|
|
96
|
+
Buffer,
|
|
97
|
+
DeferredBuffer,
|
|
98
|
+
DeferredParameter,
|
|
99
|
+
Parameter,
|
|
100
|
+
Variable,
|
|
101
|
+
)
|
|
102
|
+
from .lint import check_unintentional_sharing
|
|
103
|
+
from .rng import Rngs, RngStream, resolve_rngs, seed
|
|
104
|
+
from .sharding.mesh import SpxMesh, create_mesh
|
|
105
|
+
from .transforms import (
|
|
106
|
+
StateAxes,
|
|
107
|
+
associative_scan,
|
|
108
|
+
cond,
|
|
109
|
+
eval_shape,
|
|
110
|
+
fori_loop,
|
|
111
|
+
grad,
|
|
112
|
+
jit,
|
|
113
|
+
jvp,
|
|
114
|
+
remat,
|
|
115
|
+
remat_scan,
|
|
116
|
+
scan,
|
|
117
|
+
split_rngs,
|
|
118
|
+
split_stream_keys,
|
|
119
|
+
switch,
|
|
120
|
+
value_and_grad,
|
|
121
|
+
vjp,
|
|
122
|
+
vmap,
|
|
123
|
+
while_loop,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
__all__ = [
|
|
127
|
+
"AxisNames",
|
|
128
|
+
"Buffer",
|
|
129
|
+
"CyclicGraphError",
|
|
130
|
+
"Everything",
|
|
131
|
+
"GraphDef",
|
|
132
|
+
"IllegalMutationError",
|
|
133
|
+
"LazyInitUnderTransformError",
|
|
134
|
+
"Module",
|
|
135
|
+
"Nothing",
|
|
136
|
+
"Opaque",
|
|
137
|
+
"Parameter",
|
|
138
|
+
"Policy",
|
|
139
|
+
"RngStream",
|
|
140
|
+
"Rngs",
|
|
141
|
+
"Selector",
|
|
142
|
+
"SelectorError",
|
|
143
|
+
"Sharding",
|
|
144
|
+
"SpecTraxError",
|
|
145
|
+
"SpxMesh",
|
|
146
|
+
"State",
|
|
147
|
+
"StateAxes",
|
|
148
|
+
"Static",
|
|
149
|
+
"Variable",
|
|
150
|
+
"__version__",
|
|
151
|
+
"all_of",
|
|
152
|
+
"any_of",
|
|
153
|
+
"as_selector",
|
|
154
|
+
"assign_stage",
|
|
155
|
+
"associative_scan",
|
|
156
|
+
"bind",
|
|
157
|
+
"check_unintentional_sharing",
|
|
158
|
+
"clone",
|
|
159
|
+
"cond",
|
|
160
|
+
"context",
|
|
161
|
+
"create_mesh",
|
|
162
|
+
"eval_shape",
|
|
163
|
+
"export",
|
|
164
|
+
"find",
|
|
165
|
+
"fori_loop",
|
|
166
|
+
"functional",
|
|
167
|
+
"grad",
|
|
168
|
+
"hooks",
|
|
169
|
+
"init",
|
|
170
|
+
"inspect",
|
|
171
|
+
"iter_modules",
|
|
172
|
+
"iter_variables",
|
|
173
|
+
"jit",
|
|
174
|
+
"jvp",
|
|
175
|
+
"lazy_init",
|
|
176
|
+
"live_variables",
|
|
177
|
+
"nn",
|
|
178
|
+
"not_",
|
|
179
|
+
"of_type",
|
|
180
|
+
"path_contains",
|
|
181
|
+
"path_endswith",
|
|
182
|
+
"path_startswith",
|
|
183
|
+
"pop",
|
|
184
|
+
"remat",
|
|
185
|
+
"remat_scan",
|
|
186
|
+
"resolve_rngs",
|
|
187
|
+
"run",
|
|
188
|
+
"runtime",
|
|
189
|
+
"scan",
|
|
190
|
+
"scope",
|
|
191
|
+
"seed",
|
|
192
|
+
"select",
|
|
193
|
+
"sharding",
|
|
194
|
+
"split_rngs",
|
|
195
|
+
"split_stream_keys",
|
|
196
|
+
"switch",
|
|
197
|
+
"tree_state",
|
|
198
|
+
"typing",
|
|
199
|
+
"update",
|
|
200
|
+
"value_and_grad",
|
|
201
|
+
"vjp",
|
|
202
|
+
"vmap",
|
|
203
|
+
"while_loop",
|
|
204
|
+
]
|
spectrax/_version.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Copyright 2026 Erfan Zare Chavoshi
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Package version string.
|
|
15
|
+
|
|
16
|
+
The version number is used by :mod:`spectrax.__init__` via
|
|
17
|
+
``from ._version import __version__`` and by packaging tooling.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
__version__ = "0.0.1"
|
spectrax/api.py
ADDED
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
# Copyright 2026 Erfan Zare Chavoshi
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
""":func:`spectrax.run` — one entry point for SPMD or MPMD execution.
|
|
15
|
+
|
|
16
|
+
Write the model once as a normal :class:`spectrax.Module`. Pass it
|
|
17
|
+
to ``spx.run(model, inputs=..., mesh=mesh, mode=...)`` along with a
|
|
18
|
+
mesh; the mesh decides whether to run pure SPMD (``pjit``) or MPMD
|
|
19
|
+
(``sxcall`` with auto-split into per-rank stages).
|
|
20
|
+
|
|
21
|
+
Examples::
|
|
22
|
+
|
|
23
|
+
# SPMD only — pjit, no pipeline parallelism
|
|
24
|
+
mesh = spx.create_mesh(axis_dims=(1, 1, -1, 1, 1, 1))
|
|
25
|
+
out = spx.run(model, inputs=ids, mesh=mesh, mode='forward')
|
|
26
|
+
|
|
27
|
+
# Add pipeline parallelism — same call site, mesh changes
|
|
28
|
+
mesh = spx.create_mesh(axis_dims=(2, 1, -1, 1, 1, 1), mpmd_axis='pp')
|
|
29
|
+
loss, grads = spx.run(model, inputs=ids, targets=labels,
|
|
30
|
+
mesh=mesh, mode='train', loss_fn=ce,
|
|
31
|
+
microbatches=4)
|
|
32
|
+
|
|
33
|
+
``inputs`` and ``targets`` accept three forms (Option C):
|
|
34
|
+
|
|
35
|
+
* a single array -> forwarded as-is
|
|
36
|
+
* a tuple/list -> unpacked as positional args
|
|
37
|
+
* a dict -> unpacked as kwargs
|
|
38
|
+
|
|
39
|
+
So ``inputs=ids`` calls ``model.forward(ids)``;
|
|
40
|
+
``inputs=(ids, mask)`` calls ``model.forward(ids, mask)``;
|
|
41
|
+
``inputs=dict(ids=ids, mask=mask)`` calls ``model.forward(ids=ids, mask=mask)``.
|
|
42
|
+
Same shape rules for ``targets`` against ``loss_fn``.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
from collections.abc import Callable, Mapping
|
|
48
|
+
from typing import Any, Literal
|
|
49
|
+
|
|
50
|
+
import jax
|
|
51
|
+
|
|
52
|
+
from .core._weakcache import weak_invalidate
|
|
53
|
+
from .core.graph import bind, export
|
|
54
|
+
from .core.module import Module
|
|
55
|
+
from .runtime.mpmd.runtime import sxcall
|
|
56
|
+
from .runtime.schedules import GPipe as _DefaultGPipe
|
|
57
|
+
from .sharding.mesh import SpxMesh
|
|
58
|
+
from .sharding.partition import get_named_sharding
|
|
59
|
+
|
|
60
|
+
__all__ = ["run"]
|
|
61
|
+
|
|
62
|
+
_SPMD_FWD_CACHE: dict[int, Callable] = {}
|
|
63
|
+
_SPMD_TRAIN_CACHE: dict[tuple, Callable] = {}
|
|
64
|
+
_SPMD_STATE_CACHE: dict[tuple, Any] = {}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _as_call(payload: Any) -> tuple[tuple, dict]:
|
|
68
|
+
"""Normalize an inputs/targets payload into ``(args, kwargs)`` for call.
|
|
69
|
+
|
|
70
|
+
* ``None`` -> ``((), {})``
|
|
71
|
+
* single array -> ``((array,), {})``
|
|
72
|
+
* tuple / list of values -> ``(tuple(...), {})``
|
|
73
|
+
* dict -> ``((), dict(...))``
|
|
74
|
+
"""
|
|
75
|
+
if payload is None:
|
|
76
|
+
return (), {}
|
|
77
|
+
if isinstance(payload, Mapping):
|
|
78
|
+
return (), dict(payload)
|
|
79
|
+
if isinstance(payload, (tuple, list)):
|
|
80
|
+
return tuple(payload), {}
|
|
81
|
+
return (payload,), {}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _place_state(state, model: Module, mesh):
|
|
85
|
+
"""Apply per-leaf shardings derived from the model's logical
|
|
86
|
+
axis-name annotations (resolved against the active
|
|
87
|
+
:func:`logical_axis_rules` context) to a State pytree."""
|
|
88
|
+
shards = get_named_sharding(model, mesh)
|
|
89
|
+
raw = state.raw()
|
|
90
|
+
out: dict[str, dict[str, Any]] = {}
|
|
91
|
+
for col, paths in raw.items():
|
|
92
|
+
col_shards = shards.get(col, {})
|
|
93
|
+
out_col: dict[str, Any] = {}
|
|
94
|
+
for path, leaf in paths.items():
|
|
95
|
+
sh = col_shards.get(path)
|
|
96
|
+
out_col[path] = jax.device_put(leaf, sh) if sh is not None else leaf
|
|
97
|
+
out[col] = out_col
|
|
98
|
+
return type(state)(out)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _mpmd_dummy_loss(y, *_a):
|
|
102
|
+
"""Stable dummy loss for forward-only MPMD (module-level so ``id()`` is constant)."""
|
|
103
|
+
return jax.numpy.zeros((), dtype=jax.numpy.float32)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _run_spmd(
|
|
107
|
+
model: Module,
|
|
108
|
+
args: tuple,
|
|
109
|
+
kwargs: dict,
|
|
110
|
+
*,
|
|
111
|
+
mesh: SpxMesh,
|
|
112
|
+
mode: str,
|
|
113
|
+
loss_args: tuple,
|
|
114
|
+
loss_kwargs: dict,
|
|
115
|
+
loss_fn: Callable | None,
|
|
116
|
+
):
|
|
117
|
+
"""Run ``model`` under pjit: forward-only or train (value_and_grad).
|
|
118
|
+
|
|
119
|
+
Jit-wrapped functions and placed state are cached so repeated
|
|
120
|
+
calls (e.g. decode loop) reuse the compiled program and skip
|
|
121
|
+
redundant ``_place_state`` walks.
|
|
122
|
+
"""
|
|
123
|
+
gdef, state = export(model)
|
|
124
|
+
jax_mesh = mesh.jax_mesh
|
|
125
|
+
|
|
126
|
+
state_key = (id(gdef), id(jax_mesh))
|
|
127
|
+
cached_state = _SPMD_STATE_CACHE.get(state_key)
|
|
128
|
+
if cached_state is not None:
|
|
129
|
+
state = cached_state
|
|
130
|
+
else:
|
|
131
|
+
state = _place_state(state, model, jax_mesh)
|
|
132
|
+
_SPMD_STATE_CACHE[state_key] = state
|
|
133
|
+
weak_invalidate(gdef, _SPMD_STATE_CACHE, state_key)
|
|
134
|
+
weak_invalidate(jax_mesh, _SPMD_STATE_CACHE, state_key)
|
|
135
|
+
|
|
136
|
+
if mode == "forward":
|
|
137
|
+
cache_key = id(gdef)
|
|
138
|
+
_fwd = _SPMD_FWD_CACHE.get(cache_key)
|
|
139
|
+
if _fwd is None:
|
|
140
|
+
|
|
141
|
+
@jax.jit
|
|
142
|
+
def _fwd(state, *a, **kw):
|
|
143
|
+
"""Bind ``gdef`` to ``state`` and call the resulting module."""
|
|
144
|
+
return bind(gdef, state)(*a, **kw)
|
|
145
|
+
|
|
146
|
+
_SPMD_FWD_CACHE[cache_key] = _fwd
|
|
147
|
+
weak_invalidate(gdef, _SPMD_FWD_CACHE, cache_key)
|
|
148
|
+
|
|
149
|
+
with jax_mesh:
|
|
150
|
+
return _fwd(state, *args, **kwargs)
|
|
151
|
+
|
|
152
|
+
if loss_fn is None:
|
|
153
|
+
raise ValueError("loss_fn required for mode='train'.")
|
|
154
|
+
|
|
155
|
+
train_key = (id(gdef), id(loss_fn))
|
|
156
|
+
_step = _SPMD_TRAIN_CACHE.get(train_key)
|
|
157
|
+
if _step is None:
|
|
158
|
+
|
|
159
|
+
@jax.jit
|
|
160
|
+
def _step(state, args, kwargs, l_args, l_kwargs):
|
|
161
|
+
"""Compute ``(loss, grads)`` via :func:`jax.value_and_grad` on ``state``."""
|
|
162
|
+
|
|
163
|
+
def loss(state):
|
|
164
|
+
"""Forward pass + loss closure over captured args/kwargs."""
|
|
165
|
+
out = bind(gdef, state)(*args, **kwargs)
|
|
166
|
+
return loss_fn(out, *l_args, **l_kwargs)
|
|
167
|
+
|
|
168
|
+
return jax.value_and_grad(loss)(state)
|
|
169
|
+
|
|
170
|
+
_SPMD_TRAIN_CACHE[train_key] = _step
|
|
171
|
+
weak_invalidate(gdef, _SPMD_TRAIN_CACHE, train_key)
|
|
172
|
+
weak_invalidate(loss_fn, _SPMD_TRAIN_CACHE, train_key)
|
|
173
|
+
|
|
174
|
+
with jax_mesh:
|
|
175
|
+
loss_val, grads = _step(state, args, kwargs, loss_args, loss_kwargs)
|
|
176
|
+
return loss_val, grads
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _run_mpmd(
|
|
180
|
+
model: Module,
|
|
181
|
+
args: tuple,
|
|
182
|
+
kwargs: dict,
|
|
183
|
+
*,
|
|
184
|
+
mesh: SpxMesh,
|
|
185
|
+
mode: str,
|
|
186
|
+
loss_args: tuple,
|
|
187
|
+
loss_kwargs: dict,
|
|
188
|
+
loss_fn: Callable | None,
|
|
189
|
+
microbatches: int,
|
|
190
|
+
schedule: Any = None,
|
|
191
|
+
fuse_1f1b: bool | None = None,
|
|
192
|
+
fuse_zb: bool | None = None,
|
|
193
|
+
has_aux: bool = False,
|
|
194
|
+
):
|
|
195
|
+
"""Auto-split ``model`` into per-rank stages and dispatch.
|
|
196
|
+
|
|
197
|
+
When ``schedule`` is provided, routes through :func:`sxcall`
|
|
198
|
+
which handles all 9 schedules (flat + virtual-stage) at any model
|
|
199
|
+
scale. When ``schedule`` is ``None``, uses the default GPipe path
|
|
200
|
+
via :func:`sxcall`.
|
|
201
|
+
|
|
202
|
+
``fuse_1f1b`` and ``fuse_zb`` enable :class:`FusedTask` steady-state
|
|
203
|
+
fusion on :func:`sxcall` (dispatch-count reduction for 1F1B and
|
|
204
|
+
ZeroBubble families). When ``None`` (default), fusion is enabled
|
|
205
|
+
automatically for compatible schedules.
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
if loss_kwargs:
|
|
209
|
+
target_keys = tuple(loss_kwargs.keys())
|
|
210
|
+
target_vals = tuple(loss_kwargs.values())
|
|
211
|
+
original_loss = loss_fn
|
|
212
|
+
|
|
213
|
+
def _wrapped_loss(out, *vals):
|
|
214
|
+
"""Call ``original_loss`` with positional ``vals`` re-keyed to ``target_keys``."""
|
|
215
|
+
return original_loss(out, **dict(zip(target_keys, vals, strict=True)))
|
|
216
|
+
|
|
217
|
+
loss_fn = _wrapped_loss
|
|
218
|
+
loss_args = loss_args + target_vals
|
|
219
|
+
|
|
220
|
+
if schedule is not None:
|
|
221
|
+
batch = (args[0], *tuple(loss_args)) if mode == "train" else (args[0],)
|
|
222
|
+
return sxcall(
|
|
223
|
+
model,
|
|
224
|
+
batch,
|
|
225
|
+
mesh=mesh.mpmd_mesh,
|
|
226
|
+
schedule=schedule,
|
|
227
|
+
loss_fn=loss_fn,
|
|
228
|
+
fuse_1f1b=fuse_1f1b,
|
|
229
|
+
fuse_zb=fuse_zb,
|
|
230
|
+
mode=mode,
|
|
231
|
+
has_aux=has_aux,
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
batch = (args[0], *tuple(loss_args)) if mode == "train" else (args[0],)
|
|
235
|
+
return sxcall(
|
|
236
|
+
model,
|
|
237
|
+
batch,
|
|
238
|
+
mesh=mesh.mpmd_mesh,
|
|
239
|
+
schedule=_DefaultGPipe(microbatches=max(microbatches, 1)),
|
|
240
|
+
loss_fn=loss_fn if loss_fn is not None else _mpmd_dummy_loss,
|
|
241
|
+
mode=mode,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def run(
|
|
246
|
+
model: Module,
|
|
247
|
+
*,
|
|
248
|
+
inputs: Any,
|
|
249
|
+
targets: Any = None,
|
|
250
|
+
mesh: SpxMesh,
|
|
251
|
+
mode: Literal["train", "forward"] = "forward",
|
|
252
|
+
loss_fn: Callable | None = None,
|
|
253
|
+
microbatches: int = 1,
|
|
254
|
+
schedule: Any = None,
|
|
255
|
+
fuse_1f1b: bool | None = None,
|
|
256
|
+
fuse_zb: bool | None = None,
|
|
257
|
+
has_aux: bool = False,
|
|
258
|
+
) -> Any:
|
|
259
|
+
"""Run a model under SPMD or MPMD — the mesh decides which.
|
|
260
|
+
|
|
261
|
+
Two modes, because that's all there is:
|
|
262
|
+
|
|
263
|
+
* ``"forward"`` — run the model, no autograd. This is both
|
|
264
|
+
inference and the decode primitive: to generate, call
|
|
265
|
+
``"forward"`` in a loop and feed the returned per-stage state
|
|
266
|
+
back in next step (KV cache, beam records, whatever lives in
|
|
267
|
+
stage state).
|
|
268
|
+
* ``"train"`` — forward + ``loss_fn`` + backward. Returns
|
|
269
|
+
``(loss, grads)``.
|
|
270
|
+
|
|
271
|
+
Args:
|
|
272
|
+
model: An :class:`spectrax.Module`. For MPMD, the model is
|
|
273
|
+
auto-split into per-rank stages — either via
|
|
274
|
+
``model.pipeline_split(n_pp)`` if defined or by detecting
|
|
275
|
+
a ``blocks: ModuleList`` attribute.
|
|
276
|
+
inputs: Forward arguments for ``model.forward``. Accepts:
|
|
277
|
+
|
|
278
|
+
* a single array -> ``forward(array)``
|
|
279
|
+
* tuple/list -> ``forward(*payload)``
|
|
280
|
+
* dict -> ``forward(**payload)``
|
|
281
|
+
|
|
282
|
+
targets: Loss targets passed to ``loss_fn`` after the output.
|
|
283
|
+
Same shape rules as ``inputs``. Required for ``mode="train"``.
|
|
284
|
+
mesh: An :class:`SpxMesh` (built via :func:`spectrax.create_mesh`).
|
|
285
|
+
``mesh.is_mpmd`` decides the path:
|
|
286
|
+
|
|
287
|
+
* ``False`` -> pjit (pure SPMD, FSDP/TP via the model's
|
|
288
|
+
:func:`logical_axis_rules` annotations).
|
|
289
|
+
* ``True`` -> split the model and call
|
|
290
|
+
:func:`sxcall` (PP x FSDP x TP).
|
|
291
|
+
|
|
292
|
+
mode: ``"forward"`` or ``"train"``.
|
|
293
|
+
loss_fn: Required for ``mode="train"``. Called as
|
|
294
|
+
``loss_fn(output, *target_args, **target_kwargs)``.
|
|
295
|
+
microbatches: Pipeline microbatch count. Ignored for SPMD.
|
|
296
|
+
|
|
297
|
+
Returns:
|
|
298
|
+
* ``mode="forward"`` -> ``output`` (same shape under SPMD and MPMD).
|
|
299
|
+
* ``mode="train"`` -> ``(loss_scalar, grads)``. Under SPMD ``grads``
|
|
300
|
+
is a single State; under MPMD it's a ``tuple[per_rank_State]``.
|
|
301
|
+
|
|
302
|
+
Note:
|
|
303
|
+
For MPMD decode that needs the per-rank stage state out (KV cache
|
|
304
|
+
threading), drop to :func:`sxcall` directly — its
|
|
305
|
+
``mode='forward'`` returns ``(output, tuple[per_rank_state])``.
|
|
306
|
+
``spx.run`` hides that to keep its return shape uniform across
|
|
307
|
+
SPMD and MPMD.
|
|
308
|
+
"""
|
|
309
|
+
if not isinstance(mesh, SpxMesh):
|
|
310
|
+
raise TypeError(f"mesh must be an SpxMesh (build via spx.create_mesh); got {type(mesh).__name__}.")
|
|
311
|
+
|
|
312
|
+
args, kwargs = _as_call(inputs)
|
|
313
|
+
loss_args, loss_kwargs = _as_call(targets)
|
|
314
|
+
|
|
315
|
+
if mesh.is_mpmd:
|
|
316
|
+
if len(args) != 1 or kwargs:
|
|
317
|
+
raise ValueError(
|
|
318
|
+
"MPMD (pipeline) execution requires exactly one positional "
|
|
319
|
+
"model input, microbatched along its leading axis; got "
|
|
320
|
+
f"{len(args)} positional and {len(kwargs)} keyword inputs. "
|
|
321
|
+
"Pass `inputs=<single_array>` (a bare array or a 1-tuple). "
|
|
322
|
+
"Multi-input / dict-shaped inputs are not supported under "
|
|
323
|
+
"MPMD yet — use an SPMD mesh or fold the inputs into one "
|
|
324
|
+
"batched array."
|
|
325
|
+
)
|
|
326
|
+
return _run_mpmd(
|
|
327
|
+
model,
|
|
328
|
+
args,
|
|
329
|
+
kwargs,
|
|
330
|
+
mesh=mesh,
|
|
331
|
+
mode=mode,
|
|
332
|
+
loss_args=loss_args,
|
|
333
|
+
loss_kwargs=loss_kwargs,
|
|
334
|
+
loss_fn=loss_fn,
|
|
335
|
+
microbatches=microbatches,
|
|
336
|
+
schedule=schedule,
|
|
337
|
+
fuse_1f1b=fuse_1f1b,
|
|
338
|
+
fuse_zb=fuse_zb,
|
|
339
|
+
has_aux=has_aux,
|
|
340
|
+
)
|
|
341
|
+
if schedule is not None:
|
|
342
|
+
raise ValueError(
|
|
343
|
+
"schedule= is only meaningful with an MPMD mesh "
|
|
344
|
+
"(create_mesh(mpmd_axis=...)); got a pure SPMD mesh. "
|
|
345
|
+
"SPMD has no pipeline schedule — drop schedule= or use "
|
|
346
|
+
"an MPMD mesh."
|
|
347
|
+
)
|
|
348
|
+
return _run_spmd(
|
|
349
|
+
model,
|
|
350
|
+
args,
|
|
351
|
+
kwargs,
|
|
352
|
+
mesh=mesh,
|
|
353
|
+
mode=mode,
|
|
354
|
+
loss_args=loss_args,
|
|
355
|
+
loss_kwargs=loss_kwargs,
|
|
356
|
+
loss_fn=loss_fn,
|
|
357
|
+
)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Copyright 2026 Erfan Zare Chavoshi
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Opt-in helpers that depend on third-party libraries.
|
|
15
|
+
|
|
16
|
+
Anything here lives behind an optional install (e.g.
|
|
17
|
+
``pip install spectrax-lib[contrib]``) and must import gracefully when its
|
|
18
|
+
dependency is missing.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from .optimizer import MultiOptimizer, Optimizer
|
|
22
|
+
|
|
23
|
+
__all__ = ["MultiOptimizer", "Optimizer"]
|