spectrax-lib 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. spectrax/__init__.py +204 -0
  2. spectrax/_version.py +20 -0
  3. spectrax/api.py +357 -0
  4. spectrax/contrib/__init__.py +23 -0
  5. spectrax/contrib/optimizer.py +377 -0
  6. spectrax/core/__init__.py +57 -0
  7. spectrax/core/_typing.py +144 -0
  8. spectrax/core/_weakcache.py +56 -0
  9. spectrax/core/containers.py +336 -0
  10. spectrax/core/context.py +187 -0
  11. spectrax/core/errors.py +89 -0
  12. spectrax/core/graph.py +858 -0
  13. spectrax/core/lazy_init.py +63 -0
  14. spectrax/core/module.py +975 -0
  15. spectrax/core/paths.py +128 -0
  16. spectrax/core/policy.py +114 -0
  17. spectrax/core/registry.py +102 -0
  18. spectrax/core/selector.py +553 -0
  19. spectrax/core/sharding.py +156 -0
  20. spectrax/core/stage_assignment.py +109 -0
  21. spectrax/core/state.py +361 -0
  22. spectrax/core/static.py +85 -0
  23. spectrax/core/variable.py +898 -0
  24. spectrax/functional/__init__.py +84 -0
  25. spectrax/functional/activation.py +132 -0
  26. spectrax/functional/attention.py +81 -0
  27. spectrax/functional/conv.py +129 -0
  28. spectrax/functional/dropout.py +56 -0
  29. spectrax/functional/linear.py +74 -0
  30. spectrax/functional/norm.py +83 -0
  31. spectrax/functional/pool.py +106 -0
  32. spectrax/functional/util.py +43 -0
  33. spectrax/hooks/__init__.py +30 -0
  34. spectrax/hooks/forward.py +48 -0
  35. spectrax/hooks/variable.py +60 -0
  36. spectrax/init/__init__.py +41 -0
  37. spectrax/init/constant.py +47 -0
  38. spectrax/init/kaiming.py +75 -0
  39. spectrax/init/normal.py +49 -0
  40. spectrax/init/orthogonal.py +54 -0
  41. spectrax/init/uniform.py +31 -0
  42. spectrax/init/xavier.py +82 -0
  43. spectrax/inspect/__init__.py +35 -0
  44. spectrax/inspect/counting.py +31 -0
  45. spectrax/inspect/display.py +37 -0
  46. spectrax/inspect/repr.py +107 -0
  47. spectrax/inspect/summary.py +83 -0
  48. spectrax/inspect/tabulate.py +142 -0
  49. spectrax/inspect/tree.py +47 -0
  50. spectrax/lint.py +66 -0
  51. spectrax/nn/__init__.py +125 -0
  52. spectrax/nn/activation.py +95 -0
  53. spectrax/nn/attention.py +277 -0
  54. spectrax/nn/conv.py +306 -0
  55. spectrax/nn/dense.py +202 -0
  56. spectrax/nn/dropout.py +97 -0
  57. spectrax/nn/embed.py +101 -0
  58. spectrax/nn/fp8.py +724 -0
  59. spectrax/nn/identity.py +36 -0
  60. spectrax/nn/linear.py +188 -0
  61. spectrax/nn/lora.py +356 -0
  62. spectrax/nn/mlp.py +112 -0
  63. spectrax/nn/norm.py +358 -0
  64. spectrax/nn/pipeline_sequential.py +141 -0
  65. spectrax/nn/pool.py +170 -0
  66. spectrax/nn/recurrent.py +556 -0
  67. spectrax/py.typed +0 -0
  68. spectrax/rng/__init__.py +26 -0
  69. spectrax/rng/rngs.py +258 -0
  70. spectrax/rng/seed.py +81 -0
  71. spectrax/runtime/__init__.py +81 -0
  72. spectrax/runtime/mpmd/__init__.py +95 -0
  73. spectrax/runtime/mpmd/compiler.py +334 -0
  74. spectrax/runtime/mpmd/markers.py +447 -0
  75. spectrax/runtime/mpmd/per_rank.py +341 -0
  76. spectrax/runtime/mpmd/pscan_compiler.py +977 -0
  77. spectrax/runtime/mpmd/runtime.py +3064 -0
  78. spectrax/runtime/mpmd/treduce.py +483 -0
  79. spectrax/runtime/primitives/__init__.py +25 -0
  80. spectrax/runtime/primitives/boundary.py +77 -0
  81. spectrax/runtime/primitives/split.py +272 -0
  82. spectrax/runtime/schedules/__init__.py +46 -0
  83. spectrax/runtime/schedules/base.py +213 -0
  84. spectrax/runtime/schedules/dualpipe.py +260 -0
  85. spectrax/runtime/schedules/fusion.py +123 -0
  86. spectrax/runtime/schedules/gpipe.py +86 -0
  87. spectrax/runtime/schedules/interleaved.py +315 -0
  88. spectrax/runtime/schedules/one_f_one_b.py +181 -0
  89. spectrax/runtime/schedules/zero_bubble.py +115 -0
  90. spectrax/runtime/spmd/__init__.py +33 -0
  91. spectrax/runtime/spmd/api.py +998 -0
  92. spectrax/runtime/spmd/hybrid.py +360 -0
  93. spectrax/runtime/spmd/runtime.py +351 -0
  94. spectrax/runtime/spmd/shard_map.py +554 -0
  95. spectrax/runtime/types/__init__.py +29 -0
  96. spectrax/runtime/types/array.py +332 -0
  97. spectrax/runtime/types/mesh.py +229 -0
  98. spectrax/runtime/types/stage.py +44 -0
  99. spectrax/sharding/__init__.py +56 -0
  100. spectrax/sharding/logical.py +59 -0
  101. spectrax/sharding/mesh.py +706 -0
  102. spectrax/sharding/partition.py +184 -0
  103. spectrax/transforms/__init__.py +55 -0
  104. spectrax/transforms/control_flow.py +393 -0
  105. spectrax/transforms/eval_shape.py +47 -0
  106. spectrax/transforms/grad.py +683 -0
  107. spectrax/transforms/jit.py +343 -0
  108. spectrax/transforms/remat.py +254 -0
  109. spectrax/transforms/rng_axes.py +117 -0
  110. spectrax/transforms/scan.py +210 -0
  111. spectrax/transforms/split_merge.py +739 -0
  112. spectrax/transforms/vmap.py +157 -0
  113. spectrax/typing.py +54 -0
  114. spectrax_lib-0.0.1.dist-info/METADATA +437 -0
  115. spectrax_lib-0.0.1.dist-info/RECORD +116 -0
  116. spectrax_lib-0.0.1.dist-info/WHEEL +4 -0
spectrax/__init__.py ADDED
@@ -0,0 +1,204 @@
1
+ # Copyright 2026 Erfan Zare Chavoshi
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """spectrax — a JAX-only neural-network library.
15
+
16
+ Top-level public API. The library combines a PyTorch-shaped eager
17
+ surface (subclass :class:`Module`, override :meth:`~Module.forward`,
18
+ call ``model(x)``) with an explicit graph/state seam
19
+ (:func:`export`, :func:`bind`, :class:`GraphDef`, :class:`State`,
20
+ :class:`Selector`) and module-aware JAX transforms
21
+ (:func:`eval_shape`, :func:`jit`, :func:`grad`,
22
+ :func:`value_and_grad`, :func:`vmap`, :func:`scan`, :func:`remat`).
23
+
24
+ Submodules imported here for qualified access:
25
+
26
+ * :mod:`spectrax.nn` — neural-network layers.
27
+ * :mod:`spectrax.functional` — stateless tensor ops.
28
+ * :mod:`spectrax.init` — parameter initializers.
29
+ * :mod:`spectrax.hooks` — forward and variable hooks.
30
+ * :mod:`spectrax.inspect` — introspection helpers.
31
+ * :mod:`spectrax.typing` — public type aliases.
32
+ * :mod:`spectrax.sharding` — logical axis rules, :class:`SpxMesh`
33
+ construction, partition-spec helpers.
34
+ * :mod:`spectrax.runtime` — pipeline-parallel runtimes and schedules
35
+ (:class:`GPipe`, :class:`Std1F1B`, :class:`ZeroBubbleH1`, …),
36
+ :func:`sxcall` / :func:`sxjit`, :class:`MpMdMesh`,
37
+ :class:`PipelineSequential`.
38
+ """
39
+
40
+ from . import (
41
+ functional,
42
+ hooks,
43
+ init,
44
+ inspect,
45
+ nn,
46
+ runtime,
47
+ sharding,
48
+ typing,
49
+ )
50
+ from ._version import __version__
51
+ from .api import run
52
+ from .core import context
53
+ from .core.context import scope
54
+ from .core.errors import (
55
+ CyclicGraphError,
56
+ IllegalMutationError,
57
+ LazyInitUnderTransformError,
58
+ SelectorError,
59
+ SpecTraxError,
60
+ )
61
+ from .core.graph import (
62
+ GraphDef,
63
+ bind,
64
+ clone,
65
+ export,
66
+ find,
67
+ iter_modules,
68
+ iter_variables,
69
+ live_variables,
70
+ pop,
71
+ tree_state,
72
+ update,
73
+ )
74
+ from .core.lazy_init import lazy_init
75
+ from .core.module import Module, Opaque
76
+ from .core.policy import Policy
77
+ from .core.selector import (
78
+ Everything,
79
+ Nothing,
80
+ Selector,
81
+ all_of,
82
+ any_of,
83
+ as_selector,
84
+ not_,
85
+ of_type,
86
+ path_contains,
87
+ path_endswith,
88
+ path_startswith,
89
+ select,
90
+ )
91
+ from .core.sharding import AxisNames, Sharding
92
+ from .core.stage_assignment import assign_stage
93
+ from .core.state import State
94
+ from .core.static import Static
95
+ from .core.variable import (
96
+ Buffer,
97
+ DeferredBuffer,
98
+ DeferredParameter,
99
+ Parameter,
100
+ Variable,
101
+ )
102
+ from .lint import check_unintentional_sharing
103
+ from .rng import Rngs, RngStream, resolve_rngs, seed
104
+ from .sharding.mesh import SpxMesh, create_mesh
105
+ from .transforms import (
106
+ StateAxes,
107
+ associative_scan,
108
+ cond,
109
+ eval_shape,
110
+ fori_loop,
111
+ grad,
112
+ jit,
113
+ jvp,
114
+ remat,
115
+ remat_scan,
116
+ scan,
117
+ split_rngs,
118
+ split_stream_keys,
119
+ switch,
120
+ value_and_grad,
121
+ vjp,
122
+ vmap,
123
+ while_loop,
124
+ )
125
+
126
+ __all__ = [
127
+ "AxisNames",
128
+ "Buffer",
129
+ "CyclicGraphError",
130
+ "Everything",
131
+ "GraphDef",
132
+ "IllegalMutationError",
133
+ "LazyInitUnderTransformError",
134
+ "Module",
135
+ "Nothing",
136
+ "Opaque",
137
+ "Parameter",
138
+ "Policy",
139
+ "RngStream",
140
+ "Rngs",
141
+ "Selector",
142
+ "SelectorError",
143
+ "Sharding",
144
+ "SpecTraxError",
145
+ "SpxMesh",
146
+ "State",
147
+ "StateAxes",
148
+ "Static",
149
+ "Variable",
150
+ "__version__",
151
+ "all_of",
152
+ "any_of",
153
+ "as_selector",
154
+ "assign_stage",
155
+ "associative_scan",
156
+ "bind",
157
+ "check_unintentional_sharing",
158
+ "clone",
159
+ "cond",
160
+ "context",
161
+ "create_mesh",
162
+ "eval_shape",
163
+ "export",
164
+ "find",
165
+ "fori_loop",
166
+ "functional",
167
+ "grad",
168
+ "hooks",
169
+ "init",
170
+ "inspect",
171
+ "iter_modules",
172
+ "iter_variables",
173
+ "jit",
174
+ "jvp",
175
+ "lazy_init",
176
+ "live_variables",
177
+ "nn",
178
+ "not_",
179
+ "of_type",
180
+ "path_contains",
181
+ "path_endswith",
182
+ "path_startswith",
183
+ "pop",
184
+ "remat",
185
+ "remat_scan",
186
+ "resolve_rngs",
187
+ "run",
188
+ "runtime",
189
+ "scan",
190
+ "scope",
191
+ "seed",
192
+ "select",
193
+ "sharding",
194
+ "split_rngs",
195
+ "split_stream_keys",
196
+ "switch",
197
+ "tree_state",
198
+ "typing",
199
+ "update",
200
+ "value_and_grad",
201
+ "vjp",
202
+ "vmap",
203
+ "while_loop",
204
+ ]
spectrax/_version.py ADDED
@@ -0,0 +1,20 @@
1
+ # Copyright 2026 Erfan Zare Chavoshi
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Package version string.
15
+
16
+ The version number is used by :mod:`spectrax.__init__` via
17
+ ``from ._version import __version__`` and by packaging tooling.
18
+ """
19
+
20
+ __version__ = "0.0.1"
spectrax/api.py ADDED
@@ -0,0 +1,357 @@
1
+ # Copyright 2026 Erfan Zare Chavoshi
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """:func:`spectrax.run` — one entry point for SPMD or MPMD execution.
15
+
16
+ Write the model once as a normal :class:`spectrax.Module`. Pass it
17
+ to ``spx.run(model, inputs=..., mesh=mesh, mode=...)`` along with a
18
+ mesh; the mesh decides whether to run pure SPMD (``pjit``) or MPMD
19
+ (``sxcall`` with auto-split into per-rank stages).
20
+
21
+ Examples::
22
+
23
+ # SPMD only — pjit, no pipeline parallelism
24
+ mesh = spx.create_mesh(axis_dims=(1, 1, -1, 1, 1, 1))
25
+ out = spx.run(model, inputs=ids, mesh=mesh, mode='forward')
26
+
27
+ # Add pipeline parallelism — same call site, mesh changes
28
+ mesh = spx.create_mesh(axis_dims=(2, 1, -1, 1, 1, 1), mpmd_axis='pp')
29
+ loss, grads = spx.run(model, inputs=ids, targets=labels,
30
+ mesh=mesh, mode='train', loss_fn=ce,
31
+ microbatches=4)
32
+
33
+ ``inputs`` and ``targets`` accept three forms (Option C):
34
+
35
+ * a single array -> forwarded as-is
36
+ * a tuple/list -> unpacked as positional args
37
+ * a dict -> unpacked as kwargs
38
+
39
+ So ``inputs=ids`` calls ``model.forward(ids)``;
40
+ ``inputs=(ids, mask)`` calls ``model.forward(ids, mask)``;
41
+ ``inputs=dict(ids=ids, mask=mask)`` calls ``model.forward(ids=ids, mask=mask)``.
42
+ Same shape rules for ``targets`` against ``loss_fn``.
43
+ """
44
+
45
+ from __future__ import annotations
46
+
47
+ from collections.abc import Callable, Mapping
48
+ from typing import Any, Literal
49
+
50
+ import jax
51
+
52
+ from .core._weakcache import weak_invalidate
53
+ from .core.graph import bind, export
54
+ from .core.module import Module
55
+ from .runtime.mpmd.runtime import sxcall
56
+ from .runtime.schedules import GPipe as _DefaultGPipe
57
+ from .sharding.mesh import SpxMesh
58
+ from .sharding.partition import get_named_sharding
59
+
60
+ __all__ = ["run"]
61
+
62
+ _SPMD_FWD_CACHE: dict[int, Callable] = {}
63
+ _SPMD_TRAIN_CACHE: dict[tuple, Callable] = {}
64
+ _SPMD_STATE_CACHE: dict[tuple, Any] = {}
65
+
66
+
67
+ def _as_call(payload: Any) -> tuple[tuple, dict]:
68
+ """Normalize an inputs/targets payload into ``(args, kwargs)`` for call.
69
+
70
+ * ``None`` -> ``((), {})``
71
+ * single array -> ``((array,), {})``
72
+ * tuple / list of values -> ``(tuple(...), {})``
73
+ * dict -> ``((), dict(...))``
74
+ """
75
+ if payload is None:
76
+ return (), {}
77
+ if isinstance(payload, Mapping):
78
+ return (), dict(payload)
79
+ if isinstance(payload, (tuple, list)):
80
+ return tuple(payload), {}
81
+ return (payload,), {}
82
+
83
+
84
+ def _place_state(state, model: Module, mesh):
85
+ """Apply per-leaf shardings derived from the model's logical
86
+ axis-name annotations (resolved against the active
87
+ :func:`logical_axis_rules` context) to a State pytree."""
88
+ shards = get_named_sharding(model, mesh)
89
+ raw = state.raw()
90
+ out: dict[str, dict[str, Any]] = {}
91
+ for col, paths in raw.items():
92
+ col_shards = shards.get(col, {})
93
+ out_col: dict[str, Any] = {}
94
+ for path, leaf in paths.items():
95
+ sh = col_shards.get(path)
96
+ out_col[path] = jax.device_put(leaf, sh) if sh is not None else leaf
97
+ out[col] = out_col
98
+ return type(state)(out)
99
+
100
+
101
+ def _mpmd_dummy_loss(y, *_a):
102
+ """Stable dummy loss for forward-only MPMD (module-level so ``id()`` is constant)."""
103
+ return jax.numpy.zeros((), dtype=jax.numpy.float32)
104
+
105
+
106
+ def _run_spmd(
107
+ model: Module,
108
+ args: tuple,
109
+ kwargs: dict,
110
+ *,
111
+ mesh: SpxMesh,
112
+ mode: str,
113
+ loss_args: tuple,
114
+ loss_kwargs: dict,
115
+ loss_fn: Callable | None,
116
+ ):
117
+ """Run ``model`` under pjit: forward-only or train (value_and_grad).
118
+
119
+ Jit-wrapped functions and placed state are cached so repeated
120
+ calls (e.g. decode loop) reuse the compiled program and skip
121
+ redundant ``_place_state`` walks.
122
+ """
123
+ gdef, state = export(model)
124
+ jax_mesh = mesh.jax_mesh
125
+
126
+ state_key = (id(gdef), id(jax_mesh))
127
+ cached_state = _SPMD_STATE_CACHE.get(state_key)
128
+ if cached_state is not None:
129
+ state = cached_state
130
+ else:
131
+ state = _place_state(state, model, jax_mesh)
132
+ _SPMD_STATE_CACHE[state_key] = state
133
+ weak_invalidate(gdef, _SPMD_STATE_CACHE, state_key)
134
+ weak_invalidate(jax_mesh, _SPMD_STATE_CACHE, state_key)
135
+
136
+ if mode == "forward":
137
+ cache_key = id(gdef)
138
+ _fwd = _SPMD_FWD_CACHE.get(cache_key)
139
+ if _fwd is None:
140
+
141
+ @jax.jit
142
+ def _fwd(state, *a, **kw):
143
+ """Bind ``gdef`` to ``state`` and call the resulting module."""
144
+ return bind(gdef, state)(*a, **kw)
145
+
146
+ _SPMD_FWD_CACHE[cache_key] = _fwd
147
+ weak_invalidate(gdef, _SPMD_FWD_CACHE, cache_key)
148
+
149
+ with jax_mesh:
150
+ return _fwd(state, *args, **kwargs)
151
+
152
+ if loss_fn is None:
153
+ raise ValueError("loss_fn required for mode='train'.")
154
+
155
+ train_key = (id(gdef), id(loss_fn))
156
+ _step = _SPMD_TRAIN_CACHE.get(train_key)
157
+ if _step is None:
158
+
159
+ @jax.jit
160
+ def _step(state, args, kwargs, l_args, l_kwargs):
161
+ """Compute ``(loss, grads)`` via :func:`jax.value_and_grad` on ``state``."""
162
+
163
+ def loss(state):
164
+ """Forward pass + loss closure over captured args/kwargs."""
165
+ out = bind(gdef, state)(*args, **kwargs)
166
+ return loss_fn(out, *l_args, **l_kwargs)
167
+
168
+ return jax.value_and_grad(loss)(state)
169
+
170
+ _SPMD_TRAIN_CACHE[train_key] = _step
171
+ weak_invalidate(gdef, _SPMD_TRAIN_CACHE, train_key)
172
+ weak_invalidate(loss_fn, _SPMD_TRAIN_CACHE, train_key)
173
+
174
+ with jax_mesh:
175
+ loss_val, grads = _step(state, args, kwargs, loss_args, loss_kwargs)
176
+ return loss_val, grads
177
+
178
+
179
+ def _run_mpmd(
180
+ model: Module,
181
+ args: tuple,
182
+ kwargs: dict,
183
+ *,
184
+ mesh: SpxMesh,
185
+ mode: str,
186
+ loss_args: tuple,
187
+ loss_kwargs: dict,
188
+ loss_fn: Callable | None,
189
+ microbatches: int,
190
+ schedule: Any = None,
191
+ fuse_1f1b: bool | None = None,
192
+ fuse_zb: bool | None = None,
193
+ has_aux: bool = False,
194
+ ):
195
+ """Auto-split ``model`` into per-rank stages and dispatch.
196
+
197
+ When ``schedule`` is provided, routes through :func:`sxcall`
198
+ which handles all 9 schedules (flat + virtual-stage) at any model
199
+ scale. When ``schedule`` is ``None``, uses the default GPipe path
200
+ via :func:`sxcall`.
201
+
202
+ ``fuse_1f1b`` and ``fuse_zb`` enable :class:`FusedTask` steady-state
203
+ fusion on :func:`sxcall` (dispatch-count reduction for 1F1B and
204
+ ZeroBubble families). When ``None`` (default), fusion is enabled
205
+ automatically for compatible schedules.
206
+ """
207
+
208
+ if loss_kwargs:
209
+ target_keys = tuple(loss_kwargs.keys())
210
+ target_vals = tuple(loss_kwargs.values())
211
+ original_loss = loss_fn
212
+
213
+ def _wrapped_loss(out, *vals):
214
+ """Call ``original_loss`` with positional ``vals`` re-keyed to ``target_keys``."""
215
+ return original_loss(out, **dict(zip(target_keys, vals, strict=True)))
216
+
217
+ loss_fn = _wrapped_loss
218
+ loss_args = loss_args + target_vals
219
+
220
+ if schedule is not None:
221
+ batch = (args[0], *tuple(loss_args)) if mode == "train" else (args[0],)
222
+ return sxcall(
223
+ model,
224
+ batch,
225
+ mesh=mesh.mpmd_mesh,
226
+ schedule=schedule,
227
+ loss_fn=loss_fn,
228
+ fuse_1f1b=fuse_1f1b,
229
+ fuse_zb=fuse_zb,
230
+ mode=mode,
231
+ has_aux=has_aux,
232
+ )
233
+
234
+ batch = (args[0], *tuple(loss_args)) if mode == "train" else (args[0],)
235
+ return sxcall(
236
+ model,
237
+ batch,
238
+ mesh=mesh.mpmd_mesh,
239
+ schedule=_DefaultGPipe(microbatches=max(microbatches, 1)),
240
+ loss_fn=loss_fn if loss_fn is not None else _mpmd_dummy_loss,
241
+ mode=mode,
242
+ )
243
+
244
+
245
+ def run(
246
+ model: Module,
247
+ *,
248
+ inputs: Any,
249
+ targets: Any = None,
250
+ mesh: SpxMesh,
251
+ mode: Literal["train", "forward"] = "forward",
252
+ loss_fn: Callable | None = None,
253
+ microbatches: int = 1,
254
+ schedule: Any = None,
255
+ fuse_1f1b: bool | None = None,
256
+ fuse_zb: bool | None = None,
257
+ has_aux: bool = False,
258
+ ) -> Any:
259
+ """Run a model under SPMD or MPMD — the mesh decides which.
260
+
261
+ Two modes, because that's all there is:
262
+
263
+ * ``"forward"`` — run the model, no autograd. This is both
264
+ inference and the decode primitive: to generate, call
265
+ ``"forward"`` in a loop and feed the returned per-stage state
266
+ back in next step (KV cache, beam records, whatever lives in
267
+ stage state).
268
+ * ``"train"`` — forward + ``loss_fn`` + backward. Returns
269
+ ``(loss, grads)``.
270
+
271
+ Args:
272
+ model: An :class:`spectrax.Module`. For MPMD, the model is
273
+ auto-split into per-rank stages — either via
274
+ ``model.pipeline_split(n_pp)`` if defined or by detecting
275
+ a ``blocks: ModuleList`` attribute.
276
+ inputs: Forward arguments for ``model.forward``. Accepts:
277
+
278
+ * a single array -> ``forward(array)``
279
+ * tuple/list -> ``forward(*payload)``
280
+ * dict -> ``forward(**payload)``
281
+
282
+ targets: Loss targets passed to ``loss_fn`` after the output.
283
+ Same shape rules as ``inputs``. Required for ``mode="train"``.
284
+ mesh: An :class:`SpxMesh` (built via :func:`spectrax.create_mesh`).
285
+ ``mesh.is_mpmd`` decides the path:
286
+
287
+ * ``False`` -> pjit (pure SPMD, FSDP/TP via the model's
288
+ :func:`logical_axis_rules` annotations).
289
+ * ``True`` -> split the model and call
290
+ :func:`sxcall` (PP x FSDP x TP).
291
+
292
+ mode: ``"forward"`` or ``"train"``.
293
+ loss_fn: Required for ``mode="train"``. Called as
294
+ ``loss_fn(output, *target_args, **target_kwargs)``.
295
+ microbatches: Pipeline microbatch count. Ignored for SPMD.
296
+
297
+ Returns:
298
+ * ``mode="forward"`` -> ``output`` (same shape under SPMD and MPMD).
299
+ * ``mode="train"`` -> ``(loss_scalar, grads)``. Under SPMD ``grads``
300
+ is a single State; under MPMD it's a ``tuple[per_rank_State]``.
301
+
302
+ Note:
303
+ For MPMD decode that needs the per-rank stage state out (KV cache
304
+ threading), drop to :func:`sxcall` directly — its
305
+ ``mode='forward'`` returns ``(output, tuple[per_rank_state])``.
306
+ ``spx.run`` hides that to keep its return shape uniform across
307
+ SPMD and MPMD.
308
+ """
309
+ if not isinstance(mesh, SpxMesh):
310
+ raise TypeError(f"mesh must be an SpxMesh (build via spx.create_mesh); got {type(mesh).__name__}.")
311
+
312
+ args, kwargs = _as_call(inputs)
313
+ loss_args, loss_kwargs = _as_call(targets)
314
+
315
+ if mesh.is_mpmd:
316
+ if len(args) != 1 or kwargs:
317
+ raise ValueError(
318
+ "MPMD (pipeline) execution requires exactly one positional "
319
+ "model input, microbatched along its leading axis; got "
320
+ f"{len(args)} positional and {len(kwargs)} keyword inputs. "
321
+ "Pass `inputs=<single_array>` (a bare array or a 1-tuple). "
322
+ "Multi-input / dict-shaped inputs are not supported under "
323
+ "MPMD yet — use an SPMD mesh or fold the inputs into one "
324
+ "batched array."
325
+ )
326
+ return _run_mpmd(
327
+ model,
328
+ args,
329
+ kwargs,
330
+ mesh=mesh,
331
+ mode=mode,
332
+ loss_args=loss_args,
333
+ loss_kwargs=loss_kwargs,
334
+ loss_fn=loss_fn,
335
+ microbatches=microbatches,
336
+ schedule=schedule,
337
+ fuse_1f1b=fuse_1f1b,
338
+ fuse_zb=fuse_zb,
339
+ has_aux=has_aux,
340
+ )
341
+ if schedule is not None:
342
+ raise ValueError(
343
+ "schedule= is only meaningful with an MPMD mesh "
344
+ "(create_mesh(mpmd_axis=...)); got a pure SPMD mesh. "
345
+ "SPMD has no pipeline schedule — drop schedule= or use "
346
+ "an MPMD mesh."
347
+ )
348
+ return _run_spmd(
349
+ model,
350
+ args,
351
+ kwargs,
352
+ mesh=mesh,
353
+ mode=mode,
354
+ loss_args=loss_args,
355
+ loss_kwargs=loss_kwargs,
356
+ loss_fn=loss_fn,
357
+ )
@@ -0,0 +1,23 @@
1
+ # Copyright 2026 Erfan Zare Chavoshi
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Opt-in helpers that depend on third-party libraries.
15
+
16
+ Anything here lives behind an optional install (e.g.
17
+ ``pip install spectrax-lib[contrib]``) and must import gracefully when its
18
+ dependency is missing.
19
+ """
20
+
21
+ from .optimizer import MultiOptimizer, Optimizer
22
+
23
+ __all__ = ["MultiOptimizer", "Optimizer"]