eae-runtime 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. eae_runtime-0.1.0/.github/workflows/ci.yml +80 -0
  2. eae_runtime-0.1.0/.github/workflows/publish.yml +75 -0
  3. eae_runtime-0.1.0/.gitignore +28 -0
  4. eae_runtime-0.1.0/LICENSE +13 -0
  5. eae_runtime-0.1.0/PKG-INFO +229 -0
  6. eae_runtime-0.1.0/README.md +192 -0
  7. eae_runtime-0.1.0/eae_runtime/__init__.py +80 -0
  8. eae_runtime-0.1.0/eae_runtime/_version.py +24 -0
  9. eae_runtime-0.1.0/eae_runtime/adjoint.py +128 -0
  10. eae_runtime-0.1.0/eae_runtime/backend.py +52 -0
  11. eae_runtime-0.1.0/eae_runtime/blocks.py +69 -0
  12. eae_runtime-0.1.0/eae_runtime/boundary_store.py +65 -0
  13. eae_runtime-0.1.0/eae_runtime/config.py +48 -0
  14. eae_runtime-0.1.0/eae_runtime/contrib/__init__.py +25 -0
  15. eae_runtime-0.1.0/eae_runtime/contrib/transformer_blocks.py +259 -0
  16. eae_runtime-0.1.0/eae_runtime/events.py +84 -0
  17. eae_runtime-0.1.0/eae_runtime/forward_executor.py +35 -0
  18. eae_runtime-0.1.0/eae_runtime/logging_utils.py +41 -0
  19. eae_runtime-0.1.0/eae_runtime/memory.py +175 -0
  20. eae_runtime-0.1.0/eae_runtime/passes/__init__.py +18 -0
  21. eae_runtime-0.1.0/eae_runtime/passes/base.py +31 -0
  22. eae_runtime-0.1.0/eae_runtime/passes/clip.py +25 -0
  23. eae_runtime-0.1.0/eae_runtime/passes/logging_pass.py +29 -0
  24. eae_runtime-0.1.0/eae_runtime/passes/quantize.py +55 -0
  25. eae_runtime-0.1.0/eae_runtime/passes/regularization.py +47 -0
  26. eae_runtime-0.1.0/eae_runtime/passes/synthetic_gradient.py +67 -0
  27. eae_runtime-0.1.0/eae_runtime/pipeline.py +58 -0
  28. eae_runtime-0.1.0/eae_runtime/profiler.py +47 -0
  29. eae_runtime-0.1.0/eae_runtime/py.typed +0 -0
  30. eae_runtime-0.1.0/eae_runtime/reconstruction.py +89 -0
  31. eae_runtime-0.1.0/eae_runtime/runtime.py +180 -0
  32. eae_runtime-0.1.0/eae_runtime/schedulers/__init__.py +38 -0
  33. eae_runtime-0.1.0/eae_runtime/schedulers/async_scheduler.py +65 -0
  34. eae_runtime-0.1.0/eae_runtime/schedulers/base.py +87 -0
  35. eae_runtime-0.1.0/eae_runtime/schedulers/distributed.py +78 -0
  36. eae_runtime-0.1.0/eae_runtime/schedulers/pipeline_scheduler.py +67 -0
  37. eae_runtime-0.1.0/eae_runtime/schedulers/sequential.py +27 -0
  38. eae_runtime-0.1.0/eae_runtime.egg-info/PKG-INFO +229 -0
  39. eae_runtime-0.1.0/eae_runtime.egg-info/SOURCES.txt +63 -0
  40. eae_runtime-0.1.0/eae_runtime.egg-info/dependency_links.txt +1 -0
  41. eae_runtime-0.1.0/eae_runtime.egg-info/requires.txt +9 -0
  42. eae_runtime-0.1.0/eae_runtime.egg-info/scm_file_list.json +59 -0
  43. eae_runtime-0.1.0/eae_runtime.egg-info/scm_version.json +8 -0
  44. eae_runtime-0.1.0/eae_runtime.egg-info/top_level.txt +1 -0
  45. eae_runtime-0.1.0/examples/train_transformer_lm.py +124 -0
  46. eae_runtime-0.1.0/pyproject.toml +120 -0
  47. eae_runtime-0.1.0/setup.cfg +4 -0
  48. eae_runtime-0.1.0/tests/conftest.py +40 -0
  49. eae_runtime-0.1.0/tests/test_adjoint.py +85 -0
  50. eae_runtime-0.1.0/tests/test_backend.py +57 -0
  51. eae_runtime-0.1.0/tests/test_boundary_store.py +70 -0
  52. eae_runtime-0.1.0/tests/test_config_and_blocks.py +87 -0
  53. eae_runtime-0.1.0/tests/test_contrib_transformer_blocks.py +154 -0
  54. eae_runtime-0.1.0/tests/test_distributed_multiprocess.py +98 -0
  55. eae_runtime-0.1.0/tests/test_end_to_end_transformer.py +124 -0
  56. eae_runtime-0.1.0/tests/test_events.py +51 -0
  57. eae_runtime-0.1.0/tests/test_gradient_equivalence.py +115 -0
  58. eae_runtime-0.1.0/tests/test_memory_leaks.py +77 -0
  59. eae_runtime-0.1.0/tests/test_memory_manager.py +67 -0
  60. eae_runtime-0.1.0/tests/test_mixed_precision.py +64 -0
  61. eae_runtime-0.1.0/tests/test_passes.py +137 -0
  62. eae_runtime-0.1.0/tests/test_profiler.py +47 -0
  63. eae_runtime-0.1.0/tests/test_reconstruction.py +65 -0
  64. eae_runtime-0.1.0/tests/test_runtime.py +68 -0
  65. eae_runtime-0.1.0/tests/test_schedulers.py +155 -0
@@ -0,0 +1,80 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+ workflow_dispatch:
9
+
10
+ jobs:
11
+ test:
12
+ name: test (py${{ matrix.python-version }}, ${{ matrix.os }})
13
+ runs-on: ${{ matrix.os }}
14
+ strategy:
15
+ fail-fast: false
16
+ matrix:
17
+ os: [ubuntu-latest, macos-latest, windows-latest]
18
+ python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
19
+ exclude:
20
+ # CPU-only torch wheels aren't always available same-day for the
21
+ # newest Python on every OS; drop flaky combos here if CI is red.
22
+ - os: windows-latest
23
+ python-version: "3.9"
24
+
25
+ steps:
26
+ - uses: actions/checkout@v4
27
+ with:
28
+ fetch-depth: 0 # setuptools_scm needs full history/tags to compute version
29
+
30
+ - name: Set up Python ${{ matrix.python-version }}
31
+ uses: actions/setup-python@v5
32
+ with:
33
+ python-version: ${{ matrix.python-version }}
34
+ cache: pip
35
+
36
+ - name: Install package + dev deps
37
+ run: python -m pip install -e ".[dev]"
38
+
39
+ - name: Lint (ruff)
40
+ run: ruff check .
41
+
42
+ - name: Type check (mypy)
43
+ run: mypy eae_runtime
44
+
45
+ - name: Test with coverage
46
+ run: pytest --cov --cov-report=xml --cov-report=term-missing
47
+
48
+ - name: Upload coverage
49
+ if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'
50
+ uses: actions/upload-artifact@v4
51
+ with:
52
+ name: coverage-report
53
+ path: coverage.xml
54
+
55
+ build-check:
56
+ name: build sdist + wheel
57
+ runs-on: ubuntu-latest
58
+ needs: test
59
+ steps:
60
+ - uses: actions/checkout@v4
61
+ with:
62
+ fetch-depth: 0
63
+
64
+ - uses: actions/setup-python@v5
65
+ with:
66
+ python-version: "3.12"
67
+
68
+ - name: Install build tooling
69
+ run: python -m pip install build twine
70
+
71
+ - name: Build distributions
72
+ run: python -m build
73
+
74
+ - name: Validate metadata
75
+ run: twine check dist/*
76
+
77
+ - uses: actions/upload-artifact@v4
78
+ with:
79
+ name: dist
80
+ path: dist/
@@ -0,0 +1,75 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*.*.*" # e.g. v0.1.0 -> triggers a real release
7
+ release:
8
+ types: [published] # also allow triggering from a GitHub Release
9
+ workflow_dispatch: # manual "just in case" button, publishes to TestPyPI only
10
+
11
+ jobs:
12
+ build:
13
+ name: Build distribution
14
+ runs-on: ubuntu-latest
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ with:
18
+ fetch-depth: 0 # setuptools_scm needs tags to compute the version
19
+
20
+ - uses: actions/setup-python@v5
21
+ with:
22
+ python-version: "3.12"
23
+
24
+ - name: Install build tooling
25
+ run: python -m pip install build twine
26
+
27
+ - name: Build sdist + wheel
28
+ run: python -m build
29
+
30
+ - name: Check metadata
31
+ run: twine check dist/*
32
+
33
+ - name: Show computed version
34
+ run: python -c "import importlib.metadata as m; print(m.version('eae-runtime'))" || true
35
+
36
+ - uses: actions/upload-artifact@v4
37
+ with:
38
+ name: dist
39
+ path: dist/
40
+
41
+ publish-testpypi:
42
+ name: Publish to TestPyPI (manual trigger only)
43
+ if: github.event_name == 'workflow_dispatch'
44
+ needs: build
45
+ runs-on: ubuntu-latest
46
+ environment:
47
+ name: testpypi
48
+ url: https://test.pypi.org/p/eae-runtime
49
+ permissions:
50
+ id-token: write # required for OIDC trusted publishing
51
+ steps:
52
+ - uses: actions/download-artifact@v4
53
+ with:
54
+ name: dist
55
+ path: dist/
56
+ - uses: pypa/gh-action-pypi-publish@release/v1
57
+ with:
58
+ repository-url: https://test.pypi.org/legacy/
59
+
60
+ publish-pypi:
61
+ name: Publish to PyPI
62
+ if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release'
63
+ needs: build
64
+ runs-on: ubuntu-latest
65
+ environment:
66
+ name: pypi
67
+ url: https://pypi.org/p/eae-runtime
68
+ permissions:
69
+ id-token: write # required for OIDC trusted publishing
70
+ steps:
71
+ - uses: actions/download-artifact@v4
72
+ with:
73
+ name: dist
74
+ path: dist/
75
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,28 @@
1
+ # Local virtualenvs
2
+ eae_env/
3
+ .venv/
4
+ venv/
5
+
6
+ # Python build artifacts
7
+ __pycache__/
8
+ *.pyc
9
+ *.pyo
10
+ *.egg-info/
11
+ build/
12
+ dist/
13
+ *.egg
14
+
15
+ # setuptools_scm-generated version file (derived from git tags at build time)
16
+ eae_runtime/_version.py
17
+
18
+ # Test / coverage / type-checker caches
19
+ .pytest_cache/
20
+ .mypy_cache/
21
+ .ruff_cache/
22
+ .coverage
23
+ htmlcov/
24
+
25
+ # Editors / OS
26
+ .vscode/
27
+ .idea/
28
+ .DS_Store
@@ -0,0 +1,13 @@
1
+ Copyright 2026 Vladimer Khasia
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
@@ -0,0 +1,229 @@
1
+ Metadata-Version: 2.4
2
+ Name: eae-runtime
3
+ Version: 0.1.0
4
+ Summary: A programmable, block-local reverse-mode execution engine that orchestrates PyTorch's local autograd (Explicit Adjoint Exposure runtime).
5
+ Author: Vladimer Khasia
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/VladimerKhasia/eae_runtime
8
+ Project-URL: Repository, https://github.com/VladimerKhasia/eae_runtime
9
+ Project-URL: Issues, https://github.com/VladimerKhasia/eae_runtime/issues
10
+ Project-URL: Changelog, https://github.com/VladimerKhasia/eae_runtime/blob/main/CHANGELOG.md
11
+ Keywords: pytorch,autograd,backpropagation,transformer,deep-learning,training-runtime,gradient-compression,distributed-training
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
16
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3.9
20
+ Classifier: Programming Language :: Python :: 3.10
21
+ Classifier: Programming Language :: Python :: 3.11
22
+ Classifier: Programming Language :: Python :: 3.12
23
+ Classifier: Programming Language :: Python :: 3.13
24
+ Classifier: Typing :: Typed
25
+ Requires-Python: >=3.9
26
+ Description-Content-Type: text/markdown
27
+ License-File: LICENSE
28
+ Requires-Dist: torch>=2.0
29
+ Provides-Extra: dev
30
+ Requires-Dist: pytest>=7.0; extra == "dev"
31
+ Requires-Dist: pytest-cov>=4.0; extra == "dev"
32
+ Requires-Dist: ruff>=0.6; extra == "dev"
33
+ Requires-Dist: mypy>=1.10; extra == "dev"
34
+ Requires-Dist: build>=1.0; extra == "dev"
35
+ Requires-Dist: twine>=5.0; extra == "dev"
36
+ Dynamic: license-file
37
+
38
+ # A programmable reverse-mode runtime for PyTorch
39
+
40
+ A production-quality runtime that replaces PyTorch's monolithic
41
+ `backward()` with a programmable, block-local reverse-mode execution engine
42
+ implementing **Explicit Adjoint Exposure (EAE)**. It builds on the the top of [EAE](https://github.com/VladimerKhasia/eae) and turnes reverse-mode execution itself into schedulable runtime.
43
+
44
+ The runtime is **not** a new autodiff engine. It orchestrates many local
45
+ autograd computations using PyTorch's existing local autograd:
46
+
47
+ * **PyTorch owns:** local autograd, VJP computation, kernels.
48
+ * **The runtime owns:** forward scheduling, reverse scheduling,
49
+ reconstruction, adjoint lifecycle, memory lifecycle, pass execution,
50
+ backend selection, communication scheduling.
51
+
52
+ Users write ordinary `nn.Sequential` / `nn.Module` models. No custom tensor
53
+ type, no compiler, no FX, no PyTorch internals modified.
54
+
55
+ ## Install
56
+
57
+ ```bash
58
+ pip install -e .
59
+ ```
60
+
61
+ ## Quick start
62
+
63
+ ```python
64
+ import torch
65
+ import torch.nn as nn
66
+ from eae_runtime import EAERuntime, RuntimeConfig
67
+ from eae_runtime.passes import ClipPass, Int8QuantizationPass
68
+
69
+ model = nn.Sequential(
70
+ nn.Linear(64, 64), nn.ReLU(),
71
+ nn.Linear(64, 64), nn.ReLU(),
72
+ nn.Linear(64, 10),
73
+ )
74
+ optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
75
+
76
+ config = RuntimeConfig(
77
+ scheduler="sequential", # "sequential" | "async" | "pipeline" | "distributed"
78
+ memory="pool", # "pool" | "none"
79
+ backend="auto", # "auto" | "cpu" | "cuda" | "triton" | "rocm"
80
+ passes=[Int8QuantizationPass(), ClipPass(max_norm=1.0)],
81
+ boundary_offload=False, # move x0..xL to CPU between fwd/bwd
82
+ boundary_precision=None, # e.g. torch.float16 to shrink boundary storage
83
+ )
84
+
85
+ runtime = EAERuntime(model, optimizer, config)
86
+
87
+ x = torch.randn(32, 64)
88
+ target = torch.randint(0, 10, (32,))
89
+ loss = runtime.train_step(x, lambda out: nn.functional.cross_entropy(out, target))
90
+ ```
91
+
92
+ ## Architecture
93
+
94
+ ```
95
+ User Model
96
+ |
97
+ Block Decomposer (blocks.py)
98
+ |
99
+ Forward Executor (forward_executor.py) -> detached forward, no autograd graph
100
+ |
101
+ Boundary State Store (boundary_store.py) -> stores only x0..xL
102
+ |
103
+ Reverse Scheduler (schedulers/) -> pluggable: sequential/async/pipeline/distributed
104
+ | \
105
+ | \
106
+ Reconstruction Memory Manager (reconstruction.py / memory.py)
107
+ Engine |
108
+ | |
109
+ Adjoint Pipeline Backend Manager (pipeline.py / backend.py)
110
+ |
111
+ Local Autograd (PyTorch)
112
+ |
113
+ CPU / CUDA
114
+ ```
115
+
116
+ ### The Adjoint Pipeline (the one addition beyond the base spec)
117
+
118
+ Every reverse step exchanges an `AdjointState` — never a raw tensor —
119
+ through a programmable pipeline:
120
+
121
+ ```
122
+ AdjointState -> Pass1 -> Pass2 -> ... -> PassN -> next local VJP
123
+ ```
124
+
125
+ A researcher working on gradient compression, synthetic gradients,
126
+ error-feedback, distributed communication, or adaptive optimization only
127
+ implements a new `EAEPass`. They never touch the reconstruction engine,
128
+ scheduler, or memory manager.
129
+
130
+ ```python
131
+ from eae_runtime.passes import EAEPass
132
+
133
+ class MyPass(EAEPass):
134
+ name = "MyPass"
135
+ def process(self, adjoint, context=None):
136
+ new = adjoint.clone()
137
+ new.tensor = new.tensor * 0.9 # e.g. decay
138
+ return new
139
+ ```
140
+
141
+ ### Extension points
142
+
143
+ | Want to add... | Subclass | Where |
144
+ |--------------------------|--------------------------|----------------------------|
145
+ | A gradient transform | `EAEPass` | `eae_runtime/passes/` |
146
+ | A reverse scheduling policy | `BaseScheduler` | `eae_runtime/schedulers/` |
147
+ | A memory allocation strategy | `MemoryPolicy` | `eae_runtime/memory.py` |
148
+ | A new backend | extend `BackendManager` | `eae_runtime/backend.py` |
149
+
150
+ None of these require modifying the runtime core.
151
+
152
+ ### Built-in passes
153
+
154
+ `ClipPass`, `FP16Pass`, `Int8QuantizationPass` (aliased as `FP8Pass`),
155
+ `SyntheticGradientPass`, `RegularizationPass`, `GaussianNoisePass`,
156
+ `LoggingPass`.
157
+
158
+ ### Built-in schedulers
159
+
160
+ * `SequentialScheduler` — the reference implementation; strict in-order
161
+ reconstruct → VJP → pipeline → free.
162
+ * `AsyncScheduler` — overlaps boundary-activation prefetch (useful with
163
+ `boundary_offload=True`) with reconstruction compute; numerically
164
+ identical to `SequentialScheduler`.
165
+ * `PipelineScheduler` — splits the batch into microbatches and runs a full
166
+ reverse pass per microbatch, accumulating gradients (single-process
167
+ emulation of pipeline-parallel microbatching).
168
+ * `DistributedScheduler` — assigns block ranges to `torch.distributed`
169
+ ranks and communicates adjoints across rank boundaries; falls back to
170
+ `SequentialScheduler` (with a warning) when running single-process.
171
+
172
+ ### Events
173
+
174
+ Every component emits structured events (`ForwardStarted`,
175
+ `BlockReconstructed`, `AdjointCreated`, `MemoryAllocated`,
176
+ `SchedulerStep`, `PassApplied`, ...) on an `EventBus`. Subscribe without
177
+ touching runtime code:
178
+
179
+ ```python
180
+ runtime.event_bus.subscribe("BlockReconstructed", lambda e: print(e.payload))
181
+ ```
182
+
183
+ Set `RuntimeConfig(log_level="DEBUG")` to also pipe every event through
184
+ Python's structured JSON logger.
185
+
186
+ ### Profiling
187
+
188
+ ```python
189
+ loss = runtime.train_step(x, loss_fn)
190
+ print(runtime.profiler.report())
191
+ # {'forward': {...}, 'reconstruct:Linear_0': {...}, 'pass:ClipPass': {...}, ...}
192
+ ```
193
+
194
+ ## Testing
195
+
196
+ ```bash
197
+ pip install -e ".[dev]"
198
+ pytest
199
+ ```
200
+
201
+ The suite (`tests/`) covers:
202
+
203
+ * `AdjointState` API (norm, statistics, quantize/dequantize, compress, clone, detach)
204
+ * Event bus pub/sub, wildcard subscribers, recording
205
+ * Memory manager: pool reuse, null policy, allocator correctness, leak checks
206
+ * Boundary store: precision downcast, CPU offload, ownership semantics
207
+ * Reconstruction engine vs. manual VJP (`torch.autograd.grad`) reference
208
+ * **Gradient equivalence**: every scheduler vs. plain `model.backward()`,
209
+ across float32, boundary-offloaded, and microbatched configurations
210
+ * All built-in passes, including a full multi-pass pipeline
211
+ * All built-in schedulers, plus a user-defined custom scheduler
212
+ * Backend manager device/dtype resolution and graceful CUDA→CPU fallback
213
+ * Profiler timing aggregation
214
+ * Mixed precision (compute dtype + boundary storage dtype) numerical bounds
215
+ * Memory-leak / allocator-stability checks over many training steps
216
+ * A full Transformer-style block stack (`nn.MultiheadAttention` + FFN,
217
+ pre-norm, residual) trained end-to-end with a full pass pipeline
218
+ * Real two-process `torch.distributed` (gloo) correctness check for
219
+ `DistributedScheduler` (skips gracefully if the sandbox blocks socket use)
220
+
221
+ ## Non-goals by design
222
+
223
+ * Automatic partitioning of arbitrary `nn.Module` graphs — blocks are
224
+ specified manually or via `BlockDecomposer.from_sequential`/`from_list`.
225
+ * Replacing or modifying PyTorch's autograd internals.
226
+ * A new differentiation algorithm — local VJP is always PyTorch's own.
227
+ * A new compiler or graph IR.
228
+ * Day-one support for every architecture — the runtime targets
229
+ Transformer-style models with repeated block structure first.
@@ -0,0 +1,192 @@
1
+ # A programmable reverse-mode runtime for PyTorch
2
+
3
+ A production-quality runtime that replaces PyTorch's monolithic
4
+ `backward()` with a programmable, block-local reverse-mode execution engine
5
+ implementing **Explicit Adjoint Exposure (EAE)**. It builds on the the top of [EAE](https://github.com/VladimerKhasia/eae) and turnes reverse-mode execution itself into schedulable runtime.
6
+
7
+ The runtime is **not** a new autodiff engine. It orchestrates many local
8
+ autograd computations using PyTorch's existing local autograd:
9
+
10
+ * **PyTorch owns:** local autograd, VJP computation, kernels.
11
+ * **The runtime owns:** forward scheduling, reverse scheduling,
12
+ reconstruction, adjoint lifecycle, memory lifecycle, pass execution,
13
+ backend selection, communication scheduling.
14
+
15
+ Users write ordinary `nn.Sequential` / `nn.Module` models. No custom tensor
16
+ type, no compiler, no FX, no PyTorch internals modified.
17
+
18
+ ## Install
19
+
20
+ ```bash
21
+ pip install -e .
22
+ ```
23
+
24
+ ## Quick start
25
+
26
+ ```python
27
+ import torch
28
+ import torch.nn as nn
29
+ from eae_runtime import EAERuntime, RuntimeConfig
30
+ from eae_runtime.passes import ClipPass, Int8QuantizationPass
31
+
32
+ model = nn.Sequential(
33
+ nn.Linear(64, 64), nn.ReLU(),
34
+ nn.Linear(64, 64), nn.ReLU(),
35
+ nn.Linear(64, 10),
36
+ )
37
+ optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
38
+
39
+ config = RuntimeConfig(
40
+ scheduler="sequential", # "sequential" | "async" | "pipeline" | "distributed"
41
+ memory="pool", # "pool" | "none"
42
+ backend="auto", # "auto" | "cpu" | "cuda" | "triton" | "rocm"
43
+ passes=[Int8QuantizationPass(), ClipPass(max_norm=1.0)],
44
+ boundary_offload=False, # move x0..xL to CPU between fwd/bwd
45
+ boundary_precision=None, # e.g. torch.float16 to shrink boundary storage
46
+ )
47
+
48
+ runtime = EAERuntime(model, optimizer, config)
49
+
50
+ x = torch.randn(32, 64)
51
+ target = torch.randint(0, 10, (32,))
52
+ loss = runtime.train_step(x, lambda out: nn.functional.cross_entropy(out, target))
53
+ ```
54
+
55
+ ## Architecture
56
+
57
+ ```
58
+ User Model
59
+ |
60
+ Block Decomposer (blocks.py)
61
+ |
62
+ Forward Executor (forward_executor.py) -> detached forward, no autograd graph
63
+ |
64
+ Boundary State Store (boundary_store.py) -> stores only x0..xL
65
+ |
66
+ Reverse Scheduler (schedulers/) -> pluggable: sequential/async/pipeline/distributed
67
+ | \
68
+ | \
69
+ Reconstruction Memory Manager (reconstruction.py / memory.py)
70
+ Engine |
71
+ | |
72
+ Adjoint Pipeline Backend Manager (pipeline.py / backend.py)
73
+ |
74
+ Local Autograd (PyTorch)
75
+ |
76
+ CPU / CUDA
77
+ ```
78
+
79
+ ### The Adjoint Pipeline (the one addition beyond the base spec)
80
+
81
+ Every reverse step exchanges an `AdjointState` — never a raw tensor —
82
+ through a programmable pipeline:
83
+
84
+ ```
85
+ AdjointState -> Pass1 -> Pass2 -> ... -> PassN -> next local VJP
86
+ ```
87
+
88
+ A researcher working on gradient compression, synthetic gradients,
89
+ error-feedback, distributed communication, or adaptive optimization only
90
+ implements a new `EAEPass`. They never touch the reconstruction engine,
91
+ scheduler, or memory manager.
92
+
93
+ ```python
94
+ from eae_runtime.passes import EAEPass
95
+
96
+ class MyPass(EAEPass):
97
+ name = "MyPass"
98
+ def process(self, adjoint, context=None):
99
+ new = adjoint.clone()
100
+ new.tensor = new.tensor * 0.9 # e.g. decay
101
+ return new
102
+ ```
103
+
104
+ ### Extension points
105
+
106
+ | Want to add... | Subclass | Where |
107
+ |--------------------------|--------------------------|----------------------------|
108
+ | A gradient transform | `EAEPass` | `eae_runtime/passes/` |
109
+ | A reverse scheduling policy | `BaseScheduler` | `eae_runtime/schedulers/` |
110
+ | A memory allocation strategy | `MemoryPolicy` | `eae_runtime/memory.py` |
111
+ | A new backend | extend `BackendManager` | `eae_runtime/backend.py` |
112
+
113
+ None of these require modifying the runtime core.
114
+
115
+ ### Built-in passes
116
+
117
+ `ClipPass`, `FP16Pass`, `Int8QuantizationPass` (aliased as `FP8Pass`),
118
+ `SyntheticGradientPass`, `RegularizationPass`, `GaussianNoisePass`,
119
+ `LoggingPass`.
120
+
121
+ ### Built-in schedulers
122
+
123
+ * `SequentialScheduler` — the reference implementation; strict in-order
124
+ reconstruct → VJP → pipeline → free.
125
+ * `AsyncScheduler` — overlaps boundary-activation prefetch (useful with
126
+ `boundary_offload=True`) with reconstruction compute; numerically
127
+ identical to `SequentialScheduler`.
128
+ * `PipelineScheduler` — splits the batch into microbatches and runs a full
129
+ reverse pass per microbatch, accumulating gradients (single-process
130
+ emulation of pipeline-parallel microbatching).
131
+ * `DistributedScheduler` — assigns block ranges to `torch.distributed`
132
+ ranks and communicates adjoints across rank boundaries; falls back to
133
+ `SequentialScheduler` (with a warning) when running single-process.
134
+
135
+ ### Events
136
+
137
+ Every component emits structured events (`ForwardStarted`,
138
+ `BlockReconstructed`, `AdjointCreated`, `MemoryAllocated`,
139
+ `SchedulerStep`, `PassApplied`, ...) on an `EventBus`. Subscribe without
140
+ touching runtime code:
141
+
142
+ ```python
143
+ runtime.event_bus.subscribe("BlockReconstructed", lambda e: print(e.payload))
144
+ ```
145
+
146
+ Set `RuntimeConfig(log_level="DEBUG")` to also pipe every event through
147
+ Python's structured JSON logger.
148
+
149
+ ### Profiling
150
+
151
+ ```python
152
+ loss = runtime.train_step(x, loss_fn)
153
+ print(runtime.profiler.report())
154
+ # {'forward': {...}, 'reconstruct:Linear_0': {...}, 'pass:ClipPass': {...}, ...}
155
+ ```
156
+
157
+ ## Testing
158
+
159
+ ```bash
160
+ pip install -e ".[dev]"
161
+ pytest
162
+ ```
163
+
164
+ The suite (`tests/`) covers:
165
+
166
+ * `AdjointState` API (norm, statistics, quantize/dequantize, compress, clone, detach)
167
+ * Event bus pub/sub, wildcard subscribers, recording
168
+ * Memory manager: pool reuse, null policy, allocator correctness, leak checks
169
+ * Boundary store: precision downcast, CPU offload, ownership semantics
170
+ * Reconstruction engine vs. manual VJP (`torch.autograd.grad`) reference
171
+ * **Gradient equivalence**: every scheduler vs. plain `model.backward()`,
172
+ across float32, boundary-offloaded, and microbatched configurations
173
+ * All built-in passes, including a full multi-pass pipeline
174
+ * All built-in schedulers, plus a user-defined custom scheduler
175
+ * Backend manager device/dtype resolution and graceful CUDA→CPU fallback
176
+ * Profiler timing aggregation
177
+ * Mixed precision (compute dtype + boundary storage dtype) numerical bounds
178
+ * Memory-leak / allocator-stability checks over many training steps
179
+ * A full Transformer-style block stack (`nn.MultiheadAttention` + FFN,
180
+ pre-norm, residual) trained end-to-end with a full pass pipeline
181
+ * Real two-process `torch.distributed` (gloo) correctness check for
182
+ `DistributedScheduler` (skips gracefully if the sandbox blocks socket use)
183
+
184
+ ## Non-goals by design
185
+
186
+ * Automatic partitioning of arbitrary `nn.Module` graphs — blocks are
187
+ specified manually or via `BlockDecomposer.from_sequential`/`from_list`.
188
+ * Replacing or modifying PyTorch's autograd internals.
189
+ * A new differentiation algorithm — local VJP is always PyTorch's own.
190
+ * A new compiler or graph IR.
191
+ * Day-one support for every architecture — the runtime targets
192
+ Transformer-style models with repeated block structure first.
@@ -0,0 +1,80 @@
1
+ """
2
+ EAE Runtime: a standalone, programmable, block-local reverse-mode execution
3
+ engine that sits *above* PyTorch and orchestrates many local autograd
4
+ computations, instead of replacing PyTorch's autodiff engine.
5
+
6
+ Quick start:
7
+
8
+ import torch.nn as nn
9
+ from eae_runtime import EAERuntime, RuntimeConfig
10
+ from eae_runtime.passes import ClipPass, GaussianNoisePass
11
+
12
+ model = nn.Sequential(
13
+ nn.Linear(64, 64), nn.ReLU(),
14
+ nn.Linear(64, 64), nn.ReLU(),
15
+ nn.Linear(64, 10),
16
+ )
17
+ optimizer = torch.optim.SGD(model.parameters(), lr=1e-2)
18
+ config = RuntimeConfig(
19
+ scheduler="sequential",
20
+ memory="pool",
21
+ backend="auto",
22
+ passes=[ClipPass(max_norm=1.0)],
23
+ )
24
+ runtime = EAERuntime(model, optimizer, config)
25
+ loss = runtime.train_step(x, lambda out: nn.functional.mse_loss(out, target))
26
+ """
27
+
28
+ from .adjoint import AdjointState
29
+ from .backend import BackendManager
30
+ from .blocks import BlockDecomposer, EAEBlock
31
+ from .boundary_store import BoundaryStore
32
+ from .config import RuntimeConfig
33
+ from .events import Event, EventBus, EventType
34
+ from .forward_executor import ForwardExecutor
35
+ from .memory import MemoryManager, MemoryPolicy, NullMemoryPolicy, PoolMemoryPolicy
36
+ from .pipeline import AdjointPipeline, PassManager
37
+ from .profiler import Profiler
38
+ from .reconstruction import ReconstructionEngine
39
+ from .runtime import EAERuntime
40
+ from .schedulers import (
41
+ AsyncScheduler,
42
+ BaseScheduler,
43
+ DistributedScheduler,
44
+ PipelineScheduler,
45
+ SequentialScheduler,
46
+ build_scheduler,
47
+ )
48
+
49
+ try:
50
+ from ._version import version as __version__
51
+ except ImportError: # pragma: no cover - only hit in a non-built source tree
52
+ __version__ = "0.0.0+unknown"
53
+
54
+ __all__ = [
55
+ "AdjointState",
56
+ "BackendManager",
57
+ "BlockDecomposer",
58
+ "EAEBlock",
59
+ "BoundaryStore",
60
+ "RuntimeConfig",
61
+ "Event",
62
+ "EventBus",
63
+ "EventType",
64
+ "ForwardExecutor",
65
+ "MemoryManager",
66
+ "MemoryPolicy",
67
+ "NullMemoryPolicy",
68
+ "PoolMemoryPolicy",
69
+ "AdjointPipeline",
70
+ "PassManager",
71
+ "Profiler",
72
+ "ReconstructionEngine",
73
+ "EAERuntime",
74
+ "AsyncScheduler",
75
+ "BaseScheduler",
76
+ "DistributedScheduler",
77
+ "PipelineScheduler",
78
+ "SequentialScheduler",
79
+ "build_scheduler",
80
+ ]