eae-runtime 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eae_runtime-0.1.0/.github/workflows/ci.yml +80 -0
- eae_runtime-0.1.0/.github/workflows/publish.yml +75 -0
- eae_runtime-0.1.0/.gitignore +28 -0
- eae_runtime-0.1.0/LICENSE +13 -0
- eae_runtime-0.1.0/PKG-INFO +229 -0
- eae_runtime-0.1.0/README.md +192 -0
- eae_runtime-0.1.0/eae_runtime/__init__.py +80 -0
- eae_runtime-0.1.0/eae_runtime/_version.py +24 -0
- eae_runtime-0.1.0/eae_runtime/adjoint.py +128 -0
- eae_runtime-0.1.0/eae_runtime/backend.py +52 -0
- eae_runtime-0.1.0/eae_runtime/blocks.py +69 -0
- eae_runtime-0.1.0/eae_runtime/boundary_store.py +65 -0
- eae_runtime-0.1.0/eae_runtime/config.py +48 -0
- eae_runtime-0.1.0/eae_runtime/contrib/__init__.py +25 -0
- eae_runtime-0.1.0/eae_runtime/contrib/transformer_blocks.py +259 -0
- eae_runtime-0.1.0/eae_runtime/events.py +84 -0
- eae_runtime-0.1.0/eae_runtime/forward_executor.py +35 -0
- eae_runtime-0.1.0/eae_runtime/logging_utils.py +41 -0
- eae_runtime-0.1.0/eae_runtime/memory.py +175 -0
- eae_runtime-0.1.0/eae_runtime/passes/__init__.py +18 -0
- eae_runtime-0.1.0/eae_runtime/passes/base.py +31 -0
- eae_runtime-0.1.0/eae_runtime/passes/clip.py +25 -0
- eae_runtime-0.1.0/eae_runtime/passes/logging_pass.py +29 -0
- eae_runtime-0.1.0/eae_runtime/passes/quantize.py +55 -0
- eae_runtime-0.1.0/eae_runtime/passes/regularization.py +47 -0
- eae_runtime-0.1.0/eae_runtime/passes/synthetic_gradient.py +67 -0
- eae_runtime-0.1.0/eae_runtime/pipeline.py +58 -0
- eae_runtime-0.1.0/eae_runtime/profiler.py +47 -0
- eae_runtime-0.1.0/eae_runtime/py.typed +0 -0
- eae_runtime-0.1.0/eae_runtime/reconstruction.py +89 -0
- eae_runtime-0.1.0/eae_runtime/runtime.py +180 -0
- eae_runtime-0.1.0/eae_runtime/schedulers/__init__.py +38 -0
- eae_runtime-0.1.0/eae_runtime/schedulers/async_scheduler.py +65 -0
- eae_runtime-0.1.0/eae_runtime/schedulers/base.py +87 -0
- eae_runtime-0.1.0/eae_runtime/schedulers/distributed.py +78 -0
- eae_runtime-0.1.0/eae_runtime/schedulers/pipeline_scheduler.py +67 -0
- eae_runtime-0.1.0/eae_runtime/schedulers/sequential.py +27 -0
- eae_runtime-0.1.0/eae_runtime.egg-info/PKG-INFO +229 -0
- eae_runtime-0.1.0/eae_runtime.egg-info/SOURCES.txt +63 -0
- eae_runtime-0.1.0/eae_runtime.egg-info/dependency_links.txt +1 -0
- eae_runtime-0.1.0/eae_runtime.egg-info/requires.txt +9 -0
- eae_runtime-0.1.0/eae_runtime.egg-info/scm_file_list.json +59 -0
- eae_runtime-0.1.0/eae_runtime.egg-info/scm_version.json +8 -0
- eae_runtime-0.1.0/eae_runtime.egg-info/top_level.txt +1 -0
- eae_runtime-0.1.0/examples/train_transformer_lm.py +124 -0
- eae_runtime-0.1.0/pyproject.toml +120 -0
- eae_runtime-0.1.0/setup.cfg +4 -0
- eae_runtime-0.1.0/tests/conftest.py +40 -0
- eae_runtime-0.1.0/tests/test_adjoint.py +85 -0
- eae_runtime-0.1.0/tests/test_backend.py +57 -0
- eae_runtime-0.1.0/tests/test_boundary_store.py +70 -0
- eae_runtime-0.1.0/tests/test_config_and_blocks.py +87 -0
- eae_runtime-0.1.0/tests/test_contrib_transformer_blocks.py +154 -0
- eae_runtime-0.1.0/tests/test_distributed_multiprocess.py +98 -0
- eae_runtime-0.1.0/tests/test_end_to_end_transformer.py +124 -0
- eae_runtime-0.1.0/tests/test_events.py +51 -0
- eae_runtime-0.1.0/tests/test_gradient_equivalence.py +115 -0
- eae_runtime-0.1.0/tests/test_memory_leaks.py +77 -0
- eae_runtime-0.1.0/tests/test_memory_manager.py +67 -0
- eae_runtime-0.1.0/tests/test_mixed_precision.py +64 -0
- eae_runtime-0.1.0/tests/test_passes.py +137 -0
- eae_runtime-0.1.0/tests/test_profiler.py +47 -0
- eae_runtime-0.1.0/tests/test_reconstruction.py +65 -0
- eae_runtime-0.1.0/tests/test_runtime.py +68 -0
- eae_runtime-0.1.0/tests/test_schedulers.py +155 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
test:
|
|
12
|
+
name: test (py${{ matrix.python-version }}, ${{ matrix.os }})
|
|
13
|
+
runs-on: ${{ matrix.os }}
|
|
14
|
+
strategy:
|
|
15
|
+
fail-fast: false
|
|
16
|
+
matrix:
|
|
17
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
18
|
+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
|
19
|
+
exclude:
|
|
20
|
+
# CPU-only torch wheels aren't always available same-day for the
|
|
21
|
+
# newest Python on every OS; drop flaky combos here if CI is red.
|
|
22
|
+
- os: windows-latest
|
|
23
|
+
python-version: "3.9"
|
|
24
|
+
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
with:
|
|
28
|
+
fetch-depth: 0 # setuptools_scm needs full history/tags to compute version
|
|
29
|
+
|
|
30
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
31
|
+
uses: actions/setup-python@v5
|
|
32
|
+
with:
|
|
33
|
+
python-version: ${{ matrix.python-version }}
|
|
34
|
+
cache: pip
|
|
35
|
+
|
|
36
|
+
- name: Install package + dev deps
|
|
37
|
+
run: python -m pip install -e ".[dev]"
|
|
38
|
+
|
|
39
|
+
- name: Lint (ruff)
|
|
40
|
+
run: ruff check .
|
|
41
|
+
|
|
42
|
+
- name: Type check (mypy)
|
|
43
|
+
run: mypy eae_runtime
|
|
44
|
+
|
|
45
|
+
- name: Test with coverage
|
|
46
|
+
run: pytest --cov --cov-report=xml --cov-report=term-missing
|
|
47
|
+
|
|
48
|
+
- name: Upload coverage
|
|
49
|
+
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'
|
|
50
|
+
uses: actions/upload-artifact@v4
|
|
51
|
+
with:
|
|
52
|
+
name: coverage-report
|
|
53
|
+
path: coverage.xml
|
|
54
|
+
|
|
55
|
+
build-check:
|
|
56
|
+
name: build sdist + wheel
|
|
57
|
+
runs-on: ubuntu-latest
|
|
58
|
+
needs: test
|
|
59
|
+
steps:
|
|
60
|
+
- uses: actions/checkout@v4
|
|
61
|
+
with:
|
|
62
|
+
fetch-depth: 0
|
|
63
|
+
|
|
64
|
+
- uses: actions/setup-python@v5
|
|
65
|
+
with:
|
|
66
|
+
python-version: "3.12"
|
|
67
|
+
|
|
68
|
+
- name: Install build tooling
|
|
69
|
+
run: python -m pip install build twine
|
|
70
|
+
|
|
71
|
+
- name: Build distributions
|
|
72
|
+
run: python -m build
|
|
73
|
+
|
|
74
|
+
- name: Validate metadata
|
|
75
|
+
run: twine check dist/*
|
|
76
|
+
|
|
77
|
+
- uses: actions/upload-artifact@v4
|
|
78
|
+
with:
|
|
79
|
+
name: dist
|
|
80
|
+
path: dist/
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*.*.*" # e.g. v0.1.0 -> triggers a real release
|
|
7
|
+
release:
|
|
8
|
+
types: [published] # also allow triggering from a GitHub Release
|
|
9
|
+
workflow_dispatch: # manual "just in case" button, publishes to TestPyPI only
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build:
|
|
13
|
+
name: Build distribution
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
with:
|
|
18
|
+
fetch-depth: 0 # setuptools_scm needs tags to compute the version
|
|
19
|
+
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: "3.12"
|
|
23
|
+
|
|
24
|
+
- name: Install build tooling
|
|
25
|
+
run: python -m pip install build twine
|
|
26
|
+
|
|
27
|
+
- name: Build sdist + wheel
|
|
28
|
+
run: python -m build
|
|
29
|
+
|
|
30
|
+
- name: Check metadata
|
|
31
|
+
run: twine check dist/*
|
|
32
|
+
|
|
33
|
+
- name: Show computed version
|
|
34
|
+
run: python -c "import importlib.metadata as m; print(m.version('eae-runtime'))" || true
|
|
35
|
+
|
|
36
|
+
- uses: actions/upload-artifact@v4
|
|
37
|
+
with:
|
|
38
|
+
name: dist
|
|
39
|
+
path: dist/
|
|
40
|
+
|
|
41
|
+
publish-testpypi:
|
|
42
|
+
name: Publish to TestPyPI (manual trigger only)
|
|
43
|
+
if: github.event_name == 'workflow_dispatch'
|
|
44
|
+
needs: build
|
|
45
|
+
runs-on: ubuntu-latest
|
|
46
|
+
environment:
|
|
47
|
+
name: testpypi
|
|
48
|
+
url: https://test.pypi.org/p/eae-runtime
|
|
49
|
+
permissions:
|
|
50
|
+
id-token: write # required for OIDC trusted publishing
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/download-artifact@v4
|
|
53
|
+
with:
|
|
54
|
+
name: dist
|
|
55
|
+
path: dist/
|
|
56
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
57
|
+
with:
|
|
58
|
+
repository-url: https://test.pypi.org/legacy/
|
|
59
|
+
|
|
60
|
+
publish-pypi:
|
|
61
|
+
name: Publish to PyPI
|
|
62
|
+
if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release'
|
|
63
|
+
needs: build
|
|
64
|
+
runs-on: ubuntu-latest
|
|
65
|
+
environment:
|
|
66
|
+
name: pypi
|
|
67
|
+
url: https://pypi.org/p/eae-runtime
|
|
68
|
+
permissions:
|
|
69
|
+
id-token: write # required for OIDC trusted publishing
|
|
70
|
+
steps:
|
|
71
|
+
- uses: actions/download-artifact@v4
|
|
72
|
+
with:
|
|
73
|
+
name: dist
|
|
74
|
+
path: dist/
|
|
75
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Local virtualenvs
|
|
2
|
+
eae_env/
|
|
3
|
+
.venv/
|
|
4
|
+
venv/
|
|
5
|
+
|
|
6
|
+
# Python build artifacts
|
|
7
|
+
__pycache__/
|
|
8
|
+
*.pyc
|
|
9
|
+
*.pyo
|
|
10
|
+
*.egg-info/
|
|
11
|
+
build/
|
|
12
|
+
dist/
|
|
13
|
+
*.egg
|
|
14
|
+
|
|
15
|
+
# setuptools_scm-generated version file (derived from git tags at build time)
|
|
16
|
+
eae_runtime/_version.py
|
|
17
|
+
|
|
18
|
+
# Test / coverage / type-checker caches
|
|
19
|
+
.pytest_cache/
|
|
20
|
+
.mypy_cache/
|
|
21
|
+
.ruff_cache/
|
|
22
|
+
.coverage
|
|
23
|
+
htmlcov/
|
|
24
|
+
|
|
25
|
+
# Editors / OS
|
|
26
|
+
.vscode/
|
|
27
|
+
.idea/
|
|
28
|
+
.DS_Store
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Copyright 2026 Vladimer Khasia
|
|
2
|
+
|
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
you may not use this file except in compliance with the License.
|
|
5
|
+
You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
See the License for the specific language governing permissions and
|
|
13
|
+
limitations under the License.
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: eae-runtime
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A programmable, block-local reverse-mode execution engine that orchestrates PyTorch's local autograd (Explicit Adjoint Exposure runtime).
|
|
5
|
+
Author: Vladimer Khasia
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/VladimerKhasia/eae_runtime
|
|
8
|
+
Project-URL: Repository, https://github.com/VladimerKhasia/eae_runtime
|
|
9
|
+
Project-URL: Issues, https://github.com/VladimerKhasia/eae_runtime/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/VladimerKhasia/eae_runtime/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: pytorch,autograd,backpropagation,transformer,deep-learning,training-runtime,gradient-compression,distributed-training
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: torch>=2.0
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
31
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
32
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
33
|
+
Requires-Dist: mypy>=1.10; extra == "dev"
|
|
34
|
+
Requires-Dist: build>=1.0; extra == "dev"
|
|
35
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
# A programmable reverse-mode runtime for PyTorch
|
|
39
|
+
|
|
40
|
+
A production-quality runtime that replaces PyTorch's monolithic
|
|
41
|
+
`backward()` with a programmable, block-local reverse-mode execution engine
|
|
42
|
+
implementing **Explicit Adjoint Exposure (EAE)**. It builds on the the top of [EAE](https://github.com/VladimerKhasia/eae) and turnes reverse-mode execution itself into schedulable runtime.
|
|
43
|
+
|
|
44
|
+
The runtime is **not** a new autodiff engine. It orchestrates many local
|
|
45
|
+
autograd computations using PyTorch's existing local autograd:
|
|
46
|
+
|
|
47
|
+
* **PyTorch owns:** local autograd, VJP computation, kernels.
|
|
48
|
+
* **The runtime owns:** forward scheduling, reverse scheduling,
|
|
49
|
+
reconstruction, adjoint lifecycle, memory lifecycle, pass execution,
|
|
50
|
+
backend selection, communication scheduling.
|
|
51
|
+
|
|
52
|
+
Users write ordinary `nn.Sequential` / `nn.Module` models. No custom tensor
|
|
53
|
+
type, no compiler, no FX, no PyTorch internals modified.
|
|
54
|
+
|
|
55
|
+
## Install
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install -e .
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Quick start
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
import torch
|
|
65
|
+
import torch.nn as nn
|
|
66
|
+
from eae_runtime import EAERuntime, RuntimeConfig
|
|
67
|
+
from eae_runtime.passes import ClipPass, Int8QuantizationPass
|
|
68
|
+
|
|
69
|
+
model = nn.Sequential(
|
|
70
|
+
nn.Linear(64, 64), nn.ReLU(),
|
|
71
|
+
nn.Linear(64, 64), nn.ReLU(),
|
|
72
|
+
nn.Linear(64, 10),
|
|
73
|
+
)
|
|
74
|
+
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
|
|
75
|
+
|
|
76
|
+
config = RuntimeConfig(
|
|
77
|
+
scheduler="sequential", # "sequential" | "async" | "pipeline" | "distributed"
|
|
78
|
+
memory="pool", # "pool" | "none"
|
|
79
|
+
backend="auto", # "auto" | "cpu" | "cuda" | "triton" | "rocm"
|
|
80
|
+
passes=[Int8QuantizationPass(), ClipPass(max_norm=1.0)],
|
|
81
|
+
boundary_offload=False, # move x0..xL to CPU between fwd/bwd
|
|
82
|
+
boundary_precision=None, # e.g. torch.float16 to shrink boundary storage
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
runtime = EAERuntime(model, optimizer, config)
|
|
86
|
+
|
|
87
|
+
x = torch.randn(32, 64)
|
|
88
|
+
target = torch.randint(0, 10, (32,))
|
|
89
|
+
loss = runtime.train_step(x, lambda out: nn.functional.cross_entropy(out, target))
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Architecture
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
User Model
|
|
96
|
+
|
|
|
97
|
+
Block Decomposer (blocks.py)
|
|
98
|
+
|
|
|
99
|
+
Forward Executor (forward_executor.py) -> detached forward, no autograd graph
|
|
100
|
+
|
|
|
101
|
+
Boundary State Store (boundary_store.py) -> stores only x0..xL
|
|
102
|
+
|
|
|
103
|
+
Reverse Scheduler (schedulers/) -> pluggable: sequential/async/pipeline/distributed
|
|
104
|
+
| \
|
|
105
|
+
| \
|
|
106
|
+
Reconstruction Memory Manager (reconstruction.py / memory.py)
|
|
107
|
+
Engine |
|
|
108
|
+
| |
|
|
109
|
+
Adjoint Pipeline Backend Manager (pipeline.py / backend.py)
|
|
110
|
+
|
|
|
111
|
+
Local Autograd (PyTorch)
|
|
112
|
+
|
|
|
113
|
+
CPU / CUDA
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### The Adjoint Pipeline (the one addition beyond the base spec)
|
|
117
|
+
|
|
118
|
+
Every reverse step exchanges an `AdjointState` — never a raw tensor —
|
|
119
|
+
through a programmable pipeline:
|
|
120
|
+
|
|
121
|
+
```
|
|
122
|
+
AdjointState -> Pass1 -> Pass2 -> ... -> PassN -> next local VJP
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
A researcher working on gradient compression, synthetic gradients,
|
|
126
|
+
error-feedback, distributed communication, or adaptive optimization only
|
|
127
|
+
implements a new `EAEPass`. They never touch the reconstruction engine,
|
|
128
|
+
scheduler, or memory manager.
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from eae_runtime.passes import EAEPass
|
|
132
|
+
|
|
133
|
+
class MyPass(EAEPass):
|
|
134
|
+
name = "MyPass"
|
|
135
|
+
def process(self, adjoint, context=None):
|
|
136
|
+
new = adjoint.clone()
|
|
137
|
+
new.tensor = new.tensor * 0.9 # e.g. decay
|
|
138
|
+
return new
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### Extension points
|
|
142
|
+
|
|
143
|
+
| Want to add... | Subclass | Where |
|
|
144
|
+
|--------------------------|--------------------------|----------------------------|
|
|
145
|
+
| A gradient transform | `EAEPass` | `eae_runtime/passes/` |
|
|
146
|
+
| A reverse scheduling policy | `BaseScheduler` | `eae_runtime/schedulers/` |
|
|
147
|
+
| A memory allocation strategy | `MemoryPolicy` | `eae_runtime/memory.py` |
|
|
148
|
+
| A new backend | extend `BackendManager` | `eae_runtime/backend.py` |
|
|
149
|
+
|
|
150
|
+
None of these require modifying the runtime core.
|
|
151
|
+
|
|
152
|
+
### Built-in passes
|
|
153
|
+
|
|
154
|
+
`ClipPass`, `FP16Pass`, `Int8QuantizationPass` (aliased as `FP8Pass`),
|
|
155
|
+
`SyntheticGradientPass`, `RegularizationPass`, `GaussianNoisePass`,
|
|
156
|
+
`LoggingPass`.
|
|
157
|
+
|
|
158
|
+
### Built-in schedulers
|
|
159
|
+
|
|
160
|
+
* `SequentialScheduler` — the reference implementation; strict in-order
|
|
161
|
+
reconstruct → VJP → pipeline → free.
|
|
162
|
+
* `AsyncScheduler` — overlaps boundary-activation prefetch (useful with
|
|
163
|
+
`boundary_offload=True`) with reconstruction compute; numerically
|
|
164
|
+
identical to `SequentialScheduler`.
|
|
165
|
+
* `PipelineScheduler` — splits the batch into microbatches and runs a full
|
|
166
|
+
reverse pass per microbatch, accumulating gradients (single-process
|
|
167
|
+
emulation of pipeline-parallel microbatching).
|
|
168
|
+
* `DistributedScheduler` — assigns block ranges to `torch.distributed`
|
|
169
|
+
ranks and communicates adjoints across rank boundaries; falls back to
|
|
170
|
+
`SequentialScheduler` (with a warning) when running single-process.
|
|
171
|
+
|
|
172
|
+
### Events
|
|
173
|
+
|
|
174
|
+
Every component emits structured events (`ForwardStarted`,
|
|
175
|
+
`BlockReconstructed`, `AdjointCreated`, `MemoryAllocated`,
|
|
176
|
+
`SchedulerStep`, `PassApplied`, ...) on an `EventBus`. Subscribe without
|
|
177
|
+
touching runtime code:
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
runtime.event_bus.subscribe("BlockReconstructed", lambda e: print(e.payload))
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Set `RuntimeConfig(log_level="DEBUG")` to also pipe every event through
|
|
184
|
+
Python's structured JSON logger.
|
|
185
|
+
|
|
186
|
+
### Profiling
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
loss = runtime.train_step(x, loss_fn)
|
|
190
|
+
print(runtime.profiler.report())
|
|
191
|
+
# {'forward': {...}, 'reconstruct:Linear_0': {...}, 'pass:ClipPass': {...}, ...}
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
## Testing
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
pip install -e ".[dev]"
|
|
198
|
+
pytest
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
The suite (`tests/`) covers:
|
|
202
|
+
|
|
203
|
+
* `AdjointState` API (norm, statistics, quantize/dequantize, compress, clone, detach)
|
|
204
|
+
* Event bus pub/sub, wildcard subscribers, recording
|
|
205
|
+
* Memory manager: pool reuse, null policy, allocator correctness, leak checks
|
|
206
|
+
* Boundary store: precision downcast, CPU offload, ownership semantics
|
|
207
|
+
* Reconstruction engine vs. manual VJP (`torch.autograd.grad`) reference
|
|
208
|
+
* **Gradient equivalence**: every scheduler vs. plain `model.backward()`,
|
|
209
|
+
across float32, boundary-offloaded, and microbatched configurations
|
|
210
|
+
* All built-in passes, including a full multi-pass pipeline
|
|
211
|
+
* All built-in schedulers, plus a user-defined custom scheduler
|
|
212
|
+
* Backend manager device/dtype resolution and graceful CUDA→CPU fallback
|
|
213
|
+
* Profiler timing aggregation
|
|
214
|
+
* Mixed precision (compute dtype + boundary storage dtype) numerical bounds
|
|
215
|
+
* Memory-leak / allocator-stability checks over many training steps
|
|
216
|
+
* A full Transformer-style block stack (`nn.MultiheadAttention` + FFN,
|
|
217
|
+
pre-norm, residual) trained end-to-end with a full pass pipeline
|
|
218
|
+
* Real two-process `torch.distributed` (gloo) correctness check for
|
|
219
|
+
`DistributedScheduler` (skips gracefully if the sandbox blocks socket use)
|
|
220
|
+
|
|
221
|
+
## Non-goals by design
|
|
222
|
+
|
|
223
|
+
* Automatic partitioning of arbitrary `nn.Module` graphs — blocks are
|
|
224
|
+
specified manually or via `BlockDecomposer.from_sequential`/`from_list`.
|
|
225
|
+
* Replacing or modifying PyTorch's autograd internals.
|
|
226
|
+
* A new differentiation algorithm — local VJP is always PyTorch's own.
|
|
227
|
+
* A new compiler or graph IR.
|
|
228
|
+
* Day-one support for every architecture — the runtime targets
|
|
229
|
+
Transformer-style models with repeated block structure first.
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# A programmable reverse-mode runtime for PyTorch
|
|
2
|
+
|
|
3
|
+
A production-quality runtime that replaces PyTorch's monolithic
|
|
4
|
+
`backward()` with a programmable, block-local reverse-mode execution engine
|
|
5
|
+
implementing **Explicit Adjoint Exposure (EAE)**. It builds on the the top of [EAE](https://github.com/VladimerKhasia/eae) and turnes reverse-mode execution itself into schedulable runtime.
|
|
6
|
+
|
|
7
|
+
The runtime is **not** a new autodiff engine. It orchestrates many local
|
|
8
|
+
autograd computations using PyTorch's existing local autograd:
|
|
9
|
+
|
|
10
|
+
* **PyTorch owns:** local autograd, VJP computation, kernels.
|
|
11
|
+
* **The runtime owns:** forward scheduling, reverse scheduling,
|
|
12
|
+
reconstruction, adjoint lifecycle, memory lifecycle, pass execution,
|
|
13
|
+
backend selection, communication scheduling.
|
|
14
|
+
|
|
15
|
+
Users write ordinary `nn.Sequential` / `nn.Module` models. No custom tensor
|
|
16
|
+
type, no compiler, no FX, no PyTorch internals modified.
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pip install -e .
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Quick start
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
import torch
|
|
28
|
+
import torch.nn as nn
|
|
29
|
+
from eae_runtime import EAERuntime, RuntimeConfig
|
|
30
|
+
from eae_runtime.passes import ClipPass, Int8QuantizationPass
|
|
31
|
+
|
|
32
|
+
model = nn.Sequential(
|
|
33
|
+
nn.Linear(64, 64), nn.ReLU(),
|
|
34
|
+
nn.Linear(64, 64), nn.ReLU(),
|
|
35
|
+
nn.Linear(64, 10),
|
|
36
|
+
)
|
|
37
|
+
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
|
|
38
|
+
|
|
39
|
+
config = RuntimeConfig(
|
|
40
|
+
scheduler="sequential", # "sequential" | "async" | "pipeline" | "distributed"
|
|
41
|
+
memory="pool", # "pool" | "none"
|
|
42
|
+
backend="auto", # "auto" | "cpu" | "cuda" | "triton" | "rocm"
|
|
43
|
+
passes=[Int8QuantizationPass(), ClipPass(max_norm=1.0)],
|
|
44
|
+
boundary_offload=False, # move x0..xL to CPU between fwd/bwd
|
|
45
|
+
boundary_precision=None, # e.g. torch.float16 to shrink boundary storage
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
runtime = EAERuntime(model, optimizer, config)
|
|
49
|
+
|
|
50
|
+
x = torch.randn(32, 64)
|
|
51
|
+
target = torch.randint(0, 10, (32,))
|
|
52
|
+
loss = runtime.train_step(x, lambda out: nn.functional.cross_entropy(out, target))
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Architecture
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
User Model
|
|
59
|
+
|
|
|
60
|
+
Block Decomposer (blocks.py)
|
|
61
|
+
|
|
|
62
|
+
Forward Executor (forward_executor.py) -> detached forward, no autograd graph
|
|
63
|
+
|
|
|
64
|
+
Boundary State Store (boundary_store.py) -> stores only x0..xL
|
|
65
|
+
|
|
|
66
|
+
Reverse Scheduler (schedulers/) -> pluggable: sequential/async/pipeline/distributed
|
|
67
|
+
| \
|
|
68
|
+
| \
|
|
69
|
+
Reconstruction Memory Manager (reconstruction.py / memory.py)
|
|
70
|
+
Engine |
|
|
71
|
+
| |
|
|
72
|
+
Adjoint Pipeline Backend Manager (pipeline.py / backend.py)
|
|
73
|
+
|
|
|
74
|
+
Local Autograd (PyTorch)
|
|
75
|
+
|
|
|
76
|
+
CPU / CUDA
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### The Adjoint Pipeline (the one addition beyond the base spec)
|
|
80
|
+
|
|
81
|
+
Every reverse step exchanges an `AdjointState` — never a raw tensor —
|
|
82
|
+
through a programmable pipeline:
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
AdjointState -> Pass1 -> Pass2 -> ... -> PassN -> next local VJP
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
A researcher working on gradient compression, synthetic gradients,
|
|
89
|
+
error-feedback, distributed communication, or adaptive optimization only
|
|
90
|
+
implements a new `EAEPass`. They never touch the reconstruction engine,
|
|
91
|
+
scheduler, or memory manager.
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from eae_runtime.passes import EAEPass
|
|
95
|
+
|
|
96
|
+
class MyPass(EAEPass):
|
|
97
|
+
name = "MyPass"
|
|
98
|
+
def process(self, adjoint, context=None):
|
|
99
|
+
new = adjoint.clone()
|
|
100
|
+
new.tensor = new.tensor * 0.9 # e.g. decay
|
|
101
|
+
return new
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Extension points
|
|
105
|
+
|
|
106
|
+
| Want to add... | Subclass | Where |
|
|
107
|
+
|--------------------------|--------------------------|----------------------------|
|
|
108
|
+
| A gradient transform | `EAEPass` | `eae_runtime/passes/` |
|
|
109
|
+
| A reverse scheduling policy | `BaseScheduler` | `eae_runtime/schedulers/` |
|
|
110
|
+
| A memory allocation strategy | `MemoryPolicy` | `eae_runtime/memory.py` |
|
|
111
|
+
| A new backend | extend `BackendManager` | `eae_runtime/backend.py` |
|
|
112
|
+
|
|
113
|
+
None of these require modifying the runtime core.
|
|
114
|
+
|
|
115
|
+
### Built-in passes
|
|
116
|
+
|
|
117
|
+
`ClipPass`, `FP16Pass`, `Int8QuantizationPass` (aliased as `FP8Pass`),
|
|
118
|
+
`SyntheticGradientPass`, `RegularizationPass`, `GaussianNoisePass`,
|
|
119
|
+
`LoggingPass`.
|
|
120
|
+
|
|
121
|
+
### Built-in schedulers
|
|
122
|
+
|
|
123
|
+
* `SequentialScheduler` — the reference implementation; strict in-order
|
|
124
|
+
reconstruct → VJP → pipeline → free.
|
|
125
|
+
* `AsyncScheduler` — overlaps boundary-activation prefetch (useful with
|
|
126
|
+
`boundary_offload=True`) with reconstruction compute; numerically
|
|
127
|
+
identical to `SequentialScheduler`.
|
|
128
|
+
* `PipelineScheduler` — splits the batch into microbatches and runs a full
|
|
129
|
+
reverse pass per microbatch, accumulating gradients (single-process
|
|
130
|
+
emulation of pipeline-parallel microbatching).
|
|
131
|
+
* `DistributedScheduler` — assigns block ranges to `torch.distributed`
|
|
132
|
+
ranks and communicates adjoints across rank boundaries; falls back to
|
|
133
|
+
`SequentialScheduler` (with a warning) when running single-process.
|
|
134
|
+
|
|
135
|
+
### Events
|
|
136
|
+
|
|
137
|
+
Every component emits structured events (`ForwardStarted`,
|
|
138
|
+
`BlockReconstructed`, `AdjointCreated`, `MemoryAllocated`,
|
|
139
|
+
`SchedulerStep`, `PassApplied`, ...) on an `EventBus`. Subscribe without
|
|
140
|
+
touching runtime code:
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
runtime.event_bus.subscribe("BlockReconstructed", lambda e: print(e.payload))
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Set `RuntimeConfig(log_level="DEBUG")` to also pipe every event through
|
|
147
|
+
Python's structured JSON logger.
|
|
148
|
+
|
|
149
|
+
### Profiling
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
loss = runtime.train_step(x, loss_fn)
|
|
153
|
+
print(runtime.profiler.report())
|
|
154
|
+
# {'forward': {...}, 'reconstruct:Linear_0': {...}, 'pass:ClipPass': {...}, ...}
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Testing
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
pip install -e ".[dev]"
|
|
161
|
+
pytest
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
The suite (`tests/`) covers:
|
|
165
|
+
|
|
166
|
+
* `AdjointState` API (norm, statistics, quantize/dequantize, compress, clone, detach)
|
|
167
|
+
* Event bus pub/sub, wildcard subscribers, recording
|
|
168
|
+
* Memory manager: pool reuse, null policy, allocator correctness, leak checks
|
|
169
|
+
* Boundary store: precision downcast, CPU offload, ownership semantics
|
|
170
|
+
* Reconstruction engine vs. manual VJP (`torch.autograd.grad`) reference
|
|
171
|
+
* **Gradient equivalence**: every scheduler vs. plain `model.backward()`,
|
|
172
|
+
across float32, boundary-offloaded, and microbatched configurations
|
|
173
|
+
* All built-in passes, including a full multi-pass pipeline
|
|
174
|
+
* All built-in schedulers, plus a user-defined custom scheduler
|
|
175
|
+
* Backend manager device/dtype resolution and graceful CUDA→CPU fallback
|
|
176
|
+
* Profiler timing aggregation
|
|
177
|
+
* Mixed precision (compute dtype + boundary storage dtype) numerical bounds
|
|
178
|
+
* Memory-leak / allocator-stability checks over many training steps
|
|
179
|
+
* A full Transformer-style block stack (`nn.MultiheadAttention` + FFN,
|
|
180
|
+
pre-norm, residual) trained end-to-end with a full pass pipeline
|
|
181
|
+
* Real two-process `torch.distributed` (gloo) correctness check for
|
|
182
|
+
`DistributedScheduler` (skips gracefully if the sandbox blocks socket use)
|
|
183
|
+
|
|
184
|
+
## Non-goals by design
|
|
185
|
+
|
|
186
|
+
* Automatic partitioning of arbitrary `nn.Module` graphs — blocks are
|
|
187
|
+
specified manually or via `BlockDecomposer.from_sequential`/`from_list`.
|
|
188
|
+
* Replacing or modifying PyTorch's autograd internals.
|
|
189
|
+
* A new differentiation algorithm — local VJP is always PyTorch's own.
|
|
190
|
+
* A new compiler or graph IR.
|
|
191
|
+
* Day-one support for every architecture — the runtime targets
|
|
192
|
+
Transformer-style models with repeated block structure first.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""
|
|
2
|
+
EAE Runtime: a standalone, programmable, block-local reverse-mode execution
|
|
3
|
+
engine that sits *above* PyTorch and orchestrates many local autograd
|
|
4
|
+
computations, instead of replacing PyTorch's autodiff engine.
|
|
5
|
+
|
|
6
|
+
Quick start:
|
|
7
|
+
|
|
8
|
+
import torch.nn as nn
|
|
9
|
+
from eae_runtime import EAERuntime, RuntimeConfig
|
|
10
|
+
from eae_runtime.passes import ClipPass, GaussianNoisePass
|
|
11
|
+
|
|
12
|
+
model = nn.Sequential(
|
|
13
|
+
nn.Linear(64, 64), nn.ReLU(),
|
|
14
|
+
nn.Linear(64, 64), nn.ReLU(),
|
|
15
|
+
nn.Linear(64, 10),
|
|
16
|
+
)
|
|
17
|
+
optimizer = torch.optim.SGD(model.parameters(), lr=1e-2)
|
|
18
|
+
config = RuntimeConfig(
|
|
19
|
+
scheduler="sequential",
|
|
20
|
+
memory="pool",
|
|
21
|
+
backend="auto",
|
|
22
|
+
passes=[ClipPass(max_norm=1.0)],
|
|
23
|
+
)
|
|
24
|
+
runtime = EAERuntime(model, optimizer, config)
|
|
25
|
+
loss = runtime.train_step(x, lambda out: nn.functional.mse_loss(out, target))
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from .adjoint import AdjointState
|
|
29
|
+
from .backend import BackendManager
|
|
30
|
+
from .blocks import BlockDecomposer, EAEBlock
|
|
31
|
+
from .boundary_store import BoundaryStore
|
|
32
|
+
from .config import RuntimeConfig
|
|
33
|
+
from .events import Event, EventBus, EventType
|
|
34
|
+
from .forward_executor import ForwardExecutor
|
|
35
|
+
from .memory import MemoryManager, MemoryPolicy, NullMemoryPolicy, PoolMemoryPolicy
|
|
36
|
+
from .pipeline import AdjointPipeline, PassManager
|
|
37
|
+
from .profiler import Profiler
|
|
38
|
+
from .reconstruction import ReconstructionEngine
|
|
39
|
+
from .runtime import EAERuntime
|
|
40
|
+
from .schedulers import (
|
|
41
|
+
AsyncScheduler,
|
|
42
|
+
BaseScheduler,
|
|
43
|
+
DistributedScheduler,
|
|
44
|
+
PipelineScheduler,
|
|
45
|
+
SequentialScheduler,
|
|
46
|
+
build_scheduler,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
from ._version import version as __version__
|
|
51
|
+
except ImportError: # pragma: no cover - only hit in a non-built source tree
|
|
52
|
+
__version__ = "0.0.0+unknown"
|
|
53
|
+
|
|
54
|
+
__all__ = [
|
|
55
|
+
"AdjointState",
|
|
56
|
+
"BackendManager",
|
|
57
|
+
"BlockDecomposer",
|
|
58
|
+
"EAEBlock",
|
|
59
|
+
"BoundaryStore",
|
|
60
|
+
"RuntimeConfig",
|
|
61
|
+
"Event",
|
|
62
|
+
"EventBus",
|
|
63
|
+
"EventType",
|
|
64
|
+
"ForwardExecutor",
|
|
65
|
+
"MemoryManager",
|
|
66
|
+
"MemoryPolicy",
|
|
67
|
+
"NullMemoryPolicy",
|
|
68
|
+
"PoolMemoryPolicy",
|
|
69
|
+
"AdjointPipeline",
|
|
70
|
+
"PassManager",
|
|
71
|
+
"Profiler",
|
|
72
|
+
"ReconstructionEngine",
|
|
73
|
+
"EAERuntime",
|
|
74
|
+
"AsyncScheduler",
|
|
75
|
+
"BaseScheduler",
|
|
76
|
+
"DistributedScheduler",
|
|
77
|
+
"PipelineScheduler",
|
|
78
|
+
"SequentialScheduler",
|
|
79
|
+
"build_scheduler",
|
|
80
|
+
]
|