eae-runtime 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eae_runtime/__init__.py +80 -0
- eae_runtime/_version.py +24 -0
- eae_runtime/adjoint.py +128 -0
- eae_runtime/backend.py +52 -0
- eae_runtime/blocks.py +69 -0
- eae_runtime/boundary_store.py +65 -0
- eae_runtime/config.py +48 -0
- eae_runtime/contrib/__init__.py +25 -0
- eae_runtime/contrib/transformer_blocks.py +259 -0
- eae_runtime/events.py +84 -0
- eae_runtime/forward_executor.py +35 -0
- eae_runtime/logging_utils.py +41 -0
- eae_runtime/memory.py +175 -0
- eae_runtime/passes/__init__.py +18 -0
- eae_runtime/passes/base.py +31 -0
- eae_runtime/passes/clip.py +25 -0
- eae_runtime/passes/logging_pass.py +29 -0
- eae_runtime/passes/quantize.py +55 -0
- eae_runtime/passes/regularization.py +47 -0
- eae_runtime/passes/synthetic_gradient.py +67 -0
- eae_runtime/pipeline.py +58 -0
- eae_runtime/profiler.py +47 -0
- eae_runtime/py.typed +0 -0
- eae_runtime/reconstruction.py +89 -0
- eae_runtime/runtime.py +180 -0
- eae_runtime/schedulers/__init__.py +38 -0
- eae_runtime/schedulers/async_scheduler.py +65 -0
- eae_runtime/schedulers/base.py +87 -0
- eae_runtime/schedulers/distributed.py +78 -0
- eae_runtime/schedulers/pipeline_scheduler.py +67 -0
- eae_runtime/schedulers/sequential.py +27 -0
- eae_runtime-0.1.0.dist-info/METADATA +229 -0
- eae_runtime-0.1.0.dist-info/RECORD +38 -0
- eae_runtime-0.1.0.dist-info/WHEEL +5 -0
- eae_runtime-0.1.0.dist-info/licenses/LICENSE +13 -0
- eae_runtime-0.1.0.dist-info/scm_file_list.json +59 -0
- eae_runtime-0.1.0.dist-info/scm_version.json +8 -0
- eae_runtime-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Scheduler API. This is the heart of the runtime's reverse execution.
|
|
3
|
+
|
|
4
|
+
Instead of a hardcoded `for block in reversed(blocks): ...` loop, the
|
|
5
|
+
runtime delegates reverse execution to a swappable BaseScheduler. Every
|
|
6
|
+
scheduler performs, per block, the same logical sequence:
|
|
7
|
+
|
|
8
|
+
reconstruct -> launch local VJP -> run adjoint pipeline -> free memory -> continue
|
|
9
|
+
|
|
10
|
+
but *how* that sequence is scheduled (in order, with prefetching, in
|
|
11
|
+
pipeline stages across microbatches, or across distributed workers) is
|
|
12
|
+
entirely up to the scheduler implementation.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from typing import Dict, List, Optional
|
|
19
|
+
|
|
20
|
+
import torch
|
|
21
|
+
import torch.nn as nn
|
|
22
|
+
|
|
23
|
+
from ..adjoint import AdjointState
|
|
24
|
+
from ..boundary_store import BoundaryStore
|
|
25
|
+
from ..events import EventBus, EventType
|
|
26
|
+
from ..memory import MemoryManager
|
|
27
|
+
from ..pipeline import AdjointPipeline
|
|
28
|
+
from ..profiler import Profiler
|
|
29
|
+
from ..reconstruction import ReconstructionEngine
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class ReverseContext:
|
|
34
|
+
"""Everything a scheduler needs to run the reverse pass. Kept as a
|
|
35
|
+
plain dataclass so custom schedulers can be written without importing
|
|
36
|
+
half the runtime."""
|
|
37
|
+
|
|
38
|
+
blocks: List[nn.Module]
|
|
39
|
+
block_names: List[str]
|
|
40
|
+
boundary_store: BoundaryStore
|
|
41
|
+
reconstruction_engine: ReconstructionEngine
|
|
42
|
+
pipeline: AdjointPipeline
|
|
43
|
+
memory_manager: MemoryManager
|
|
44
|
+
event_bus: EventBus
|
|
45
|
+
profiler: Profiler
|
|
46
|
+
initial_adjoint: AdjointState
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class BaseScheduler:
|
|
50
|
+
"""Subclass and implement `run(context)`.
|
|
51
|
+
|
|
52
|
+
Must return a dict mapping each `nn.Parameter` to its accumulated
|
|
53
|
+
gradient tensor for this step.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
name: str = "BaseScheduler"
|
|
57
|
+
|
|
58
|
+
def run(self, context: ReverseContext) -> Dict[nn.Parameter, torch.Tensor]:
|
|
59
|
+
raise NotImplementedError
|
|
60
|
+
|
|
61
|
+
# -- shared helpers available to all schedulers --------------------- #
|
|
62
|
+
@staticmethod
|
|
63
|
+
def _accumulate(target: Dict[nn.Parameter, torch.Tensor], updates: Dict[nn.Parameter, torch.Tensor]) -> None:
|
|
64
|
+
for p, g in updates.items():
|
|
65
|
+
if p in target:
|
|
66
|
+
target[p] = target[p] + g
|
|
67
|
+
else:
|
|
68
|
+
target[p] = g.clone()
|
|
69
|
+
|
|
70
|
+
@staticmethod
|
|
71
|
+
def _reconstruct_one(context: ReverseContext, i: int, adjoint: AdjointState):
|
|
72
|
+
"""Reconstruct block `i` (0-indexed) given the adjoint at its output
|
|
73
|
+
boundary (i+1). Returns (new_adjoint_at_input_boundary, param_grads)."""
|
|
74
|
+
block = context.blocks[i]
|
|
75
|
+
name = context.block_names[i]
|
|
76
|
+
input_activation = context.boundary_store.get(i)
|
|
77
|
+
|
|
78
|
+
with context.profiler.track(f"reconstruct:{name}"):
|
|
79
|
+
new_adjoint, param_grads = context.reconstruction_engine.reconstruct(
|
|
80
|
+
block, input_activation, adjoint, block_name=name
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
new_adjoint = context.pipeline.run(new_adjoint, context={"block_index": i, "block_name": name})
|
|
84
|
+
|
|
85
|
+
context.memory_manager.release(input_activation)
|
|
86
|
+
context.event_bus.emit(EventType.SCHEDULER_STEP, block_index=i, block_name=name)
|
|
87
|
+
return new_adjoint, param_grads
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import warnings
|
|
4
|
+
from typing import Any, Dict, List, Optional
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
import torch.nn as nn
|
|
8
|
+
|
|
9
|
+
from .base import BaseScheduler, ReverseContext
|
|
10
|
+
from .sequential import SequentialScheduler
|
|
11
|
+
|
|
12
|
+
dist: Optional[Any]
|
|
13
|
+
try:
|
|
14
|
+
import torch.distributed as dist
|
|
15
|
+
except ImportError: # pragma: no cover
|
|
16
|
+
dist = None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class DistributedScheduler(BaseScheduler):
|
|
20
|
+
"""Assigns contiguous block ranges to ranks and pipelines the adjoint
|
|
21
|
+
across process boundaries via a Communication Pass (compress -> send ->
|
|
22
|
+
receive -> decompress) at each rank boundary.
|
|
23
|
+
|
|
24
|
+
Requires `torch.distributed` to be initialized (e.g. via
|
|
25
|
+
`torch.distributed.init_process_group`) with world_size > 1. When that
|
|
26
|
+
is not the case - e.g. running single-process, as in unit tests and
|
|
27
|
+
most local development - it transparently falls back to
|
|
28
|
+
SequentialScheduler so the same code path is exercised and produces
|
|
29
|
+
identical, verifiable gradients.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
name = "DistributedScheduler"
|
|
33
|
+
|
|
34
|
+
def __init__(self, block_ranks: Optional[List[int]] = None):
|
|
35
|
+
"""`block_ranks[i]` = rank that owns block i. If omitted, blocks are
|
|
36
|
+
striped evenly across the current world size."""
|
|
37
|
+
self.block_ranks = block_ranks
|
|
38
|
+
self._fallback = SequentialScheduler()
|
|
39
|
+
|
|
40
|
+
def _is_distributed_active(self) -> bool:
|
|
41
|
+
return dist is not None and dist.is_available() and dist.is_initialized() and dist.get_world_size() > 1
|
|
42
|
+
|
|
43
|
+
def run(self, context: ReverseContext) -> Dict[nn.Parameter, torch.Tensor]:
|
|
44
|
+
if not self._is_distributed_active():
|
|
45
|
+
warnings.warn(
|
|
46
|
+
"DistributedScheduler: torch.distributed is not initialized "
|
|
47
|
+
"with world_size > 1; falling back to SequentialScheduler "
|
|
48
|
+
"(numerically identical, single-process).",
|
|
49
|
+
RuntimeWarning,
|
|
50
|
+
)
|
|
51
|
+
return self._fallback.run(context)
|
|
52
|
+
|
|
53
|
+
# -- real multi-process path -------------------------------------
|
|
54
|
+
assert dist is not None # guaranteed by _is_distributed_active()
|
|
55
|
+
rank = dist.get_rank()
|
|
56
|
+
world_size = dist.get_world_size()
|
|
57
|
+
num_blocks = len(context.blocks)
|
|
58
|
+
ranks = self.block_ranks or [i * world_size // num_blocks for i in range(num_blocks)]
|
|
59
|
+
|
|
60
|
+
adjoint = context.initial_adjoint
|
|
61
|
+
param_grads: Dict[nn.Parameter, torch.Tensor] = {}
|
|
62
|
+
|
|
63
|
+
for i in reversed(range(num_blocks)):
|
|
64
|
+
owner = ranks[i]
|
|
65
|
+
if owner == rank:
|
|
66
|
+
adjoint, updates = self._reconstruct_one(context, i, adjoint)
|
|
67
|
+
self._accumulate(param_grads, updates)
|
|
68
|
+
# Communication pass: hand the adjoint to the neighboring
|
|
69
|
+
# rank that owns block i-1, if different.
|
|
70
|
+
if i > 0 and ranks[i - 1] != rank:
|
|
71
|
+
dist.send(adjoint.tensor.contiguous(), dst=ranks[i - 1])
|
|
72
|
+
elif i > 0 and ranks[i - 1] == rank and owner != rank:
|
|
73
|
+
buf = torch.empty_like(adjoint.tensor)
|
|
74
|
+
dist.recv(buf, src=owner)
|
|
75
|
+
adjoint = adjoint.clone()
|
|
76
|
+
adjoint.tensor = buf
|
|
77
|
+
|
|
78
|
+
return param_grads
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Dict
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn as nn
|
|
7
|
+
|
|
8
|
+
from ..adjoint import AdjointState
|
|
9
|
+
from ..events import EventType
|
|
10
|
+
from .base import BaseScheduler, ReverseContext
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class PipelineScheduler(BaseScheduler):
|
|
14
|
+
"""Splits the batch dimension into `num_microbatches` chunks and runs a
|
|
15
|
+
full reverse pass per chunk, accumulating parameter gradients across
|
|
16
|
+
chunks. This is a single-process emulation of pipeline-parallel
|
|
17
|
+
microbatching: for any block whose forward is applied independently
|
|
18
|
+
per-sample (the common case - Linear/activation/attention blocks
|
|
19
|
+
without cross-batch statistics), the accumulated result is numerically
|
|
20
|
+
identical to running the whole batch through SequentialScheduler.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
name = "PipelineScheduler"
|
|
24
|
+
|
|
25
|
+
def __init__(self, num_microbatches: int = 1):
|
|
26
|
+
self.num_microbatches = max(1, num_microbatches)
|
|
27
|
+
|
|
28
|
+
@staticmethod
|
|
29
|
+
def _slice_adjoint(adjoint: AdjointState, start: int, end: int) -> AdjointState:
|
|
30
|
+
new = adjoint.clone()
|
|
31
|
+
new.tensor = adjoint.tensor[start:end]
|
|
32
|
+
return new
|
|
33
|
+
|
|
34
|
+
def run(self, context: ReverseContext) -> Dict[nn.Parameter, torch.Tensor]:
|
|
35
|
+
num_blocks = len(context.blocks)
|
|
36
|
+
param_grads: Dict[nn.Parameter, torch.Tensor] = {}
|
|
37
|
+
|
|
38
|
+
batch_size = context.initial_adjoint.tensor.shape[0]
|
|
39
|
+
num_mb = max(1, min(self.num_microbatches, batch_size))
|
|
40
|
+
chunk = (batch_size + num_mb - 1) // num_mb
|
|
41
|
+
|
|
42
|
+
for mb in range(num_mb):
|
|
43
|
+
start, end = mb * chunk, min(batch_size, (mb + 1) * chunk)
|
|
44
|
+
if start >= end:
|
|
45
|
+
continue
|
|
46
|
+
adjoint = self._slice_adjoint(context.initial_adjoint, start, end)
|
|
47
|
+
|
|
48
|
+
for i in reversed(range(num_blocks)):
|
|
49
|
+
block = context.blocks[i]
|
|
50
|
+
name = context.block_names[i]
|
|
51
|
+
full_input = context.boundary_store.get(i)
|
|
52
|
+
input_activation = full_input[start:end]
|
|
53
|
+
|
|
54
|
+
with context.profiler.track(f"reconstruct:{name}:mb{mb}"):
|
|
55
|
+
new_adjoint, updates = context.reconstruction_engine.reconstruct(
|
|
56
|
+
block, input_activation, adjoint, block_name=name
|
|
57
|
+
)
|
|
58
|
+
new_adjoint = context.pipeline.run(
|
|
59
|
+
new_adjoint, context={"block_index": i, "block_name": name, "microbatch": mb}
|
|
60
|
+
)
|
|
61
|
+
context.event_bus.emit(
|
|
62
|
+
EventType.SCHEDULER_STEP, block_index=i, block_name=name, microbatch=mb
|
|
63
|
+
)
|
|
64
|
+
adjoint = new_adjoint
|
|
65
|
+
self._accumulate(param_grads, updates)
|
|
66
|
+
|
|
67
|
+
return param_grads
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Dict
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn as nn
|
|
7
|
+
|
|
8
|
+
from .base import BaseScheduler, ReverseContext
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class SequentialScheduler(BaseScheduler):
|
|
12
|
+
"""The reference scheduler: reconstruct -> VJP -> pipeline -> free,
|
|
13
|
+
strictly in order from the last block to the first. All other
|
|
14
|
+
schedulers must reproduce this scheduler's numerical result exactly."""
|
|
15
|
+
|
|
16
|
+
name = "SequentialScheduler"
|
|
17
|
+
|
|
18
|
+
def run(self, context: ReverseContext) -> Dict[nn.Parameter, torch.Tensor]:
|
|
19
|
+
num_blocks = len(context.blocks)
|
|
20
|
+
adjoint = context.initial_adjoint
|
|
21
|
+
param_grads: Dict[nn.Parameter, torch.Tensor] = {}
|
|
22
|
+
|
|
23
|
+
for i in reversed(range(num_blocks)):
|
|
24
|
+
adjoint, updates = self._reconstruct_one(context, i, adjoint)
|
|
25
|
+
self._accumulate(param_grads, updates)
|
|
26
|
+
|
|
27
|
+
return param_grads
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: eae-runtime
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A programmable, block-local reverse-mode execution engine that orchestrates PyTorch's local autograd (Explicit Adjoint Exposure runtime).
|
|
5
|
+
Author: Vladimer Khasia
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/VladimerKhasia/eae_runtime
|
|
8
|
+
Project-URL: Repository, https://github.com/VladimerKhasia/eae_runtime
|
|
9
|
+
Project-URL: Issues, https://github.com/VladimerKhasia/eae_runtime/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/VladimerKhasia/eae_runtime/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: pytorch,autograd,backpropagation,transformer,deep-learning,training-runtime,gradient-compression,distributed-training
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: torch>=2.0
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
31
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
32
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
33
|
+
Requires-Dist: mypy>=1.10; extra == "dev"
|
|
34
|
+
Requires-Dist: build>=1.0; extra == "dev"
|
|
35
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
# A programmable reverse-mode runtime for PyTorch
|
|
39
|
+
|
|
40
|
+
A production-quality runtime that replaces PyTorch's monolithic
|
|
41
|
+
`backward()` with a programmable, block-local reverse-mode execution engine
|
|
42
|
+
implementing **Explicit Adjoint Exposure (EAE)**. It builds on the the top of [EAE](https://github.com/VladimerKhasia/eae) and turnes reverse-mode execution itself into schedulable runtime.
|
|
43
|
+
|
|
44
|
+
The runtime is **not** a new autodiff engine. It orchestrates many local
|
|
45
|
+
autograd computations using PyTorch's existing local autograd:
|
|
46
|
+
|
|
47
|
+
* **PyTorch owns:** local autograd, VJP computation, kernels.
|
|
48
|
+
* **The runtime owns:** forward scheduling, reverse scheduling,
|
|
49
|
+
reconstruction, adjoint lifecycle, memory lifecycle, pass execution,
|
|
50
|
+
backend selection, communication scheduling.
|
|
51
|
+
|
|
52
|
+
Users write ordinary `nn.Sequential` / `nn.Module` models. No custom tensor
|
|
53
|
+
type, no compiler, no FX, no PyTorch internals modified.
|
|
54
|
+
|
|
55
|
+
## Install
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install -e .
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Quick start
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
import torch
|
|
65
|
+
import torch.nn as nn
|
|
66
|
+
from eae_runtime import EAERuntime, RuntimeConfig
|
|
67
|
+
from eae_runtime.passes import ClipPass, Int8QuantizationPass
|
|
68
|
+
|
|
69
|
+
model = nn.Sequential(
|
|
70
|
+
nn.Linear(64, 64), nn.ReLU(),
|
|
71
|
+
nn.Linear(64, 64), nn.ReLU(),
|
|
72
|
+
nn.Linear(64, 10),
|
|
73
|
+
)
|
|
74
|
+
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
|
|
75
|
+
|
|
76
|
+
config = RuntimeConfig(
|
|
77
|
+
scheduler="sequential", # "sequential" | "async" | "pipeline" | "distributed"
|
|
78
|
+
memory="pool", # "pool" | "none"
|
|
79
|
+
backend="auto", # "auto" | "cpu" | "cuda" | "triton" | "rocm"
|
|
80
|
+
passes=[Int8QuantizationPass(), ClipPass(max_norm=1.0)],
|
|
81
|
+
boundary_offload=False, # move x0..xL to CPU between fwd/bwd
|
|
82
|
+
boundary_precision=None, # e.g. torch.float16 to shrink boundary storage
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
runtime = EAERuntime(model, optimizer, config)
|
|
86
|
+
|
|
87
|
+
x = torch.randn(32, 64)
|
|
88
|
+
target = torch.randint(0, 10, (32,))
|
|
89
|
+
loss = runtime.train_step(x, lambda out: nn.functional.cross_entropy(out, target))
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Architecture
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
User Model
|
|
96
|
+
|
|
|
97
|
+
Block Decomposer (blocks.py)
|
|
98
|
+
|
|
|
99
|
+
Forward Executor (forward_executor.py) -> detached forward, no autograd graph
|
|
100
|
+
|
|
|
101
|
+
Boundary State Store (boundary_store.py) -> stores only x0..xL
|
|
102
|
+
|
|
|
103
|
+
Reverse Scheduler (schedulers/) -> pluggable: sequential/async/pipeline/distributed
|
|
104
|
+
| \
|
|
105
|
+
| \
|
|
106
|
+
Reconstruction Memory Manager (reconstruction.py / memory.py)
|
|
107
|
+
Engine |
|
|
108
|
+
| |
|
|
109
|
+
Adjoint Pipeline Backend Manager (pipeline.py / backend.py)
|
|
110
|
+
|
|
|
111
|
+
Local Autograd (PyTorch)
|
|
112
|
+
|
|
|
113
|
+
CPU / CUDA
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### The Adjoint Pipeline (the one addition beyond the base spec)
|
|
117
|
+
|
|
118
|
+
Every reverse step exchanges an `AdjointState` — never a raw tensor —
|
|
119
|
+
through a programmable pipeline:
|
|
120
|
+
|
|
121
|
+
```
|
|
122
|
+
AdjointState -> Pass1 -> Pass2 -> ... -> PassN -> next local VJP
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
A researcher working on gradient compression, synthetic gradients,
|
|
126
|
+
error-feedback, distributed communication, or adaptive optimization only
|
|
127
|
+
implements a new `EAEPass`. They never touch the reconstruction engine,
|
|
128
|
+
scheduler, or memory manager.
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from eae_runtime.passes import EAEPass
|
|
132
|
+
|
|
133
|
+
class MyPass(EAEPass):
|
|
134
|
+
name = "MyPass"
|
|
135
|
+
def process(self, adjoint, context=None):
|
|
136
|
+
new = adjoint.clone()
|
|
137
|
+
new.tensor = new.tensor * 0.9 # e.g. decay
|
|
138
|
+
return new
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### Extension points
|
|
142
|
+
|
|
143
|
+
| Want to add... | Subclass | Where |
|
|
144
|
+
|--------------------------|--------------------------|----------------------------|
|
|
145
|
+
| A gradient transform | `EAEPass` | `eae_runtime/passes/` |
|
|
146
|
+
| A reverse scheduling policy | `BaseScheduler` | `eae_runtime/schedulers/` |
|
|
147
|
+
| A memory allocation strategy | `MemoryPolicy` | `eae_runtime/memory.py` |
|
|
148
|
+
| A new backend | extend `BackendManager` | `eae_runtime/backend.py` |
|
|
149
|
+
|
|
150
|
+
None of these require modifying the runtime core.
|
|
151
|
+
|
|
152
|
+
### Built-in passes
|
|
153
|
+
|
|
154
|
+
`ClipPass`, `FP16Pass`, `Int8QuantizationPass` (aliased as `FP8Pass`),
|
|
155
|
+
`SyntheticGradientPass`, `RegularizationPass`, `GaussianNoisePass`,
|
|
156
|
+
`LoggingPass`.
|
|
157
|
+
|
|
158
|
+
### Built-in schedulers
|
|
159
|
+
|
|
160
|
+
* `SequentialScheduler` — the reference implementation; strict in-order
|
|
161
|
+
reconstruct → VJP → pipeline → free.
|
|
162
|
+
* `AsyncScheduler` — overlaps boundary-activation prefetch (useful with
|
|
163
|
+
`boundary_offload=True`) with reconstruction compute; numerically
|
|
164
|
+
identical to `SequentialScheduler`.
|
|
165
|
+
* `PipelineScheduler` — splits the batch into microbatches and runs a full
|
|
166
|
+
reverse pass per microbatch, accumulating gradients (single-process
|
|
167
|
+
emulation of pipeline-parallel microbatching).
|
|
168
|
+
* `DistributedScheduler` — assigns block ranges to `torch.distributed`
|
|
169
|
+
ranks and communicates adjoints across rank boundaries; falls back to
|
|
170
|
+
`SequentialScheduler` (with a warning) when running single-process.
|
|
171
|
+
|
|
172
|
+
### Events
|
|
173
|
+
|
|
174
|
+
Every component emits structured events (`ForwardStarted`,
|
|
175
|
+
`BlockReconstructed`, `AdjointCreated`, `MemoryAllocated`,
|
|
176
|
+
`SchedulerStep`, `PassApplied`, ...) on an `EventBus`. Subscribe without
|
|
177
|
+
touching runtime code:
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
runtime.event_bus.subscribe("BlockReconstructed", lambda e: print(e.payload))
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Set `RuntimeConfig(log_level="DEBUG")` to also pipe every event through
|
|
184
|
+
Python's structured JSON logger.
|
|
185
|
+
|
|
186
|
+
### Profiling
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
loss = runtime.train_step(x, loss_fn)
|
|
190
|
+
print(runtime.profiler.report())
|
|
191
|
+
# {'forward': {...}, 'reconstruct:Linear_0': {...}, 'pass:ClipPass': {...}, ...}
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
## Testing
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
pip install -e ".[dev]"
|
|
198
|
+
pytest
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
The suite (`tests/`) covers:
|
|
202
|
+
|
|
203
|
+
* `AdjointState` API (norm, statistics, quantize/dequantize, compress, clone, detach)
|
|
204
|
+
* Event bus pub/sub, wildcard subscribers, recording
|
|
205
|
+
* Memory manager: pool reuse, null policy, allocator correctness, leak checks
|
|
206
|
+
* Boundary store: precision downcast, CPU offload, ownership semantics
|
|
207
|
+
* Reconstruction engine vs. manual VJP (`torch.autograd.grad`) reference
|
|
208
|
+
* **Gradient equivalence**: every scheduler vs. plain `model.backward()`,
|
|
209
|
+
across float32, boundary-offloaded, and microbatched configurations
|
|
210
|
+
* All built-in passes, including a full multi-pass pipeline
|
|
211
|
+
* All built-in schedulers, plus a user-defined custom scheduler
|
|
212
|
+
* Backend manager device/dtype resolution and graceful CUDA→CPU fallback
|
|
213
|
+
* Profiler timing aggregation
|
|
214
|
+
* Mixed precision (compute dtype + boundary storage dtype) numerical bounds
|
|
215
|
+
* Memory-leak / allocator-stability checks over many training steps
|
|
216
|
+
* A full Transformer-style block stack (`nn.MultiheadAttention` + FFN,
|
|
217
|
+
pre-norm, residual) trained end-to-end with a full pass pipeline
|
|
218
|
+
* Real two-process `torch.distributed` (gloo) correctness check for
|
|
219
|
+
`DistributedScheduler` (skips gracefully if the sandbox blocks socket use)
|
|
220
|
+
|
|
221
|
+
## Non-goals by design
|
|
222
|
+
|
|
223
|
+
* Automatic partitioning of arbitrary `nn.Module` graphs — blocks are
|
|
224
|
+
specified manually or via `BlockDecomposer.from_sequential`/`from_list`.
|
|
225
|
+
* Replacing or modifying PyTorch's autograd internals.
|
|
226
|
+
* A new differentiation algorithm — local VJP is always PyTorch's own.
|
|
227
|
+
* A new compiler or graph IR.
|
|
228
|
+
* Day-one support for every architecture — the runtime targets
|
|
229
|
+
Transformer-style models with repeated block structure first.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
eae_runtime/__init__.py,sha256=dyoSq-IDEUuSojucACdao1ZTkXBuIEJ4Q02936UPUts,2266
|
|
2
|
+
eae_runtime/_version.py,sha256=obMoDE79VgohpsGVxWupnx1D5WyFXxlX-YJV2Bv_O3s,528
|
|
3
|
+
eae_runtime/adjoint.py,sha256=Sek-sxnTGb6AiVazStjy8-30Fxo55jRES_JuHXXdSLI,5002
|
|
4
|
+
eae_runtime/backend.py,sha256=ycedtyXUeNAjNwv51DuYRWgePhOwHCiMQ-fyJ5d-Nl0,2015
|
|
5
|
+
eae_runtime/blocks.py,sha256=2TuVvz2Gyusvx3XL-qbdRqwHxtukAmN17LhyN-EZBe8,2629
|
|
6
|
+
eae_runtime/boundary_store.py,sha256=_CEZhxnaCBscUEEMDsUAs0QfUOfiBGdqNj12yn2mgqA,2183
|
|
7
|
+
eae_runtime/config.py,sha256=6jCCzE2DSPU_txtA1qlMWj5Hjj1tfn5mkZE_0-xHvxk,1485
|
|
8
|
+
eae_runtime/events.py,sha256=mGLHfSQpcTBKOJgyH7ITfH2o-qK-j60fL47zasUrqnM,2846
|
|
9
|
+
eae_runtime/forward_executor.py,sha256=AIaVcdZpONf5-U-945qkYp4Ff6Lo3lnZRyu5BlgPDUI,1189
|
|
10
|
+
eae_runtime/logging_utils.py,sha256=y-RcsKGuqUA55W-VNfdbPzzMiTM3CgaA_JmG4lMaNnY,1397
|
|
11
|
+
eae_runtime/memory.py,sha256=9nBJ0nhtCfXWwEgZC3ZWLFCTxV8kTO7UHfBZ1z9LzEM,5788
|
|
12
|
+
eae_runtime/pipeline.py,sha256=RJoSwRW6U3PjTISKWvNsnyRImKsIfvIVXSNMIbEInrQ,2144
|
|
13
|
+
eae_runtime/profiler.py,sha256=dyDDjNKT2kbofrAC1TUP42dd84OPObXQwAQ9B1fXHA4,1391
|
|
14
|
+
eae_runtime/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
|
+
eae_runtime/reconstruction.py,sha256=jzfql7QxP-YdIL6W5VnuPkeNMgH4S1HDwIekoIrFraY,3026
|
|
16
|
+
eae_runtime/runtime.py,sha256=31k2S25drgLjnHe373ncHPAGZohDp-bzPfcAdEZtbkc,7148
|
|
17
|
+
eae_runtime/contrib/__init__.py,sha256=x8_ozItvdqXl7cifLFlBkd9BY-tnkwcD6ErkoC8ph-E,653
|
|
18
|
+
eae_runtime/contrib/transformer_blocks.py,sha256=MQPXwXiolhlU1llHP9yMiqZUdPeDaVThdAhQejqTpFw,10544
|
|
19
|
+
eae_runtime/passes/__init__.py,sha256=m5Qz5mIxr5BPs5C__Z3qfvN9_dfPBK7YuVcEKrl7Pro,477
|
|
20
|
+
eae_runtime/passes/base.py,sha256=Az8dUMbU4vYyTenhxib15Jv8x_WhxrHy1AFOKdU3bO8,972
|
|
21
|
+
eae_runtime/passes/clip.py,sha256=wUjbMxdjxDFPpnnxsYztRQiUMd84wP-pQtWusCv-RGY,767
|
|
22
|
+
eae_runtime/passes/logging_pass.py,sha256=4kjfF3e411C0SgbxSrtPtyzZsotD4r6bCpkl72-WMtI,888
|
|
23
|
+
eae_runtime/passes/quantize.py,sha256=YTXesgAWNNr_UQD0_ENpqqOQqgujXEzay0yfPK0c2tk,1899
|
|
24
|
+
eae_runtime/passes/regularization.py,sha256=fKUt8Vi1_eCsDwVZnsQBXSh0_-34VYLDYT4sAOEgxLU,1504
|
|
25
|
+
eae_runtime/passes/synthetic_gradient.py,sha256=afaNlE_0aCUhyo9GkwCUhkjZwOegka31tYJnpiQCC24,2444
|
|
26
|
+
eae_runtime/schedulers/__init__.py,sha256=cZcoRb4CZ-GDSRGLh-lzW9oO72YrqeHW9lmfhFX4LcE,1152
|
|
27
|
+
eae_runtime/schedulers/async_scheduler.py,sha256=q_5pm-JuoZNN9Jr_bKPZJWogHtNDvaiuutkAdzw5_p0,2484
|
|
28
|
+
eae_runtime/schedulers/base.py,sha256=s8nk_nNoDryQ_0cnWW0VluDXuQGwlbuNJ-pM7n2Dai8,3076
|
|
29
|
+
eae_runtime/schedulers/distributed.py,sha256=o-iB7QEXVimegpPFigjUAY0kqA4KAWYCqNVj6xg5oXc,3147
|
|
30
|
+
eae_runtime/schedulers/pipeline_scheduler.py,sha256=fPBAr0iS0LN-93kvAgXBstIulgVv61q-QMAeu7nYgxU,2654
|
|
31
|
+
eae_runtime/schedulers/sequential.py,sha256=jwtzqsdGsSj-X4hn04GQWmKO5lvZ-DWOPAMn-IKJKWo,860
|
|
32
|
+
eae_runtime-0.1.0.dist-info/licenses/LICENSE,sha256=ojKENiVsyN8E7oy0zMwPD3QxlcXSrTemue3f2whrp6o,555
|
|
33
|
+
eae_runtime-0.1.0.dist-info/METADATA,sha256=XQ8OJ5KhV7uQ6FCqCuG3Ggev1t4dwoOQV2ZNIzieySw,8957
|
|
34
|
+
eae_runtime-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
35
|
+
eae_runtime-0.1.0.dist-info/scm_file_list.json,sha256=OBrx3FNG3VnHpmm3wQLpYDedjzOkcpsAFCvfTI4a6HQ,1941
|
|
36
|
+
eae_runtime-0.1.0.dist-info/scm_version.json,sha256=lmTMI8NbneyiyDTdQpOy0NUQkXa0gaeW-rJPiAib7sE,160
|
|
37
|
+
eae_runtime-0.1.0.dist-info/top_level.txt,sha256=UQI-zlkwW3U0ukZvMmL3hb0F0v5F-VU2IP1CoitVorw,12
|
|
38
|
+
eae_runtime-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Copyright 2026 Vladimer Khasia
|
|
2
|
+
|
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
you may not use this file except in compliance with the License.
|
|
5
|
+
You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
See the License for the specific language governing permissions and
|
|
13
|
+
limitations under the License.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"files": [
|
|
3
|
+
".gitignore",
|
|
4
|
+
"README.md",
|
|
5
|
+
"pyproject.toml",
|
|
6
|
+
"LICENSE",
|
|
7
|
+
".github/workflows/publish.yml",
|
|
8
|
+
".github/workflows/ci.yml",
|
|
9
|
+
"eae_runtime/adjoint.py",
|
|
10
|
+
"eae_runtime/memory.py",
|
|
11
|
+
"eae_runtime/profiler.py",
|
|
12
|
+
"eae_runtime/config.py",
|
|
13
|
+
"eae_runtime/reconstruction.py",
|
|
14
|
+
"eae_runtime/pipeline.py",
|
|
15
|
+
"eae_runtime/backend.py",
|
|
16
|
+
"eae_runtime/forward_executor.py",
|
|
17
|
+
"eae_runtime/boundary_store.py",
|
|
18
|
+
"eae_runtime/blocks.py",
|
|
19
|
+
"eae_runtime/runtime.py",
|
|
20
|
+
"eae_runtime/py.typed",
|
|
21
|
+
"eae_runtime/logging_utils.py",
|
|
22
|
+
"eae_runtime/__init__.py",
|
|
23
|
+
"eae_runtime/events.py",
|
|
24
|
+
"eae_runtime/contrib/transformer_blocks.py",
|
|
25
|
+
"eae_runtime/contrib/__init__.py",
|
|
26
|
+
"eae_runtime/schedulers/distributed.py",
|
|
27
|
+
"eae_runtime/schedulers/base.py",
|
|
28
|
+
"eae_runtime/schedulers/pipeline_scheduler.py",
|
|
29
|
+
"eae_runtime/schedulers/async_scheduler.py",
|
|
30
|
+
"eae_runtime/schedulers/sequential.py",
|
|
31
|
+
"eae_runtime/schedulers/__init__.py",
|
|
32
|
+
"eae_runtime/passes/base.py",
|
|
33
|
+
"eae_runtime/passes/quantize.py",
|
|
34
|
+
"eae_runtime/passes/regularization.py",
|
|
35
|
+
"eae_runtime/passes/clip.py",
|
|
36
|
+
"eae_runtime/passes/synthetic_gradient.py",
|
|
37
|
+
"eae_runtime/passes/logging_pass.py",
|
|
38
|
+
"eae_runtime/passes/__init__.py",
|
|
39
|
+
"examples/train_transformer_lm.py",
|
|
40
|
+
"tests/test_mixed_precision.py",
|
|
41
|
+
"tests/test_schedulers.py",
|
|
42
|
+
"tests/test_profiler.py",
|
|
43
|
+
"tests/conftest.py",
|
|
44
|
+
"tests/test_gradient_equivalence.py",
|
|
45
|
+
"tests/test_events.py",
|
|
46
|
+
"tests/test_distributed_multiprocess.py",
|
|
47
|
+
"tests/test_backend.py",
|
|
48
|
+
"tests/test_end_to_end_transformer.py",
|
|
49
|
+
"tests/test_runtime.py",
|
|
50
|
+
"tests/test_boundary_store.py",
|
|
51
|
+
"tests/test_memory_leaks.py",
|
|
52
|
+
"tests/test_memory_manager.py",
|
|
53
|
+
"tests/test_adjoint.py",
|
|
54
|
+
"tests/test_reconstruction.py",
|
|
55
|
+
"tests/test_contrib_transformer_blocks.py",
|
|
56
|
+
"tests/test_config_and_blocks.py",
|
|
57
|
+
"tests/test_passes.py"
|
|
58
|
+
]
|
|
59
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
eae_runtime
|