torch-npu-memory-saver 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- torch_npu_memory_saver-0.1.0/LICENSE +21 -0
- torch_npu_memory_saver-0.1.0/MANIFEST.in +2 -0
- torch_npu_memory_saver-0.1.0/PKG-INFO +206 -0
- torch_npu_memory_saver-0.1.0/README.md +168 -0
- torch_npu_memory_saver-0.1.0/docs/validation.md +51 -0
- torch_npu_memory_saver-0.1.0/native/CMakeLists.txt +48 -0
- torch_npu_memory_saver-0.1.0/native/include/ascend_saver.h +64 -0
- torch_npu_memory_saver-0.1.0/native/src/ascend_saver.cpp +472 -0
- torch_npu_memory_saver-0.1.0/native/tests/fake_acl.cpp +134 -0
- torch_npu_memory_saver-0.1.0/native/tests/fake_acl.h +68 -0
- torch_npu_memory_saver-0.1.0/native/tests/state_tests.cpp +341 -0
- torch_npu_memory_saver-0.1.0/pyproject.toml +26 -0
- torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver/__init__.py +34 -0
- torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver/saver.py +293 -0
- torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver/state.py +158 -0
- torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver.egg-info/PKG-INFO +206 -0
- torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver.egg-info/SOURCES.txt +24 -0
- torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver.egg-info/dependency_links.txt +1 -0
- torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver.egg-info/requires.txt +5 -0
- torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver.egg-info/top_level.txt +1 -0
- torch_npu_memory_saver-0.1.0/setup.cfg +4 -0
- torch_npu_memory_saver-0.1.0/setup.py +37 -0
- torch_npu_memory_saver-0.1.0/tests/test_import_boundary.py +18 -0
- torch_npu_memory_saver-0.1.0/tests/test_library_path.py +36 -0
- torch_npu_memory_saver-0.1.0/tests/test_npu_graph.py +108 -0
- torch_npu_memory_saver-0.1.0/tests/test_state.py +158 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 LightSeek Foundation <contact@lightseek.org>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: torch-npu-memory-saver
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Managed Ascend memory with stable virtual addresses
|
|
5
|
+
License: MIT License
|
|
6
|
+
|
|
7
|
+
Copyright (c) 2026 LightSeek Foundation <contact@lightseek.org>
|
|
8
|
+
|
|
9
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
in the Software without restriction, including without limitation the rights
|
|
12
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
furnished to do so, subject to the following conditions:
|
|
15
|
+
|
|
16
|
+
The above copyright notice and this permission notice shall be included in all
|
|
17
|
+
copies or substantial portions of the Software.
|
|
18
|
+
|
|
19
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
SOFTWARE.
|
|
26
|
+
|
|
27
|
+
Project-URL: Repository, https://github.com/yanlonghu/torch_npu_memory_saver
|
|
28
|
+
Project-URL: Issues, https://github.com/yanlonghu/torch_npu_memory_saver/issues
|
|
29
|
+
Project-URL: Documentation, https://github.com/yanlonghu/torch_npu_memory_saver/blob/main/README.md
|
|
30
|
+
Requires-Python: >=3.11
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
License-File: LICENSE
|
|
33
|
+
Requires-Dist: torch>=2.9
|
|
34
|
+
Requires-Dist: torch-npu>=2.9
|
|
35
|
+
Provides-Extra: test
|
|
36
|
+
Requires-Dist: pytest>=8; extra == "test"
|
|
37
|
+
Dynamic: license-file
|
|
38
|
+
|
|
39
|
+
# torch_npu_memory_saver
|
|
40
|
+
|
|
41
|
+
Release managed Ascend device memory and restore it at the same virtual addresses.
|
|
42
|
+
Extracted from [TokenSpeed PR #3530](https://github.com/lightseekorg/tokenspeed/pull/3530)
|
|
43
|
+
at `269b59a9`. The native allocator retains ABI 2. No PyPI package has been published.
|
|
44
|
+
|
|
45
|
+
## Dependencies
|
|
46
|
+
|
|
47
|
+
| Component | Requirement / tested configuration |
|
|
48
|
+
| --- | --- |
|
|
49
|
+
| Hardware and OS | Validated on Atlas A2 / 910B3, Linux aarch64. |
|
|
50
|
+
| Python | Package requires 3.11+; device validation used 3.12. |
|
|
51
|
+
| PyTorch | Device validation used `torch==2.9.0+cpu`, paired with PTA below. |
|
|
52
|
+
| PTA (`torch_npu`) | `9d40ac3` (2.9-730) **with six official backports**; a stock version number alone is insufficient. |
|
|
53
|
+
| CANN / HCCL | Validated with 9.0.0 and the deployment's installed Ascend driver. |
|
|
54
|
+
| Native build | CANN development headers and `acl_rt`, CMake 3.18+, a C++17 compiler. |
|
|
55
|
+
| Python build | Setuptools 69+, wheel; pip provisions these and CMake in its build environment. |
|
|
56
|
+
| Tests | Pytest 8+; the real-device test additionally needs the configured NPU stack. |
|
|
57
|
+
|
|
58
|
+
The six PTA backports are
|
|
59
|
+
[!39221](https://gitcode.com/Ascend/pytorch/merge_requests/39221),
|
|
60
|
+
[!45270](https://gitcode.com/Ascend/pytorch/merge_requests/45270),
|
|
61
|
+
[!30625](https://gitcode.com/Ascend/pytorch/merge_requests/30625),
|
|
62
|
+
[!32154](https://gitcode.com/Ascend/pytorch/merge_requests/32154),
|
|
63
|
+
[!32564](https://gitcode.com/Ascend/pytorch/merge_requests/32564), and
|
|
64
|
+
[!38431](https://gitcode.com/Ascend/pytorch/merge_requests/38431), applied to
|
|
65
|
+
[9d40ac3](https://gitcode.com/Ascend/pytorch/commit/9d40ac3a2b3714df7e5d164bda28a42eff3282cd).
|
|
66
|
+
This is the validated deployment prerequisite, not a claim that every backport
|
|
67
|
+
is needed for every standalone operation. See [validation coverage](docs/validation.md)
|
|
68
|
+
for the installed build identity and integration results. PTA 2.13 and other
|
|
69
|
+
hardware/CANN combinations have not been requalified for this extraction.
|
|
70
|
+
|
|
71
|
+
Package metadata declares `torch>=2.9` and `torch-npu>=2.9` as minimum versions,
|
|
72
|
+
**not a compatibility matrix**. Pip cannot verify the required source patches
|
|
73
|
+
or choose a matching Torch/PTA/CANN stack. Prepare that stack first, then use
|
|
74
|
+
`--no-deps` below to preserve it. This package does not install CANN, drivers,
|
|
75
|
+
TokenSpeed, Triton or CUDA's `torch_memory_saver`.
|
|
76
|
+
|
|
77
|
+
## Install with pip
|
|
78
|
+
|
|
79
|
+
After preparing the matching Torch/PTA stack and loading the CANN toolkit
|
|
80
|
+
environment, install directly from the public repository:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
python -m pip install --no-deps \
|
|
84
|
+
"git+https://github.com/yanlonghu/torch_npu_memory_saver.git@main"
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
This requires Git and builds the native library locally. For reproducible
|
|
88
|
+
deployments, replace `main` with a reviewed full commit hash. `--no-deps`
|
|
89
|
+
preserves the preinstalled Torch/PTA versions; build tools are still provisioned
|
|
90
|
+
by pip. For an offline build, preinstall the listed build tools and additionally
|
|
91
|
+
pass `--no-build-isolation`, using a local source checkout.
|
|
92
|
+
|
|
93
|
+
The project is not yet published on PyPI: the bare command
|
|
94
|
+
`pip install torch-npu-memory-saver` is not an available installation route.
|
|
95
|
+
|
|
96
|
+
### Build and keep a wheel
|
|
97
|
+
|
|
98
|
+
In an environment with the matching Torch/PTA installation:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
git clone https://github.com/yanlonghu/torch_npu_memory_saver.git
|
|
102
|
+
cd torch_npu_memory_saver
|
|
103
|
+
|
|
104
|
+
# Replace this with your deployment's CANN toolkit directory.
|
|
105
|
+
export ASCEND_HOME_PATH=/path/to/cann
|
|
106
|
+
source "$ASCEND_HOME_PATH/set_env.sh"
|
|
107
|
+
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
|
|
108
|
+
unset PYTORCH_NO_NPU_MEMORY_CACHING
|
|
109
|
+
|
|
110
|
+
python -m pip wheel --no-deps . --wheel-dir dist
|
|
111
|
+
python -m pip install --no-deps dist/torch_npu_memory_saver-0.1.0-*.whl
|
|
112
|
+
python -c 'from importlib.metadata import version; print(version("torch-npu-memory-saver"))'
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
If your allocation configuration has other options, preserve them while setting
|
|
116
|
+
`expandable_segments:False`. Set the environment before starting the worker.
|
|
117
|
+
Build on the target platform against its CANN installation. The wheel contains
|
|
118
|
+
the Saver `.so`, not CANN or system libraries; those must remain available at
|
|
119
|
+
runtime. Do not use editable installation as a substitute for the wheel build.
|
|
120
|
+
|
|
121
|
+
The package locates its bundled `.so` automatically. For a separate native build,
|
|
122
|
+
`TORCH_NPU_MEMORY_SAVER_LIBRARY` may override it with an existing absolute path.
|
|
123
|
+
The old `TOKENSPEED_NPU_MEMORY_SAVER_LIBRARY` variable is not used. Import
|
|
124
|
+
`torch_npu` and select the device before the first allocation or transition.
|
|
125
|
+
|
|
126
|
+
## API and contract
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
import torch
|
|
130
|
+
import torch_npu
|
|
131
|
+
from torch_npu_memory_saver import memory_saver
|
|
132
|
+
|
|
133
|
+
torch.npu.set_device(0)
|
|
134
|
+
with memory_saver.region(tag="weights", enable_cpu_backup=True):
|
|
135
|
+
tensor = torch.ones(1024, device="npu")
|
|
136
|
+
# Stop new work and coordinate other ranks before calling pause.
|
|
137
|
+
memory_saver.pause("weights")
|
|
138
|
+
# Do not read or write tensor while its physical backing is released.
|
|
139
|
+
memory_saver.resume("weights")
|
|
140
|
+
torch.testing.assert_close(tensor.cpu(), torch.ones(1024))
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
For discard, use `enable_cpu_backup=False`. After `resume()`, write all required
|
|
144
|
+
contents before reading the tensor or replaying a graph that consumes it.
|
|
145
|
+
`resume()` restores mappings; it does not reload or commit model weights.
|
|
146
|
+
|
|
147
|
+
`region(tag, enable_cpu_backup)` routes allocations through owned PTA MemPools.
|
|
148
|
+
This minimal extraction retains fixed tags: weights (backup/discard), kv_cache
|
|
149
|
+
(discard), acl_graph (backup/discard). `allocation_region(PoolKey(...))` exposes
|
|
150
|
+
separate pool scopes under one release tag. REMAIN uses the normal allocator.
|
|
151
|
+
`npu_graph(graph, *, pool, stream, tag, policy, pool_scope,
|
|
152
|
+
auto_dispatch_capture)` captures into an owned pool. `pause(tag=None)` and
|
|
153
|
+
`resume(tag=None)` target one tag or all tags. `assert_active()` rejects paused
|
|
154
|
+
or failed regions. `stats()` reports allocator accounting;
|
|
155
|
+
`validate_allocations(allocations, graphs=...)` checks storage and pool ownership.
|
|
156
|
+
Each allocation record is `(tag, name, pointer, byte_size)`.
|
|
157
|
+
|
|
158
|
+
Backup restores bytes at the same virtual address. Discard restores mappings
|
|
159
|
+
with undefined contents: initialize before use. Graph executables and HCCL
|
|
160
|
+
resources stay resident. All captured persistent state must be backed up or
|
|
161
|
+
resident, and replay must overwrite discarded temporaries before reading them.
|
|
162
|
+
One device/context per process; all instances share pools. Nested scopes are
|
|
163
|
+
rejected. Callers must drain and block new work; these guards do not intercept
|
|
164
|
+
arbitrary tensor access. A failed transition can be partial. Do not reopen work
|
|
165
|
+
until recovery is complete; callback/rollback faults require worker restart.
|
|
166
|
+
|
|
167
|
+
This library does not implement model loaders, KV invalidation, weight commits,
|
|
168
|
+
multi-rank protocol 3, communication initialization, or process-group shutdown.
|
|
169
|
+
TokenSpeed retains those responsibilities. It does not depend on TokenSpeed or
|
|
170
|
+
on the CUDA `torch_memory_saver` package.
|
|
171
|
+
|
|
172
|
+
## Tests
|
|
173
|
+
|
|
174
|
+
Host Python tests require Torch and pytest but no NPU. Use a separate CPU test
|
|
175
|
+
environment if needed; do not replace the Torch installation in a deployment:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
python -m pip install 'pytest>=8'
|
|
179
|
+
PYTHONPATH=python python -m pytest -q tests
|
|
180
|
+
|
|
181
|
+
# C++ state and error injection tests do not require Torch or CANN.
|
|
182
|
+
cmake -S native -B build-test -DSAVER_FAKE_ACL=ON
|
|
183
|
+
cmake --build build-test
|
|
184
|
+
ctest --test-dir build-test --output-on-failure
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Host tests do not establish NPU device correctness. They skip the real NPU test
|
|
188
|
+
unless explicitly enabled.
|
|
189
|
+
|
|
190
|
+
On an exclusively assigned NPU, run the independent graph test with
|
|
191
|
+
`RUN_NPU_MEMORY_SAVER_TESTS=1 python -m pytest tests/test_npu_graph.py`. It checks
|
|
192
|
+
three backup/discard cycles, same addresses, and replay after poisoning owned
|
|
193
|
+
discard storage. This is opt-in even when a native library is installed.
|
|
194
|
+
|
|
195
|
+
## Publishing
|
|
196
|
+
|
|
197
|
+
Maintainers publish source distributions with the manually triggered
|
|
198
|
+
`.github/workflows/publish.yml` workflow on `main`. The `pypi` GitHub environment
|
|
199
|
+
uses PyPI Trusted Publishing; no long-lived API token is required. Register
|
|
200
|
+
owner `yanlonghu`, repository `torch_npu_memory_saver`, workflow `publish.yml`,
|
|
201
|
+
and environment `pypi` in PyPI before the first run. Increment the project
|
|
202
|
+
version for subsequent releases; PyPI does not allow replacing a release file.
|
|
203
|
+
|
|
204
|
+
The workflow validates package metadata and uploads only the source archive.
|
|
205
|
+
Installation compiles the native library against the user's configured CANN
|
|
206
|
+
toolkit. It does not imply support for untested Torch/PTA/CANN combinations.
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# torch_npu_memory_saver
|
|
2
|
+
|
|
3
|
+
Release managed Ascend device memory and restore it at the same virtual addresses.
|
|
4
|
+
Extracted from [TokenSpeed PR #3530](https://github.com/lightseekorg/tokenspeed/pull/3530)
|
|
5
|
+
at `269b59a9`. The native allocator retains ABI 2. No PyPI package has been published.
|
|
6
|
+
|
|
7
|
+
## Dependencies
|
|
8
|
+
|
|
9
|
+
| Component | Requirement / tested configuration |
|
|
10
|
+
| --- | --- |
|
|
11
|
+
| Hardware and OS | Validated on Atlas A2 / 910B3, Linux aarch64. |
|
|
12
|
+
| Python | Package requires 3.11+; device validation used 3.12. |
|
|
13
|
+
| PyTorch | Device validation used `torch==2.9.0+cpu`, paired with PTA below. |
|
|
14
|
+
| PTA (`torch_npu`) | `9d40ac3` (2.9-730) **with six official backports**; a stock version number alone is insufficient. |
|
|
15
|
+
| CANN / HCCL | Validated with 9.0.0 and the deployment's installed Ascend driver. |
|
|
16
|
+
| Native build | CANN development headers and `acl_rt`, CMake 3.18+, a C++17 compiler. |
|
|
17
|
+
| Python build | Setuptools 69+, wheel; pip provisions these and CMake in its build environment. |
|
|
18
|
+
| Tests | Pytest 8+; the real-device test additionally needs the configured NPU stack. |
|
|
19
|
+
|
|
20
|
+
The six PTA backports are
|
|
21
|
+
[!39221](https://gitcode.com/Ascend/pytorch/merge_requests/39221),
|
|
22
|
+
[!45270](https://gitcode.com/Ascend/pytorch/merge_requests/45270),
|
|
23
|
+
[!30625](https://gitcode.com/Ascend/pytorch/merge_requests/30625),
|
|
24
|
+
[!32154](https://gitcode.com/Ascend/pytorch/merge_requests/32154),
|
|
25
|
+
[!32564](https://gitcode.com/Ascend/pytorch/merge_requests/32564), and
|
|
26
|
+
[!38431](https://gitcode.com/Ascend/pytorch/merge_requests/38431), applied to
|
|
27
|
+
[9d40ac3](https://gitcode.com/Ascend/pytorch/commit/9d40ac3a2b3714df7e5d164bda28a42eff3282cd).
|
|
28
|
+
This is the validated deployment prerequisite, not a claim that every backport
|
|
29
|
+
is needed for every standalone operation. See [validation coverage](docs/validation.md)
|
|
30
|
+
for the installed build identity and integration results. PTA 2.13 and other
|
|
31
|
+
hardware/CANN combinations have not been requalified for this extraction.
|
|
32
|
+
|
|
33
|
+
Package metadata declares `torch>=2.9` and `torch-npu>=2.9` as minimum versions,
|
|
34
|
+
**not a compatibility matrix**. Pip cannot verify the required source patches
|
|
35
|
+
or choose a matching Torch/PTA/CANN stack. Prepare that stack first, then use
|
|
36
|
+
`--no-deps` below to preserve it. This package does not install CANN, drivers,
|
|
37
|
+
TokenSpeed, Triton or CUDA's `torch_memory_saver`.
|
|
38
|
+
|
|
39
|
+
## Install with pip
|
|
40
|
+
|
|
41
|
+
After preparing the matching Torch/PTA stack and loading the CANN toolkit
|
|
42
|
+
environment, install directly from the public repository:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
python -m pip install --no-deps \
|
|
46
|
+
"git+https://github.com/yanlonghu/torch_npu_memory_saver.git@main"
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
This requires Git and builds the native library locally. For reproducible
|
|
50
|
+
deployments, replace `main` with a reviewed full commit hash. `--no-deps`
|
|
51
|
+
preserves the preinstalled Torch/PTA versions; build tools are still provisioned
|
|
52
|
+
by pip. For an offline build, preinstall the listed build tools and additionally
|
|
53
|
+
pass `--no-build-isolation`, using a local source checkout.
|
|
54
|
+
|
|
55
|
+
The project is not yet published on PyPI: the bare command
|
|
56
|
+
`pip install torch-npu-memory-saver` is not an available installation route.
|
|
57
|
+
|
|
58
|
+
### Build and keep a wheel
|
|
59
|
+
|
|
60
|
+
In an environment with the matching Torch/PTA installation:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
git clone https://github.com/yanlonghu/torch_npu_memory_saver.git
|
|
64
|
+
cd torch_npu_memory_saver
|
|
65
|
+
|
|
66
|
+
# Replace this with your deployment's CANN toolkit directory.
|
|
67
|
+
export ASCEND_HOME_PATH=/path/to/cann
|
|
68
|
+
source "$ASCEND_HOME_PATH/set_env.sh"
|
|
69
|
+
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
|
|
70
|
+
unset PYTORCH_NO_NPU_MEMORY_CACHING
|
|
71
|
+
|
|
72
|
+
python -m pip wheel --no-deps . --wheel-dir dist
|
|
73
|
+
python -m pip install --no-deps dist/torch_npu_memory_saver-0.1.0-*.whl
|
|
74
|
+
python -c 'from importlib.metadata import version; print(version("torch-npu-memory-saver"))'
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
If your allocation configuration has other options, preserve them while setting
|
|
78
|
+
`expandable_segments:False`. Set the environment before starting the worker.
|
|
79
|
+
Build on the target platform against its CANN installation. The wheel contains
|
|
80
|
+
the Saver `.so`, not CANN or system libraries; those must remain available at
|
|
81
|
+
runtime. Do not use editable installation as a substitute for the wheel build.
|
|
82
|
+
|
|
83
|
+
The package locates its bundled `.so` automatically. For a separate native build,
|
|
84
|
+
`TORCH_NPU_MEMORY_SAVER_LIBRARY` may override it with an existing absolute path.
|
|
85
|
+
The old `TOKENSPEED_NPU_MEMORY_SAVER_LIBRARY` variable is not used. Import
|
|
86
|
+
`torch_npu` and select the device before the first allocation or transition.
|
|
87
|
+
|
|
88
|
+
## API and contract
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
import torch
|
|
92
|
+
import torch_npu
|
|
93
|
+
from torch_npu_memory_saver import memory_saver
|
|
94
|
+
|
|
95
|
+
torch.npu.set_device(0)
|
|
96
|
+
with memory_saver.region(tag="weights", enable_cpu_backup=True):
|
|
97
|
+
tensor = torch.ones(1024, device="npu")
|
|
98
|
+
# Stop new work and coordinate other ranks before calling pause.
|
|
99
|
+
memory_saver.pause("weights")
|
|
100
|
+
# Do not read or write tensor while its physical backing is released.
|
|
101
|
+
memory_saver.resume("weights")
|
|
102
|
+
torch.testing.assert_close(tensor.cpu(), torch.ones(1024))
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
For discard, use `enable_cpu_backup=False`. After `resume()`, write all required
|
|
106
|
+
contents before reading the tensor or replaying a graph that consumes it.
|
|
107
|
+
`resume()` restores mappings; it does not reload or commit model weights.
|
|
108
|
+
|
|
109
|
+
`region(tag, enable_cpu_backup)` routes allocations through owned PTA MemPools.
|
|
110
|
+
This minimal extraction retains fixed tags: weights (backup/discard), kv_cache
|
|
111
|
+
(discard), acl_graph (backup/discard). `allocation_region(PoolKey(...))` exposes
|
|
112
|
+
separate pool scopes under one release tag. REMAIN uses the normal allocator.
|
|
113
|
+
`npu_graph(graph, *, pool, stream, tag, policy, pool_scope,
|
|
114
|
+
auto_dispatch_capture)` captures into an owned pool. `pause(tag=None)` and
|
|
115
|
+
`resume(tag=None)` target one tag or all tags. `assert_active()` rejects paused
|
|
116
|
+
or failed regions. `stats()` reports allocator accounting;
|
|
117
|
+
`validate_allocations(allocations, graphs=...)` checks storage and pool ownership.
|
|
118
|
+
Each allocation record is `(tag, name, pointer, byte_size)`.
|
|
119
|
+
|
|
120
|
+
Backup restores bytes at the same virtual address. Discard restores mappings
|
|
121
|
+
with undefined contents: initialize before use. Graph executables and HCCL
|
|
122
|
+
resources stay resident. All captured persistent state must be backed up or
|
|
123
|
+
resident, and replay must overwrite discarded temporaries before reading them.
|
|
124
|
+
One device/context per process; all instances share pools. Nested scopes are
|
|
125
|
+
rejected. Callers must drain and block new work; these guards do not intercept
|
|
126
|
+
arbitrary tensor access. A failed transition can be partial. Do not reopen work
|
|
127
|
+
until recovery is complete; callback/rollback faults require worker restart.
|
|
128
|
+
|
|
129
|
+
This library does not implement model loaders, KV invalidation, weight commits,
|
|
130
|
+
multi-rank protocol 3, communication initialization, or process-group shutdown.
|
|
131
|
+
TokenSpeed retains those responsibilities. It does not depend on TokenSpeed or
|
|
132
|
+
on the CUDA `torch_memory_saver` package.
|
|
133
|
+
|
|
134
|
+
## Tests
|
|
135
|
+
|
|
136
|
+
Host Python tests require Torch and pytest but no NPU. Use a separate CPU test
|
|
137
|
+
environment if needed; do not replace the Torch installation in a deployment:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
python -m pip install 'pytest>=8'
|
|
141
|
+
PYTHONPATH=python python -m pytest -q tests
|
|
142
|
+
|
|
143
|
+
# C++ state and error injection tests do not require Torch or CANN.
|
|
144
|
+
cmake -S native -B build-test -DSAVER_FAKE_ACL=ON
|
|
145
|
+
cmake --build build-test
|
|
146
|
+
ctest --test-dir build-test --output-on-failure
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Host tests do not establish NPU device correctness. They skip the real NPU test
|
|
150
|
+
unless explicitly enabled.
|
|
151
|
+
|
|
152
|
+
On an exclusively assigned NPU, run the independent graph test with
|
|
153
|
+
`RUN_NPU_MEMORY_SAVER_TESTS=1 python -m pytest tests/test_npu_graph.py`. It checks
|
|
154
|
+
three backup/discard cycles, same addresses, and replay after poisoning owned
|
|
155
|
+
discard storage. This is opt-in even when a native library is installed.
|
|
156
|
+
|
|
157
|
+
## Publishing
|
|
158
|
+
|
|
159
|
+
Maintainers publish source distributions with the manually triggered
|
|
160
|
+
`.github/workflows/publish.yml` workflow on `main`. The `pypi` GitHub environment
|
|
161
|
+
uses PyPI Trusted Publishing; no long-lived API token is required. Register
|
|
162
|
+
owner `yanlonghu`, repository `torch_npu_memory_saver`, workflow `publish.yml`,
|
|
163
|
+
and environment `pypi` in PyPI before the first run. Increment the project
|
|
164
|
+
version for subsequent releases; PyPI does not allow replacing a release file.
|
|
165
|
+
|
|
166
|
+
The workflow validates package metadata and uploads only the source archive.
|
|
167
|
+
Installation compiles the native library against the user's configured CANN
|
|
168
|
+
toolkit. It does not imply support for untested Torch/PTA/CANN combinations.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Validation coverage
|
|
2
|
+
|
|
3
|
+
The initial extraction retains the allocator implementation from TokenSpeed
|
|
4
|
+
`269b59a95f258188ce380933ef98f418bea2abe6`. The standalone library does not
|
|
5
|
+
implement TokenSpeed's model loading, cache invalidation or multi-rank protocol.
|
|
6
|
+
|
|
7
|
+
## Tested stack
|
|
8
|
+
|
|
9
|
+
- Atlas A2 / 910B3, Python 3.12, Torch 2.9.0+cpu, CANN/HCCL 9.0.0.
|
|
10
|
+
- PTA baseline [9d40ac3](https://gitcode.com/Ascend/pytorch/commit/9d40ac3a2b3714df7e5d164bda28a42eff3282cd)
|
|
11
|
+
with official backports [!39221](https://gitcode.com/Ascend/pytorch/merge_requests/39221),
|
|
12
|
+
[!45270](https://gitcode.com/Ascend/pytorch/merge_requests/45270),
|
|
13
|
+
[!30625](https://gitcode.com/Ascend/pytorch/merge_requests/30625),
|
|
14
|
+
[!32154](https://gitcode.com/Ascend/pytorch/merge_requests/32154),
|
|
15
|
+
[!32564](https://gitcode.com/Ascend/pytorch/merge_requests/32564), and
|
|
16
|
+
[!38431](https://gitcode.com/Ascend/pytorch/merge_requests/38431).
|
|
17
|
+
- Installed PTA identifies itself as `2.9.0+gitee1742b`, git
|
|
18
|
+
`ee1742b06d2c0e59cb0b357288ec29ee190cfd2d`.
|
|
19
|
+
|
|
20
|
+
## Results
|
|
21
|
+
|
|
22
|
+
| Check | Result |
|
|
23
|
+
| --- | --- |
|
|
24
|
+
| Independent Linux aarch64 wheel build and install | Passed. Built from this library's source distribution. |
|
|
25
|
+
| Standalone NPU graph | Passed three backup/discard cycles with stable addresses and replay after poisoning discarded temporary storage. TokenSpeed imports were forbidden. |
|
|
26
|
+
| Native error-state tests | CTest 4/4 passed on Linux using fake ACL, including failed allocation rollback and wrong-device/context free callbacks. |
|
|
27
|
+
| TokenSpeed TP2 partial `cpu_backup` update | Passed. Two checkpoint tensors updated; output matched a fresh complete effective checkpoint for batches 1, 3 and 4. |
|
|
28
|
+
| TokenSpeed TP2 complete `discard` reload | Passed. 311 checkpoint tensors loaded; both ranks committed before KV wake. Output matched a fresh new-checkpoint start. |
|
|
29
|
+
| Original graph and address retention | Passed for both weight policies, including two additional same-weight poison/restore cycles. No graph recapture after startup. |
|
|
30
|
+
| TokenSpeed protocol 3 | Both ranks negotiated v3. Holding one rank after its device operation prevented a public success reply for 300 ms; releasing it permitted completion. |
|
|
31
|
+
| Installed native-library provenance | All six integration ranks mapped the standalone wheel's library. Both library-path overrides were unset. |
|
|
32
|
+
| Worker cleanup | All successful cases exited normally and released their device leases. |
|
|
33
|
+
|
|
34
|
+
The installed Saver library was freshly compiled. Unchanged TokenSpeed host and
|
|
35
|
+
scheduler binaries were reused. No PTA, Rust or system-library modifications were
|
|
36
|
+
made for this validation. The protocol test verifies return/acknowledgement gating,
|
|
37
|
+
not asymmetric HCCL drain timing or native acknowledgement timestamps.
|
|
38
|
+
|
|
39
|
+
One initial protocol probe failed before device execution because two external
|
|
40
|
+
test helpers shared a module name. A dedicated observer module fixed the test;
|
|
41
|
+
no library or TokenSpeed production change was needed. The retry passed.
|
|
42
|
+
|
|
43
|
+
## Limits
|
|
44
|
+
|
|
45
|
+
This extraction has not repeated TP4/TP8, cross-host, Mooncake transport,
|
|
46
|
+
extended speculative-decoding, long-cycle leak or performance campaigns.
|
|
47
|
+
PTA 2.13 compatibility has not been revalidated. CUDA/ROCm isolation was checked
|
|
48
|
+
through unchanged source boundaries and CPU behavior; GPU hardware was unavailable.
|
|
49
|
+
|
|
50
|
+
Run the repository's host and opt-in NPU tests as described in the README.
|
|
51
|
+
TokenSpeed integration scenarios require its separate adapter and runtime.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
cmake_minimum_required(VERSION 3.18)
|
|
2
|
+
project(ascend_memory_saver LANGUAGES CXX)
|
|
3
|
+
|
|
4
|
+
option(SAVER_FAKE_ACL "Build only host-side state-machine tests" OFF)
|
|
5
|
+
set(CMAKE_CXX_STANDARD 17)
|
|
6
|
+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
|
7
|
+
set(CMAKE_CXX_EXTENSIONS OFF)
|
|
8
|
+
find_package(Threads REQUIRED)
|
|
9
|
+
|
|
10
|
+
if(SAVER_FAKE_ACL)
|
|
11
|
+
set(saver_target saver_state_tests)
|
|
12
|
+
add_executable(${saver_target} src/ascend_saver.cpp tests/fake_acl.cpp tests/state_tests.cpp)
|
|
13
|
+
target_compile_definitions(${saver_target} PRIVATE SAVER_FAKE_ACL=1)
|
|
14
|
+
target_include_directories(${saver_target} PRIVATE tests)
|
|
15
|
+
enable_testing()
|
|
16
|
+
add_test(NAME saver_state_tests COMMAND ${saver_target})
|
|
17
|
+
# Each callback fault is terminal for its process.
|
|
18
|
+
add_test(NAME saver_void_free_wrong_context COMMAND ${saver_target} wrong-context)
|
|
19
|
+
add_test(NAME saver_void_free_wrong_device COMMAND ${saver_target} wrong-device)
|
|
20
|
+
add_test(NAME saver_failed_malloc_rollback COMMAND ${saver_target} malloc-rollback)
|
|
21
|
+
else()
|
|
22
|
+
set(saver_target ascend_memory_saver)
|
|
23
|
+
add_library(${saver_target} SHARED src/ascend_saver.cpp)
|
|
24
|
+
set_target_properties(${saver_target} PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN YES)
|
|
25
|
+
set(ASCEND_HOME_PATH "$ENV{ASCEND_HOME_PATH}" CACHE PATH "CANN toolkit root")
|
|
26
|
+
if(NOT ASCEND_HOME_PATH)
|
|
27
|
+
message(FATAL_ERROR "Set ASCEND_HOME_PATH to the target CANN installation")
|
|
28
|
+
endif()
|
|
29
|
+
find_path(ACL_INCLUDE_DIR acl/acl_rt.h PATHS
|
|
30
|
+
"${ASCEND_HOME_PATH}/include"
|
|
31
|
+
"${ASCEND_HOME_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/include"
|
|
32
|
+
NO_DEFAULT_PATH REQUIRED)
|
|
33
|
+
find_library(ACL_RT_LIBRARY acl_rt PATHS
|
|
34
|
+
"${ASCEND_HOME_PATH}/lib64"
|
|
35
|
+
"${ASCEND_HOME_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/lib64"
|
|
36
|
+
NO_DEFAULT_PATH REQUIRED)
|
|
37
|
+
target_include_directories(${saver_target} PRIVATE "${ACL_INCLUDE_DIR}")
|
|
38
|
+
target_link_libraries(${saver_target} PRIVATE "${ACL_RT_LIBRARY}")
|
|
39
|
+
target_link_options(${saver_target} PRIVATE "LINKER:--no-undefined")
|
|
40
|
+
include(GNUInstallDirs)
|
|
41
|
+
install(TARGETS ${saver_target} LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}")
|
|
42
|
+
endif()
|
|
43
|
+
|
|
44
|
+
target_include_directories(${saver_target} PRIVATE include)
|
|
45
|
+
target_link_libraries(${saver_target} PRIVATE Threads::Threads)
|
|
46
|
+
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
|
|
47
|
+
target_compile_options(${saver_target} PRIVATE -Wall -Wextra -Wpedantic)
|
|
48
|
+
endif()
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <stddef.h>
|
|
4
|
+
#ifdef SAVER_FAKE_ACL
|
|
5
|
+
#include "fake_acl.h"
|
|
6
|
+
#else
|
|
7
|
+
#include <acl/acl_rt.h>
|
|
8
|
+
#endif
|
|
9
|
+
|
|
10
|
+
#if defined(_WIN32)
|
|
11
|
+
#define SAVER_API __declspec(dllexport)
|
|
12
|
+
#else
|
|
13
|
+
#define SAVER_API __attribute__((visibility("default")))
|
|
14
|
+
#endif
|
|
15
|
+
|
|
16
|
+
#ifdef __cplusplus
|
|
17
|
+
extern "C" {
|
|
18
|
+
#endif
|
|
19
|
+
|
|
20
|
+
/* ABI v2. Bind the worker's NPU device before calling any mutating API.
|
|
21
|
+
* The caller must stop new launches/allocations and wait for all prior users
|
|
22
|
+
* before pause/resume. Final free synchronizes the current device before
|
|
23
|
+
* unmapping; the owner must still prevent new uses of the released storage.
|
|
24
|
+
* Stream arguments identify the torch_npu ABI; this backend uses whole-device
|
|
25
|
+
* fences rather than inserting stream-order dependencies.
|
|
26
|
+
*/
|
|
27
|
+
SAVER_API int saver_abi_version(void);
|
|
28
|
+
/* Sticky first callback fault from a void free or failed allocation rollback.
|
|
29
|
+
* A return value of 1 means restart the worker before reuse.
|
|
30
|
+
* Allocation/pause/resume fail closed. Explicit release/cleanup remain allowed.
|
|
31
|
+
*/
|
|
32
|
+
SAVER_API int saver_has_callback_fault(void);
|
|
33
|
+
SAVER_API void *weights_malloc(size_t size, int device, aclrtStream stream);
|
|
34
|
+
SAVER_API void *weights_backup_malloc(size_t size, int device, aclrtStream stream);
|
|
35
|
+
SAVER_API void *weights_discard_malloc(size_t size, int device, aclrtStream stream);
|
|
36
|
+
SAVER_API void *kv_malloc(size_t size, int device, aclrtStream stream);
|
|
37
|
+
SAVER_API void *graph_malloc(size_t size, int device, aclrtStream stream);
|
|
38
|
+
SAVER_API void *graph_backup_malloc(size_t size, int device, aclrtStream stream);
|
|
39
|
+
SAVER_API void *graph_discard_malloc(size_t size, int device, aclrtStream stream);
|
|
40
|
+
/* Retires a known allocation even if context or device validation fails. */
|
|
41
|
+
SAVER_API void saver_free(void *ptr, size_t size, int device, aclrtStream stream);
|
|
42
|
+
|
|
43
|
+
/* tag: weights, kv_cache, acl_graph, or NULL for all.
|
|
44
|
+
* Return 0 on success, 1 on failure. A failed batch can be partially applied;
|
|
45
|
+
* keep the external launch gate closed and retry pause/resume/release.
|
|
46
|
+
*/
|
|
47
|
+
SAVER_API int saver_pause(const char *tag);
|
|
48
|
+
SAVER_API int saver_resume(const char *tag);
|
|
49
|
+
/* A context or device validation failure leaves the segment phase unchanged. */
|
|
50
|
+
SAVER_API int saver_release(void *ptr);
|
|
51
|
+
/* Retry cleanup only for blocks already marked releasing by a failed alloc/free. */
|
|
52
|
+
SAVER_API int saver_collect_failed(void);
|
|
53
|
+
|
|
54
|
+
/* Thread-local strings, valid until the next same-thread call to the same
|
|
55
|
+
* function (stats), or the next mutating operation (last_error). Copy promptly.
|
|
56
|
+
* stats also contains the latest backend error, including callback errors
|
|
57
|
+
* produced on another thread. No ACL calls are made by these read-only APIs.
|
|
58
|
+
*/
|
|
59
|
+
SAVER_API const char *saver_last_error(void);
|
|
60
|
+
SAVER_API const char *saver_stats(void);
|
|
61
|
+
|
|
62
|
+
#ifdef __cplusplus
|
|
63
|
+
}
|
|
64
|
+
#endif
|