torch-npu-memory-saver 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. torch_npu_memory_saver-0.1.0/LICENSE +21 -0
  2. torch_npu_memory_saver-0.1.0/MANIFEST.in +2 -0
  3. torch_npu_memory_saver-0.1.0/PKG-INFO +206 -0
  4. torch_npu_memory_saver-0.1.0/README.md +168 -0
  5. torch_npu_memory_saver-0.1.0/docs/validation.md +51 -0
  6. torch_npu_memory_saver-0.1.0/native/CMakeLists.txt +48 -0
  7. torch_npu_memory_saver-0.1.0/native/include/ascend_saver.h +64 -0
  8. torch_npu_memory_saver-0.1.0/native/src/ascend_saver.cpp +472 -0
  9. torch_npu_memory_saver-0.1.0/native/tests/fake_acl.cpp +134 -0
  10. torch_npu_memory_saver-0.1.0/native/tests/fake_acl.h +68 -0
  11. torch_npu_memory_saver-0.1.0/native/tests/state_tests.cpp +341 -0
  12. torch_npu_memory_saver-0.1.0/pyproject.toml +26 -0
  13. torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver/__init__.py +34 -0
  14. torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver/saver.py +293 -0
  15. torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver/state.py +158 -0
  16. torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver.egg-info/PKG-INFO +206 -0
  17. torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver.egg-info/SOURCES.txt +24 -0
  18. torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver.egg-info/dependency_links.txt +1 -0
  19. torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver.egg-info/requires.txt +5 -0
  20. torch_npu_memory_saver-0.1.0/python/torch_npu_memory_saver.egg-info/top_level.txt +1 -0
  21. torch_npu_memory_saver-0.1.0/setup.cfg +4 -0
  22. torch_npu_memory_saver-0.1.0/setup.py +37 -0
  23. torch_npu_memory_saver-0.1.0/tests/test_import_boundary.py +18 -0
  24. torch_npu_memory_saver-0.1.0/tests/test_library_path.py +36 -0
  25. torch_npu_memory_saver-0.1.0/tests/test_npu_graph.py +108 -0
  26. torch_npu_memory_saver-0.1.0/tests/test_state.py +158 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 LightSeek Foundation <contact@lightseek.org>
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,2 @@
1
+ recursive-include native *.cpp *.h CMakeLists.txt
2
+ recursive-include docs *.md
@@ -0,0 +1,206 @@
1
+ Metadata-Version: 2.4
2
+ Name: torch-npu-memory-saver
3
+ Version: 0.1.0
4
+ Summary: Managed Ascend memory with stable virtual addresses
5
+ License: MIT License
6
+
7
+ Copyright (c) 2026 LightSeek Foundation <contact@lightseek.org>
8
+
9
+ Permission is hereby granted, free of charge, to any person obtaining a copy
10
+ of this software and associated documentation files (the "Software"), to deal
11
+ in the Software without restriction, including without limitation the rights
12
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13
+ copies of the Software, and to permit persons to whom the Software is
14
+ furnished to do so, subject to the following conditions:
15
+
16
+ The above copyright notice and this permission notice shall be included in all
17
+ copies or substantial portions of the Software.
18
+
19
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25
+ SOFTWARE.
26
+
27
+ Project-URL: Repository, https://github.com/yanlonghu/torch_npu_memory_saver
28
+ Project-URL: Issues, https://github.com/yanlonghu/torch_npu_memory_saver/issues
29
+ Project-URL: Documentation, https://github.com/yanlonghu/torch_npu_memory_saver/blob/main/README.md
30
+ Requires-Python: >=3.11
31
+ Description-Content-Type: text/markdown
32
+ License-File: LICENSE
33
+ Requires-Dist: torch>=2.9
34
+ Requires-Dist: torch-npu>=2.9
35
+ Provides-Extra: test
36
+ Requires-Dist: pytest>=8; extra == "test"
37
+ Dynamic: license-file
38
+
39
+ # torch_npu_memory_saver
40
+
41
+ Release managed Ascend device memory and restore it at the same virtual addresses.
42
+ Extracted from [TokenSpeed PR #3530](https://github.com/lightseekorg/tokenspeed/pull/3530)
43
+ at `269b59a9`. The native allocator retains ABI 2. No PyPI package has been published.
44
+
45
+ ## Dependencies
46
+
47
+ | Component | Requirement / tested configuration |
48
+ | --- | --- |
49
+ | Hardware and OS | Validated on Atlas A2 / 910B3, Linux aarch64. |
50
+ | Python | Package requires 3.11+; device validation used 3.12. |
51
+ | PyTorch | Device validation used `torch==2.9.0+cpu`, paired with PTA below. |
52
+ | PTA (`torch_npu`) | `9d40ac3` (2.9-730) **with six official backports**; a stock version number alone is insufficient. |
53
+ | CANN / HCCL | Validated with 9.0.0 and the deployment's installed Ascend driver. |
54
+ | Native build | CANN development headers and `acl_rt`, CMake 3.18+, a C++17 compiler. |
55
+ | Python build | Setuptools 69+, wheel; pip provisions these and CMake in its build environment. |
56
+ | Tests | Pytest 8+; the real-device test additionally needs the configured NPU stack. |
57
+
58
+ The six PTA backports are
59
+ [!39221](https://gitcode.com/Ascend/pytorch/merge_requests/39221),
60
+ [!45270](https://gitcode.com/Ascend/pytorch/merge_requests/45270),
61
+ [!30625](https://gitcode.com/Ascend/pytorch/merge_requests/30625),
62
+ [!32154](https://gitcode.com/Ascend/pytorch/merge_requests/32154),
63
+ [!32564](https://gitcode.com/Ascend/pytorch/merge_requests/32564), and
64
+ [!38431](https://gitcode.com/Ascend/pytorch/merge_requests/38431), applied to
65
+ [9d40ac3](https://gitcode.com/Ascend/pytorch/commit/9d40ac3a2b3714df7e5d164bda28a42eff3282cd).
66
+ This is the validated deployment prerequisite, not a claim that every backport
67
+ is needed for every standalone operation. See [validation coverage](docs/validation.md)
68
+ for the installed build identity and integration results. PTA 2.13 and other
69
+ hardware/CANN combinations have not been requalified for this extraction.
70
+
71
+ Package metadata declares `torch>=2.9` and `torch-npu>=2.9` as minimum versions,
72
+ **not a compatibility matrix**. Pip cannot verify the required source patches
73
+ or choose a matching Torch/PTA/CANN stack. Prepare that stack first, then use
74
+ `--no-deps` below to preserve it. This package does not install CANN, drivers,
75
+ TokenSpeed, Triton or CUDA's `torch_memory_saver`.
76
+
77
+ ## Install with pip
78
+
79
+ After preparing the matching Torch/PTA stack and loading the CANN toolkit
80
+ environment, install directly from the public repository:
81
+
82
+ ```bash
83
+ python -m pip install --no-deps \
84
+ "git+https://github.com/yanlonghu/torch_npu_memory_saver.git@main"
85
+ ```
86
+
87
+ This requires Git and builds the native library locally. For reproducible
88
+ deployments, replace `main` with a reviewed full commit hash. `--no-deps`
89
+ preserves the preinstalled Torch/PTA versions; build tools are still provisioned
90
+ by pip. For an offline build, preinstall the listed build tools and additionally
91
+ pass `--no-build-isolation`, using a local source checkout.
92
+
93
+ The project is not yet published on PyPI: the bare command
94
+ `pip install torch-npu-memory-saver` is not an available installation route.
95
+
96
+ ### Build and keep a wheel
97
+
98
+ In an environment with the matching Torch/PTA installation:
99
+
100
+ ```bash
101
+ git clone https://github.com/yanlonghu/torch_npu_memory_saver.git
102
+ cd torch_npu_memory_saver
103
+
104
+ # Replace this with your deployment's CANN toolkit directory.
105
+ export ASCEND_HOME_PATH=/path/to/cann
106
+ source "$ASCEND_HOME_PATH/set_env.sh"
107
+ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
108
+ unset PYTORCH_NO_NPU_MEMORY_CACHING
109
+
110
+ python -m pip wheel --no-deps . --wheel-dir dist
111
+ python -m pip install --no-deps dist/torch_npu_memory_saver-0.1.0-*.whl
112
+ python -c 'from importlib.metadata import version; print(version("torch-npu-memory-saver"))'
113
+ ```
114
+
115
+ If your allocation configuration has other options, preserve them while setting
116
+ `expandable_segments:False`. Set the environment before starting the worker.
117
+ Build on the target platform against its CANN installation. The wheel contains
118
+ the Saver `.so`, not CANN or system libraries; those must remain available at
119
+ runtime. Do not use editable installation as a substitute for the wheel build.
120
+
121
+ The package locates its bundled `.so` automatically. For a separate native build,
122
+ `TORCH_NPU_MEMORY_SAVER_LIBRARY` may override it with an existing absolute path.
123
+ The old `TOKENSPEED_NPU_MEMORY_SAVER_LIBRARY` variable is not used. Import
124
+ `torch_npu` and select the device before the first allocation or transition.
125
+
126
+ ## API and contract
127
+
128
+ ```python
129
+ import torch
130
+ import torch_npu
131
+ from torch_npu_memory_saver import memory_saver
132
+
133
+ torch.npu.set_device(0)
134
+ with memory_saver.region(tag="weights", enable_cpu_backup=True):
135
+ tensor = torch.ones(1024, device="npu")
136
+ # Stop new work and coordinate other ranks before calling pause.
137
+ memory_saver.pause("weights")
138
+ # Do not read or write tensor while its physical backing is released.
139
+ memory_saver.resume("weights")
140
+ torch.testing.assert_close(tensor.cpu(), torch.ones(1024))
141
+ ```
142
+
143
+ For discard, use `enable_cpu_backup=False`. After `resume()`, write all required
144
+ contents before reading the tensor or replaying a graph that consumes it.
145
+ `resume()` restores mappings; it does not reload or commit model weights.
146
+
147
+ `region(tag, enable_cpu_backup)` routes allocations through owned PTA MemPools.
148
+ This minimal extraction retains fixed tags: weights (backup/discard), kv_cache
149
+ (discard), acl_graph (backup/discard). `allocation_region(PoolKey(...))` exposes
150
+ separate pool scopes under one release tag. REMAIN uses the normal allocator.
151
+ `npu_graph(graph, *, pool, stream, tag, policy, pool_scope,
152
+ auto_dispatch_capture)` captures into an owned pool. `pause(tag=None)` and
153
+ `resume(tag=None)` target one tag or all tags. `assert_active()` rejects paused
154
+ or failed regions. `stats()` reports allocator accounting;
155
+ `validate_allocations(allocations, graphs=...)` checks storage and pool ownership.
156
+ Each allocation record is `(tag, name, pointer, byte_size)`.
157
+
158
+ Backup restores bytes at the same virtual address. Discard restores mappings
159
+ with undefined contents: initialize before use. Graph executables and HCCL
160
+ resources stay resident. All captured persistent state must be backed up or
161
+ resident, and replay must overwrite discarded temporaries before reading them.
162
+ One device/context per process; all instances share pools. Nested scopes are
163
+ rejected. Callers must drain and block new work; these guards do not intercept
164
+ arbitrary tensor access. A failed transition can be partial. Do not reopen work
165
+ until recovery is complete; callback/rollback faults require worker restart.
166
+
167
+ This library does not implement model loaders, KV invalidation, weight commits,
168
+ multi-rank protocol 3, communication initialization, or process-group shutdown.
169
+ TokenSpeed retains those responsibilities. It does not depend on TokenSpeed or
170
+ on the CUDA `torch_memory_saver` package.
171
+
172
+ ## Tests
173
+
174
+ Host Python tests require Torch and pytest but no NPU. Use a separate CPU test
175
+ environment if needed; do not replace the Torch installation in a deployment:
176
+
177
+ ```bash
178
+ python -m pip install 'pytest>=8'
179
+ PYTHONPATH=python python -m pytest -q tests
180
+
181
+ # C++ state and error injection tests do not require Torch or CANN.
182
+ cmake -S native -B build-test -DSAVER_FAKE_ACL=ON
183
+ cmake --build build-test
184
+ ctest --test-dir build-test --output-on-failure
185
+ ```
186
+
187
+ Host tests do not establish NPU device correctness. They skip the real NPU test
188
+ unless explicitly enabled.
189
+
190
+ On an exclusively assigned NPU, run the independent graph test with
191
+ `RUN_NPU_MEMORY_SAVER_TESTS=1 python -m pytest tests/test_npu_graph.py`. It checks
192
+ three backup/discard cycles, same addresses, and replay after poisoning owned
193
+ discard storage. This is opt-in even when a native library is installed.
194
+
195
+ ## Publishing
196
+
197
+ Maintainers publish source distributions with the manually triggered
198
+ `.github/workflows/publish.yml` workflow on `main`. The `pypi` GitHub environment
199
+ uses PyPI Trusted Publishing; no long-lived API token is required. Register
200
+ owner `yanlonghu`, repository `torch_npu_memory_saver`, workflow `publish.yml`,
201
+ and environment `pypi` in PyPI before the first run. Increment the project
202
+ version for subsequent releases; PyPI does not allow replacing a release file.
203
+
204
+ The workflow validates package metadata and uploads only the source archive.
205
+ Installation compiles the native library against the user's configured CANN
206
+ toolkit. It does not imply support for untested Torch/PTA/CANN combinations.
@@ -0,0 +1,168 @@
1
+ # torch_npu_memory_saver
2
+
3
+ Release managed Ascend device memory and restore it at the same virtual addresses.
4
+ Extracted from [TokenSpeed PR #3530](https://github.com/lightseekorg/tokenspeed/pull/3530)
5
+ at `269b59a9`. The native allocator retains ABI 2. No PyPI package has been published.
6
+
7
+ ## Dependencies
8
+
9
+ | Component | Requirement / tested configuration |
10
+ | --- | --- |
11
+ | Hardware and OS | Validated on Atlas A2 / 910B3, Linux aarch64. |
12
+ | Python | Package requires 3.11+; device validation used 3.12. |
13
+ | PyTorch | Device validation used `torch==2.9.0+cpu`, paired with PTA below. |
14
+ | PTA (`torch_npu`) | `9d40ac3` (2.9-730) **with six official backports**; a stock version number alone is insufficient. |
15
+ | CANN / HCCL | Validated with 9.0.0 and the deployment's installed Ascend driver. |
16
+ | Native build | CANN development headers and `acl_rt`, CMake 3.18+, a C++17 compiler. |
17
+ | Python build | Setuptools 69+, wheel; pip provisions these and CMake in its build environment. |
18
+ | Tests | Pytest 8+; the real-device test additionally needs the configured NPU stack. |
19
+
20
+ The six PTA backports are
21
+ [!39221](https://gitcode.com/Ascend/pytorch/merge_requests/39221),
22
+ [!45270](https://gitcode.com/Ascend/pytorch/merge_requests/45270),
23
+ [!30625](https://gitcode.com/Ascend/pytorch/merge_requests/30625),
24
+ [!32154](https://gitcode.com/Ascend/pytorch/merge_requests/32154),
25
+ [!32564](https://gitcode.com/Ascend/pytorch/merge_requests/32564), and
26
+ [!38431](https://gitcode.com/Ascend/pytorch/merge_requests/38431), applied to
27
+ [9d40ac3](https://gitcode.com/Ascend/pytorch/commit/9d40ac3a2b3714df7e5d164bda28a42eff3282cd).
28
+ This is the validated deployment prerequisite, not a claim that every backport
29
+ is needed for every standalone operation. See [validation coverage](docs/validation.md)
30
+ for the installed build identity and integration results. PTA 2.13 and other
31
+ hardware/CANN combinations have not been requalified for this extraction.
32
+
33
+ Package metadata declares `torch>=2.9` and `torch-npu>=2.9` as minimum versions,
34
+ **not a compatibility matrix**. Pip cannot verify the required source patches
35
+ or choose a matching Torch/PTA/CANN stack. Prepare that stack first, then use
36
+ `--no-deps` below to preserve it. This package does not install CANN, drivers,
37
+ TokenSpeed, Triton or CUDA's `torch_memory_saver`.
38
+
39
+ ## Install with pip
40
+
41
+ After preparing the matching Torch/PTA stack and loading the CANN toolkit
42
+ environment, install directly from the public repository:
43
+
44
+ ```bash
45
+ python -m pip install --no-deps \
46
+ "git+https://github.com/yanlonghu/torch_npu_memory_saver.git@main"
47
+ ```
48
+
49
+ This requires Git and builds the native library locally. For reproducible
50
+ deployments, replace `main` with a reviewed full commit hash. `--no-deps`
51
+ preserves the preinstalled Torch/PTA versions; build tools are still provisioned
52
+ by pip. For an offline build, preinstall the listed build tools and additionally
53
+ pass `--no-build-isolation`, using a local source checkout.
54
+
55
+ The project is not yet published on PyPI: the bare command
56
+ `pip install torch-npu-memory-saver` is not an available installation route.
57
+
58
+ ### Build and keep a wheel
59
+
60
+ In an environment with the matching Torch/PTA installation:
61
+
62
+ ```bash
63
+ git clone https://github.com/yanlonghu/torch_npu_memory_saver.git
64
+ cd torch_npu_memory_saver
65
+
66
+ # Replace this with your deployment's CANN toolkit directory.
67
+ export ASCEND_HOME_PATH=/path/to/cann
68
+ source "$ASCEND_HOME_PATH/set_env.sh"
69
+ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
70
+ unset PYTORCH_NO_NPU_MEMORY_CACHING
71
+
72
+ python -m pip wheel --no-deps . --wheel-dir dist
73
+ python -m pip install --no-deps dist/torch_npu_memory_saver-0.1.0-*.whl
74
+ python -c 'from importlib.metadata import version; print(version("torch-npu-memory-saver"))'
75
+ ```
76
+
77
+ If your allocation configuration has other options, preserve them while setting
78
+ `expandable_segments:False`. Set the environment before starting the worker.
79
+ Build on the target platform against its CANN installation. The wheel contains
80
+ the Saver `.so`, not CANN or system libraries; those must remain available at
81
+ runtime. Do not use editable installation as a substitute for the wheel build.
82
+
83
+ The package locates its bundled `.so` automatically. For a separate native build,
84
+ `TORCH_NPU_MEMORY_SAVER_LIBRARY` may override it with an existing absolute path.
85
+ The old `TOKENSPEED_NPU_MEMORY_SAVER_LIBRARY` variable is not used. Import
86
+ `torch_npu` and select the device before the first allocation or transition.
87
+
88
+ ## API and contract
89
+
90
+ ```python
91
+ import torch
92
+ import torch_npu
93
+ from torch_npu_memory_saver import memory_saver
94
+
95
+ torch.npu.set_device(0)
96
+ with memory_saver.region(tag="weights", enable_cpu_backup=True):
97
+ tensor = torch.ones(1024, device="npu")
98
+ # Stop new work and coordinate other ranks before calling pause.
99
+ memory_saver.pause("weights")
100
+ # Do not read or write tensor while its physical backing is released.
101
+ memory_saver.resume("weights")
102
+ torch.testing.assert_close(tensor.cpu(), torch.ones(1024))
103
+ ```
104
+
105
+ For discard, use `enable_cpu_backup=False`. After `resume()`, write all required
106
+ contents before reading the tensor or replaying a graph that consumes it.
107
+ `resume()` restores mappings; it does not reload or commit model weights.
108
+
109
+ `region(tag, enable_cpu_backup)` routes allocations through owned PTA MemPools.
110
+ This minimal extraction retains fixed tags: weights (backup/discard), kv_cache
111
+ (discard), acl_graph (backup/discard). `allocation_region(PoolKey(...))` exposes
112
+ separate pool scopes under one release tag. REMAIN uses the normal allocator.
113
+ `npu_graph(graph, *, pool, stream, tag, policy, pool_scope,
114
+ auto_dispatch_capture)` captures into an owned pool. `pause(tag=None)` and
115
+ `resume(tag=None)` target one tag or all tags. `assert_active()` rejects paused
116
+ or failed regions. `stats()` reports allocator accounting;
117
+ `validate_allocations(allocations, graphs=...)` checks storage and pool ownership.
118
+ Each allocation record is `(tag, name, pointer, byte_size)`.
119
+
120
+ Backup restores bytes at the same virtual address. Discard restores mappings
121
+ with undefined contents: initialize before use. Graph executables and HCCL
122
+ resources stay resident. All captured persistent state must be backed up or
123
+ resident, and replay must overwrite discarded temporaries before reading them.
124
+ One device/context per process; all instances share pools. Nested scopes are
125
+ rejected. Callers must drain and block new work; these guards do not intercept
126
+ arbitrary tensor access. A failed transition can be partial. Do not reopen work
127
+ until recovery is complete; callback/rollback faults require worker restart.
128
+
129
+ This library does not implement model loaders, KV invalidation, weight commits,
130
+ multi-rank protocol 3, communication initialization, or process-group shutdown.
131
+ TokenSpeed retains those responsibilities. It does not depend on TokenSpeed or
132
+ on the CUDA `torch_memory_saver` package.
133
+
134
+ ## Tests
135
+
136
+ Host Python tests require Torch and pytest but no NPU. Use a separate CPU test
137
+ environment if needed; do not replace the Torch installation in a deployment:
138
+
139
+ ```bash
140
+ python -m pip install 'pytest>=8'
141
+ PYTHONPATH=python python -m pytest -q tests
142
+
143
+ # C++ state and error injection tests do not require Torch or CANN.
144
+ cmake -S native -B build-test -DSAVER_FAKE_ACL=ON
145
+ cmake --build build-test
146
+ ctest --test-dir build-test --output-on-failure
147
+ ```
148
+
149
+ Host tests do not establish NPU device correctness. They skip the real NPU test
150
+ unless explicitly enabled.
151
+
152
+ On an exclusively assigned NPU, run the independent graph test with
153
+ `RUN_NPU_MEMORY_SAVER_TESTS=1 python -m pytest tests/test_npu_graph.py`. It checks
154
+ three backup/discard cycles, same addresses, and replay after poisoning owned
155
+ discard storage. This is opt-in even when a native library is installed.
156
+
157
+ ## Publishing
158
+
159
+ Maintainers publish source distributions with the manually triggered
160
+ `.github/workflows/publish.yml` workflow on `main`. The `pypi` GitHub environment
161
+ uses PyPI Trusted Publishing; no long-lived API token is required. Register
162
+ owner `yanlonghu`, repository `torch_npu_memory_saver`, workflow `publish.yml`,
163
+ and environment `pypi` in PyPI before the first run. Increment the project
164
+ version for subsequent releases; PyPI does not allow replacing a release file.
165
+
166
+ The workflow validates package metadata and uploads only the source archive.
167
+ Installation compiles the native library against the user's configured CANN
168
+ toolkit. It does not imply support for untested Torch/PTA/CANN combinations.
@@ -0,0 +1,51 @@
1
+ # Validation coverage
2
+
3
+ The initial extraction retains the allocator implementation from TokenSpeed
4
+ `269b59a95f258188ce380933ef98f418bea2abe6`. The standalone library does not
5
+ implement TokenSpeed's model loading, cache invalidation or multi-rank protocol.
6
+
7
+ ## Tested stack
8
+
9
+ - Atlas A2 / 910B3, Python 3.12, Torch 2.9.0+cpu, CANN/HCCL 9.0.0.
10
+ - PTA baseline [9d40ac3](https://gitcode.com/Ascend/pytorch/commit/9d40ac3a2b3714df7e5d164bda28a42eff3282cd)
11
+ with official backports [!39221](https://gitcode.com/Ascend/pytorch/merge_requests/39221),
12
+ [!45270](https://gitcode.com/Ascend/pytorch/merge_requests/45270),
13
+ [!30625](https://gitcode.com/Ascend/pytorch/merge_requests/30625),
14
+ [!32154](https://gitcode.com/Ascend/pytorch/merge_requests/32154),
15
+ [!32564](https://gitcode.com/Ascend/pytorch/merge_requests/32564), and
16
+ [!38431](https://gitcode.com/Ascend/pytorch/merge_requests/38431).
17
+ - Installed PTA identifies itself as `2.9.0+gitee1742b`, git
18
+ `ee1742b06d2c0e59cb0b357288ec29ee190cfd2d`.
19
+
20
+ ## Results
21
+
22
+ | Check | Result |
23
+ | --- | --- |
24
+ | Independent Linux aarch64 wheel build and install | Passed. Built from this library's source distribution. |
25
+ | Standalone NPU graph | Passed three backup/discard cycles with stable addresses and replay after poisoning discarded temporary storage. TokenSpeed imports were forbidden. |
26
+ | Native error-state tests | CTest 4/4 passed on Linux using fake ACL, including failed allocation rollback and wrong-device/context free callbacks. |
27
+ | TokenSpeed TP2 partial `cpu_backup` update | Passed. Two checkpoint tensors updated; output matched a fresh complete effective checkpoint for batches 1, 3 and 4. |
28
+ | TokenSpeed TP2 complete `discard` reload | Passed. 311 checkpoint tensors loaded; both ranks committed before KV wake. Output matched a fresh new-checkpoint start. |
29
+ | Original graph and address retention | Passed for both weight policies, including two additional same-weight poison/restore cycles. No graph recapture after startup. |
30
+ | TokenSpeed protocol 3 | Both ranks negotiated v3. Holding one rank after its device operation prevented a public success reply for 300 ms; releasing it permitted completion. |
31
+ | Installed native-library provenance | All six integration ranks mapped the standalone wheel's library. Both library-path overrides were unset. |
32
+ | Worker cleanup | All successful cases exited normally and released their device leases. |
33
+
34
+ The installed Saver library was freshly compiled. Unchanged TokenSpeed host and
35
+ scheduler binaries were reused. No PTA, Rust or system-library modifications were
36
+ made for this validation. The protocol test verifies return/acknowledgement gating,
37
+ not asymmetric HCCL drain timing or native acknowledgement timestamps.
38
+
39
+ One initial protocol probe failed before device execution because two external
40
+ test helpers shared a module name. A dedicated observer module fixed the test;
41
+ no library or TokenSpeed production change was needed. The retry passed.
42
+
43
+ ## Limits
44
+
45
+ This extraction has not repeated TP4/TP8, cross-host, Mooncake transport,
46
+ extended speculative-decoding, long-cycle leak or performance campaigns.
47
+ PTA 2.13 compatibility has not been revalidated. CUDA/ROCm isolation was checked
48
+ through unchanged source boundaries and CPU behavior; GPU hardware was unavailable.
49
+
50
+ Run the repository's host and opt-in NPU tests as described in the README.
51
+ TokenSpeed integration scenarios require its separate adapter and runtime.
@@ -0,0 +1,48 @@
1
+ cmake_minimum_required(VERSION 3.18)
2
+ project(ascend_memory_saver LANGUAGES CXX)
3
+
4
+ option(SAVER_FAKE_ACL "Build only host-side state-machine tests" OFF)
5
+ set(CMAKE_CXX_STANDARD 17)
6
+ set(CMAKE_CXX_STANDARD_REQUIRED ON)
7
+ set(CMAKE_CXX_EXTENSIONS OFF)
8
+ find_package(Threads REQUIRED)
9
+
10
+ if(SAVER_FAKE_ACL)
11
+ set(saver_target saver_state_tests)
12
+ add_executable(${saver_target} src/ascend_saver.cpp tests/fake_acl.cpp tests/state_tests.cpp)
13
+ target_compile_definitions(${saver_target} PRIVATE SAVER_FAKE_ACL=1)
14
+ target_include_directories(${saver_target} PRIVATE tests)
15
+ enable_testing()
16
+ add_test(NAME saver_state_tests COMMAND ${saver_target})
17
+ # Each callback fault is terminal for its process.
18
+ add_test(NAME saver_void_free_wrong_context COMMAND ${saver_target} wrong-context)
19
+ add_test(NAME saver_void_free_wrong_device COMMAND ${saver_target} wrong-device)
20
+ add_test(NAME saver_failed_malloc_rollback COMMAND ${saver_target} malloc-rollback)
21
+ else()
22
+ set(saver_target ascend_memory_saver)
23
+ add_library(${saver_target} SHARED src/ascend_saver.cpp)
24
+ set_target_properties(${saver_target} PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN YES)
25
+ set(ASCEND_HOME_PATH "$ENV{ASCEND_HOME_PATH}" CACHE PATH "CANN toolkit root")
26
+ if(NOT ASCEND_HOME_PATH)
27
+ message(FATAL_ERROR "Set ASCEND_HOME_PATH to the target CANN installation")
28
+ endif()
29
+ find_path(ACL_INCLUDE_DIR acl/acl_rt.h PATHS
30
+ "${ASCEND_HOME_PATH}/include"
31
+ "${ASCEND_HOME_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/include"
32
+ NO_DEFAULT_PATH REQUIRED)
33
+ find_library(ACL_RT_LIBRARY acl_rt PATHS
34
+ "${ASCEND_HOME_PATH}/lib64"
35
+ "${ASCEND_HOME_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/lib64"
36
+ NO_DEFAULT_PATH REQUIRED)
37
+ target_include_directories(${saver_target} PRIVATE "${ACL_INCLUDE_DIR}")
38
+ target_link_libraries(${saver_target} PRIVATE "${ACL_RT_LIBRARY}")
39
+ target_link_options(${saver_target} PRIVATE "LINKER:--no-undefined")
40
+ include(GNUInstallDirs)
41
+ install(TARGETS ${saver_target} LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}")
42
+ endif()
43
+
44
+ target_include_directories(${saver_target} PRIVATE include)
45
+ target_link_libraries(${saver_target} PRIVATE Threads::Threads)
46
+ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
47
+ target_compile_options(${saver_target} PRIVATE -Wall -Wextra -Wpedantic)
48
+ endif()
@@ -0,0 +1,64 @@
1
+ #pragma once
2
+
3
+ #include <stddef.h>
4
+ #ifdef SAVER_FAKE_ACL
5
+ #include "fake_acl.h"
6
+ #else
7
+ #include <acl/acl_rt.h>
8
+ #endif
9
+
10
+ #if defined(_WIN32)
11
+ #define SAVER_API __declspec(dllexport)
12
+ #else
13
+ #define SAVER_API __attribute__((visibility("default")))
14
+ #endif
15
+
16
+ #ifdef __cplusplus
17
+ extern "C" {
18
+ #endif
19
+
20
+ /* ABI v2. Bind the worker's NPU device before calling any mutating API.
21
+ * The caller must stop new launches/allocations and wait for all prior users
22
+ * before pause/resume. Final free synchronizes the current device before
23
+ * unmapping; the owner must still prevent new uses of the released storage.
24
+ * Stream arguments identify the torch_npu ABI; this backend uses whole-device
25
+ * fences rather than inserting stream-order dependencies.
26
+ */
27
+ SAVER_API int saver_abi_version(void);
28
+ /* Sticky first callback fault from a void free or failed allocation rollback.
29
+ * A return value of 1 means restart the worker before reuse.
30
+ * Allocation/pause/resume fail closed. Explicit release/cleanup remain allowed.
31
+ */
32
+ SAVER_API int saver_has_callback_fault(void);
33
+ SAVER_API void *weights_malloc(size_t size, int device, aclrtStream stream);
34
+ SAVER_API void *weights_backup_malloc(size_t size, int device, aclrtStream stream);
35
+ SAVER_API void *weights_discard_malloc(size_t size, int device, aclrtStream stream);
36
+ SAVER_API void *kv_malloc(size_t size, int device, aclrtStream stream);
37
+ SAVER_API void *graph_malloc(size_t size, int device, aclrtStream stream);
38
+ SAVER_API void *graph_backup_malloc(size_t size, int device, aclrtStream stream);
39
+ SAVER_API void *graph_discard_malloc(size_t size, int device, aclrtStream stream);
40
+ /* Retires a known allocation even if context or device validation fails. */
41
+ SAVER_API void saver_free(void *ptr, size_t size, int device, aclrtStream stream);
42
+
43
+ /* tag: weights, kv_cache, acl_graph, or NULL for all.
44
+ * Return 0 on success, 1 on failure. A failed batch can be partially applied;
45
+ * keep the external launch gate closed and retry pause/resume/release.
46
+ */
47
+ SAVER_API int saver_pause(const char *tag);
48
+ SAVER_API int saver_resume(const char *tag);
49
+ /* A context or device validation failure leaves the segment phase unchanged. */
50
+ SAVER_API int saver_release(void *ptr);
51
+ /* Retry cleanup only for blocks already marked releasing by a failed alloc/free. */
52
+ SAVER_API int saver_collect_failed(void);
53
+
54
+ /* Thread-local strings, valid until the next same-thread call to the same
55
+ * function (stats), or the next mutating operation (last_error). Copy promptly.
56
+ * stats also contains the latest backend error, including callback errors
57
+ * produced on another thread. No ACL calls are made by these read-only APIs.
58
+ */
59
+ SAVER_API const char *saver_last_error(void);
60
+ SAVER_API const char *saver_stats(void);
61
+
62
+ #ifdef __cplusplus
63
+ }
64
+ #endif