tensoracle 1.5.0__tar.gz → 1.6.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {tensoracle-1.5.0 → tensoracle-1.6.1}/PKG-INFO +8 -4
  2. {tensoracle-1.5.0 → tensoracle-1.6.1}/README.md +7 -3
  3. {tensoracle-1.5.0 → tensoracle-1.6.1}/docs/index.md +4 -2
  4. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/_build.py +26 -11
  5. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/_cache.py +10 -3
  6. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/_cli.py +1 -0
  7. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/_version.py +2 -2
  8. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_build.py +37 -6
  9. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_build_memory.py +2 -0
  10. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_cache.py +15 -8
  11. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_cache_cli.py +26 -2
  12. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_cache_management.py +47 -3
  13. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_gpu_cache.py +71 -5
  14. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_weight_hash.py +2 -1
  15. {tensoracle-1.5.0 → tensoracle-1.6.1}/.gitignore +0 -0
  16. {tensoracle-1.5.0 → tensoracle-1.6.1}/LICENSE +0 -0
  17. {tensoracle-1.5.0 → tensoracle-1.6.1}/build_backend.py +0 -0
  18. {tensoracle-1.5.0 → tensoracle-1.6.1}/cpp/CMakeLists.txt +0 -0
  19. {tensoracle-1.5.0 → tensoracle-1.6.1}/cpp/cmake/TensoracleCubinPlugin.cmake +0 -0
  20. {tensoracle-1.5.0 → tensoracle-1.6.1}/cpp/cmake/TensoracleCuda.cmake +0 -0
  21. {tensoracle-1.5.0 → tensoracle-1.6.1}/cpp/include/tensoracle/cubin_plugin.hpp +0 -0
  22. {tensoracle-1.5.0 → tensoracle-1.6.1}/cpp/src/cubin_plugin.cpp +0 -0
  23. {tensoracle-1.5.0 → tensoracle-1.6.1}/cpp/tests/core.cpp +0 -0
  24. {tensoracle-1.5.0 → tensoracle-1.6.1}/docs/api/bundle.md +0 -0
  25. {tensoracle-1.5.0 → tensoracle-1.6.1}/docs/api/index.md +0 -0
  26. {tensoracle-1.5.0 → tensoracle-1.6.1}/docs/api/producer.md +0 -0
  27. {tensoracle-1.5.0 → tensoracle-1.6.1}/docs/cpp-python-repositories.md +0 -0
  28. {tensoracle-1.5.0 → tensoracle-1.6.1}/docs/optimization.md +0 -0
  29. {tensoracle-1.5.0 → tensoracle-1.6.1}/docs/precision.md +0 -0
  30. {tensoracle-1.5.0 → tensoracle-1.6.1}/docs/tensorrt-sdk.md +0 -0
  31. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/CMakeLists.txt +0 -0
  32. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/README.md +0 -0
  33. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/bundle.cpp +0 -0
  34. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/bundle.json +0 -0
  35. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm120/bias.cubin +0 -0
  36. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm120/dot.specialized.cubin +0 -0
  37. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm120/dot.tf32-control.cubin +0 -0
  38. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm120/dot.unspecialized.cubin +0 -0
  39. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm120/scale.cubin +0 -0
  40. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm86/bias.cubin +0 -0
  41. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm86/dot.specialized.cubin +0 -0
  42. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm86/dot.tf32-control.cubin +0 -0
  43. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm86/dot.unspecialized.cubin +0 -0
  44. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm86/scale.cubin +0 -0
  45. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm89/bias.cubin +0 -0
  46. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm89/dot.specialized.cubin +0 -0
  47. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm89/dot.tf32-control.cubin +0 -0
  48. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm89/dot.unspecialized.cubin +0 -0
  49. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/bundle/cubins/sm89/scale.cubin +0 -0
  50. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/example_cubin_plugin/__init__.py +0 -0
  51. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/example_cubin_plugin/weights.safetensors +0 -0
  52. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/kernels.py +0 -0
  53. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/plugin.cpp +0 -0
  54. {tensoracle-1.5.0 → tensoracle-1.6.1}/examples/cubin_plugin/pyproject.toml +0 -0
  55. {tensoracle-1.5.0 → tensoracle-1.6.1}/mkdocs.yml +0 -0
  56. {tensoracle-1.5.0 → tensoracle-1.6.1}/pyproject.toml +0 -0
  57. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/__init__.py +0 -0
  58. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/_cache_format.py +0 -0
  59. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/_comparison.py +0 -0
  60. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/_execution.py +0 -0
  61. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/_memory.py +0 -0
  62. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/_plugins.py +0 -0
  63. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/bundle.py +0 -0
  64. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/convert.py +0 -0
  65. {tensoracle-1.5.0 → tensoracle-1.6.1}/src/tensoracle/producer.py +0 -0
  66. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/build_memory_runtime.py +0 -0
  67. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/cubin_io/check.cpp +0 -0
  68. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/cubin_io/kernels.cu +0 -0
  69. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/cubin_io/plugin.cpp +0 -0
  70. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/cubins/ptxas-12.9.86/sm120.cubin +0 -0
  71. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/cubins/ptxas-12.9.86/sm86.cubin +0 -0
  72. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/cubins/ptxas-12.9.86/sm89.cubin +0 -0
  73. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/example_runtime.py +0 -0
  74. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/native/CMakeLists.txt +0 -0
  75. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/native/export.h +0 -0
  76. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/native_build.py +0 -0
  77. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/plugin_io_runtime.py +0 -0
  78. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/precision_runtime.py +0 -0
  79. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_bundle.py +0 -0
  80. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_cmake_cuda.py +0 -0
  81. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_cmake_plugin.py +0 -0
  82. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_comparison.py +0 -0
  83. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_convert.py +0 -0
  84. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_cpp_core.py +0 -0
  85. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_cuda_graphs.py +0 -0
  86. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_distribution.py +0 -0
  87. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_engine_io.py +0 -0
  88. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_example_plugin.py +0 -0
  89. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_execution.py +0 -0
  90. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_plugin_io.py +0 -0
  91. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_plugins.py +0 -0
  92. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_precision.py +0 -0
  93. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_producer.py +0 -0
  94. {tensoracle-1.5.0 → tensoracle-1.6.1}/tests/test_shape_tensor_outputs.py +0 -0
  95. {tensoracle-1.5.0 → tensoracle-1.6.1}/uv.lock +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: tensoracle
3
- Version: 1.5.0
3
+ Version: 1.6.1
4
4
  Summary: Build, save, reuse, and execute TensorRT engines from safetensors weights
5
5
  Project-URL: documentation, https://sync-dev-org.github.io/tensoracle/
6
6
  Project-URL: repository, https://github.com/sync-dev-org/tensoracle
@@ -110,7 +110,7 @@ def graph(network: trt.INetworkDefinition) -> None:
110
110
 
111
111
  declaration = EngineBuild(
112
112
  inputs={"features": InputShape.ranged((1, 1, 2), (1, 64, 2), (1, 448, 2))},
113
- settings=BuilderSettings(workspace_limit=1 << 30, optimization_level=3),
113
+ settings=BuilderSettings(workspace_limit=None, optimization_level=3),
114
114
  )
115
115
  plan: bytes = build_serialized_engine(graph, declaration)
116
116
  ```
@@ -119,6 +119,8 @@ Use `InputShape.fixed((1, 64, 2))` for one input size; the graph may declare tha
119
119
 
120
120
  The engine's precision is determined by the tensor types that the graph function places in the network. For an fp16 graph, use `trt.float16` tensors and `np.float16` weights where needed; the graph can also contain fp32 tensors. `EngineBuild` declares input shapes and builder settings. TF32 is always disabled; `BuilderSettings(tf32=True)` raises `ValueError`. The function returns serialized plan bytes. Invalid declarations raise `ValueError`, and a failed TensorRT build raises `RuntimeError`.
121
121
 
122
+ `BuilderSettings.workspace_limit` defaults to `None`, which leaves the workspace limit at TensorRT's default without calling `set_memory_pool_limit`. To impose a limit, pass a positive integer in bytes, such as `workspace_limit=4 << 30` for 4 GiB. Zero, negative values, booleans, and noninteger values raise `ValueError`.
123
+
122
124
  The same API builds bf16, int8, and fp8 graphs. Cast tensors to `trt.bfloat16` for bf16; for int8/fp8, place `add_quantize` / `add_dequantize` layers and load their scales from safetensors. The application determines scales with an external quantization library and chooses output quality limits with `compare_outputs`. fp8 targets sm89 and sm120; bf16 and int8 also target sm86. Follow [Build bf16, int8, and fp8 engines](https://github.com/sync-dev-org/tensoracle/blob/main/docs/precision.md) for a complete example, NumPy dtype constraints, and comparison against an fp32 engine. Use different engine names or graph versions for different precisions.
123
125
 
124
126
  ## Save and reopen an engine
@@ -144,7 +146,9 @@ handle = open_engine(
144
146
  engine = handle.engine # Keep handle.runtime alive while using the engine.
145
147
  ```
146
148
 
147
- Saved files live in the `tensoracle` directory under `XDG_CACHE_HOME`, or under the home directory's `.cache` when that variable is unset. The same rule applies on every OS; on Windows the home directory is `%USERPROFILE%`, so the default is `%USERPROFILE%\.cache\tensoracle`. The cache identity includes the contents of all declared safetensors weight files and optional `plugins` files, the engine name, graph version, input shapes, builder settings, the GPU's compatibility properties, OS, CPU architecture, and TensorRT version. The GPU is the device of the current CUDA context, or device 0 when no context is current. Its properties are the product name, compute capability, memory bus width, L2 cache size, shared memory limits, texture alignment, multiprocessor count, whether the GPU is integrated, ECC mode, and total memory; another device with the same properties reuses the saved engine. Raise `graph_version` when graph code changes; without a version change, the previous engine is reused. Use a different `name` or `graph_version` for engines with different tensor types, such as fp32 and fp16. Include in `graph_version` any other caller value that changes the engine contents but is not part of the cache identity, such as a plugin manifest or a build recipe. Pass `force_rebuild=True` to replace a cached engine even when it is intact. tensoracle closes a saved engine file after reading it, so another process using an opened engine does not block replacement. On Windows, a process that keeps the saved file itself open prevents replacement or removal with `PermissionError`; after that process closes the file, a file left empty by the failed rebuild is rejected as `CorruptEngineError` and can be rebuilt with `force_rebuild=True`.
149
+ Saved files live in the `tensoracle` directory under `XDG_CACHE_HOME`, or under the home directory's `.cache` when that variable is unset. The same rule applies on every OS; on Windows the home directory is `%USERPROFILE%`, so the default is `%USERPROFILE%\.cache\tensoracle`. The cache identity includes the contents of all declared safetensors weight files and optional `plugins` files, the engine name, graph version, input shapes, the builder's `optimization_level` and `tf32` settings, the GPU's compatibility properties, OS, CPU architecture, and TensorRT version. The GPU is the device of the current CUDA context, or device 0 when no context is current. Its properties are the product name, compute capability, memory bus width, L2 cache size, shared memory limits, texture alignment, multiprocessor count, whether the GPU is integrated, ECC mode, and total memory; another device with the same properties reuses the saved engine. Raise `graph_version` when graph code changes; without a version change, the previous engine is reused. Use a different `name` or `graph_version` for engines with different tensor types, such as fp32 and fp16. Include in `graph_version` any other caller value that changes the engine contents but is not part of the cache identity, such as a plugin manifest or a build recipe. Pass `force_rebuild=True` to replace a cached engine even when it is intact. tensoracle closes a saved engine file after reading it, so another process using an opened engine does not block replacement. On Windows, a process that keeps the saved file itself open prevents replacement or removal with `PermissionError`; after that process closes the file, a file left empty by the failed rebuild is rejected as `CorruptEngineError` and can be rebuilt with `force_rebuild=True`.
150
+
151
+ `workspace_limit` is excluded from the cache identity. Changing it reuses a matching saved engine; pass `force_rebuild=True` to `open_engine` to build again with the new limit. The saved header records the workspace limit used for that build, including `None`, as a reference outside the conditions used for matching. Cache reads leave this reference unchanged. `tensoracle cache list` shows the recorded limit.
148
152
 
149
153
  `plugins` names native TensorRT plugin libraries. Both `open_engine` and `build_serialized_engine` accept it. Each function checks that every library exports `getCreators`, loads it through TensorRT's registry, and retains it for the process before building the graph or deserializing an engine. Repeating the same library does not register it twice. `open_engine` also hashes the loaded library bytes into the engine identity; a changed library selects a different saved engine in a new process. A missing, unreadable, or incompatible library, or a library changed after loading in the same process, raises `PluginLibraryError` before a cached plan is opened. Keep the library and its cubin bundle together in the caller's wheel.
150
154
 
@@ -170,7 +174,7 @@ tensoracle cache prune --max-size 10GiB
170
174
 
171
175
  The commands use the same `XDG_CACHE_HOME` / home `.cache` directory as `open_engine`. They read only filesystem metadata, engine headers, and saved weight digest records. They do not read or checksum the plan body, load the original weights, deserialize engines, import TensorRT, or query a GPU. Listing recognizes the header format; it does not establish that an engine can be loaded.
172
176
 
173
- `cache list` prints one JSON object with `directory`, `entries`, and `total_size_bytes`. Each entry contains the file name, size in bytes, recorded time in UTC (`recorded_at`) and exact nanoseconds (`recorded_at_ns`), and the complete header conditions. This distinguishes engines with the same name and graph version but different weights or plugins. `weight_paths` associates each weight digest with paths from valid, checksum-verified `weight-*.json` records whose SHA-256 matches that digest; an empty `paths` array means unknown. These paths are references and do not establish which application uses an engine or whether the weight file still exists. Empty files have status `building or leftover`; unreadable or invalid headers have status `unrecognized` and a reason.
177
+ `cache list` prints one JSON object with `directory`, `entries`, and `total_size_bytes`. Each entry contains the file name, size in bytes, recorded time in UTC (`recorded_at`) and exact nanoseconds (`recorded_at_ns`), and the complete header conditions. This distinguishes engines with the same name and graph version but different weights or plugins. Each recognized entry also contains `workspace_limit`: a positive integer in bytes for an explicit build limit, JSON `null` when the build left the limit at TensorRT's default, or `"unknown"` when the header has no reference value. `weight_paths` associates each weight digest with paths from valid, checksum-verified `weight-*.json` records whose SHA-256 matches that digest; an empty `paths` array means unknown. These paths are references and do not establish which application uses an engine or whether the weight file still exists. Empty files have status `building or leftover`; unreadable or invalid headers have status `unrecognized` and a reason.
174
178
 
175
179
  An engine's `.engine` modification time records its initial save or the last successful `open_engine` cache read. Failed loads and these management commands leave that time unchanged. A timestamp write failure, including a read-only cache, does not prevent a successful load, and a disappeared file is never recreated to record its time. The record describes the time of `open_engine`, not whether inference continues afterward. Files saved before timestamp recording was introduced, and files whose timestamp could not be written, may not reflect the actual last successful read.
176
180
 
@@ -77,7 +77,7 @@ def graph(network: trt.INetworkDefinition) -> None:
77
77
 
78
78
  declaration = EngineBuild(
79
79
  inputs={"features": InputShape.ranged((1, 1, 2), (1, 64, 2), (1, 448, 2))},
80
- settings=BuilderSettings(workspace_limit=1 << 30, optimization_level=3),
80
+ settings=BuilderSettings(workspace_limit=None, optimization_level=3),
81
81
  )
82
82
  plan: bytes = build_serialized_engine(graph, declaration)
83
83
  ```
@@ -86,6 +86,8 @@ Use `InputShape.fixed((1, 64, 2))` for one input size; the graph may declare tha
86
86
 
87
87
  The engine's precision is determined by the tensor types that the graph function places in the network. For an fp16 graph, use `trt.float16` tensors and `np.float16` weights where needed; the graph can also contain fp32 tensors. `EngineBuild` declares input shapes and builder settings. TF32 is always disabled; `BuilderSettings(tf32=True)` raises `ValueError`. The function returns serialized plan bytes. Invalid declarations raise `ValueError`, and a failed TensorRT build raises `RuntimeError`.
88
88
 
89
+ `BuilderSettings.workspace_limit` defaults to `None`, which leaves the workspace limit at TensorRT's default without calling `set_memory_pool_limit`. To impose a limit, pass a positive integer in bytes, such as `workspace_limit=4 << 30` for 4 GiB. Zero, negative values, booleans, and noninteger values raise `ValueError`.
90
+
89
91
  The same API builds bf16, int8, and fp8 graphs. Cast tensors to `trt.bfloat16` for bf16; for int8/fp8, place `add_quantize` / `add_dequantize` layers and load their scales from safetensors. The application determines scales with an external quantization library and chooses output quality limits with `compare_outputs`. fp8 targets sm89 and sm120; bf16 and int8 also target sm86. Follow [Build bf16, int8, and fp8 engines](https://github.com/sync-dev-org/tensoracle/blob/main/docs/precision.md) for a complete example, NumPy dtype constraints, and comparison against an fp32 engine. Use different engine names or graph versions for different precisions.
90
92
 
91
93
  ## Save and reopen an engine
@@ -111,7 +113,9 @@ handle = open_engine(
111
113
  engine = handle.engine # Keep handle.runtime alive while using the engine.
112
114
  ```
113
115
 
114
- Saved files live in the `tensoracle` directory under `XDG_CACHE_HOME`, or under the home directory's `.cache` when that variable is unset. The same rule applies on every OS; on Windows the home directory is `%USERPROFILE%`, so the default is `%USERPROFILE%\.cache\tensoracle`. The cache identity includes the contents of all declared safetensors weight files and optional `plugins` files, the engine name, graph version, input shapes, builder settings, the GPU's compatibility properties, OS, CPU architecture, and TensorRT version. The GPU is the device of the current CUDA context, or device 0 when no context is current. Its properties are the product name, compute capability, memory bus width, L2 cache size, shared memory limits, texture alignment, multiprocessor count, whether the GPU is integrated, ECC mode, and total memory; another device with the same properties reuses the saved engine. Raise `graph_version` when graph code changes; without a version change, the previous engine is reused. Use a different `name` or `graph_version` for engines with different tensor types, such as fp32 and fp16. Include in `graph_version` any other caller value that changes the engine contents but is not part of the cache identity, such as a plugin manifest or a build recipe. Pass `force_rebuild=True` to replace a cached engine even when it is intact. tensoracle closes a saved engine file after reading it, so another process using an opened engine does not block replacement. On Windows, a process that keeps the saved file itself open prevents replacement or removal with `PermissionError`; after that process closes the file, a file left empty by the failed rebuild is rejected as `CorruptEngineError` and can be rebuilt with `force_rebuild=True`.
116
+ Saved files live in the `tensoracle` directory under `XDG_CACHE_HOME`, or under the home directory's `.cache` when that variable is unset. The same rule applies on every OS; on Windows the home directory is `%USERPROFILE%`, so the default is `%USERPROFILE%\.cache\tensoracle`. The cache identity includes the contents of all declared safetensors weight files and optional `plugins` files, the engine name, graph version, input shapes, the builder's `optimization_level` and `tf32` settings, the GPU's compatibility properties, OS, CPU architecture, and TensorRT version. The GPU is the device of the current CUDA context, or device 0 when no context is current. Its properties are the product name, compute capability, memory bus width, L2 cache size, shared memory limits, texture alignment, multiprocessor count, whether the GPU is integrated, ECC mode, and total memory; another device with the same properties reuses the saved engine. Raise `graph_version` when graph code changes; without a version change, the previous engine is reused. Use a different `name` or `graph_version` for engines with different tensor types, such as fp32 and fp16. Include in `graph_version` any other caller value that changes the engine contents but is not part of the cache identity, such as a plugin manifest or a build recipe. Pass `force_rebuild=True` to replace a cached engine even when it is intact. tensoracle closes a saved engine file after reading it, so another process using an opened engine does not block replacement. On Windows, a process that keeps the saved file itself open prevents replacement or removal with `PermissionError`; after that process closes the file, a file left empty by the failed rebuild is rejected as `CorruptEngineError` and can be rebuilt with `force_rebuild=True`.
117
+
118
+ `workspace_limit` is excluded from the cache identity. Changing it reuses a matching saved engine; pass `force_rebuild=True` to `open_engine` to build again with the new limit. The saved header records the workspace limit used for that build, including `None`, as a reference outside the conditions used for matching. Cache reads leave this reference unchanged. `tensoracle cache list` shows the recorded limit.
115
119
 
116
120
  `plugins` names native TensorRT plugin libraries. Both `open_engine` and `build_serialized_engine` accept it. Each function checks that every library exports `getCreators`, loads it through TensorRT's registry, and retains it for the process before building the graph or deserializing an engine. Repeating the same library does not register it twice. `open_engine` also hashes the loaded library bytes into the engine identity; a changed library selects a different saved engine in a new process. A missing, unreadable, or incompatible library, or a library changed after loading in the same process, raises `PluginLibraryError` before a cached plan is opened. Keep the library and its cubin bundle together in the caller's wheel.
117
121
 
@@ -137,7 +141,7 @@ tensoracle cache prune --max-size 10GiB
137
141
 
138
142
  The commands use the same `XDG_CACHE_HOME` / home `.cache` directory as `open_engine`. They read only filesystem metadata, engine headers, and saved weight digest records. They do not read or checksum the plan body, load the original weights, deserialize engines, import TensorRT, or query a GPU. Listing recognizes the header format; it does not establish that an engine can be loaded.
139
143
 
140
- `cache list` prints one JSON object with `directory`, `entries`, and `total_size_bytes`. Each entry contains the file name, size in bytes, recorded time in UTC (`recorded_at`) and exact nanoseconds (`recorded_at_ns`), and the complete header conditions. This distinguishes engines with the same name and graph version but different weights or plugins. `weight_paths` associates each weight digest with paths from valid, checksum-verified `weight-*.json` records whose SHA-256 matches that digest; an empty `paths` array means unknown. These paths are references and do not establish which application uses an engine or whether the weight file still exists. Empty files have status `building or leftover`; unreadable or invalid headers have status `unrecognized` and a reason.
144
+ `cache list` prints one JSON object with `directory`, `entries`, and `total_size_bytes`. Each entry contains the file name, size in bytes, recorded time in UTC (`recorded_at`) and exact nanoseconds (`recorded_at_ns`), and the complete header conditions. This distinguishes engines with the same name and graph version but different weights or plugins. Each recognized entry also contains `workspace_limit`: a positive integer in bytes for an explicit build limit, JSON `null` when the build left the limit at TensorRT's default, or `"unknown"` when the header has no reference value. `weight_paths` associates each weight digest with paths from valid, checksum-verified `weight-*.json` records whose SHA-256 matches that digest; an empty `paths` array means unknown. These paths are references and do not establish which application uses an engine or whether the weight file still exists. Empty files have status `building or leftover`; unreadable or invalid headers have status `unrecognized` and a reason.
141
145
 
142
146
  An engine's `.engine` modification time records its initial save or the last successful `open_engine` cache read. Failed loads and these management commands leave that time unchanged. A timestamp write failure, including a read-only cache, does not prevent a successful load, and a disappeared file is never recreated to record its time. The record describes the time of `open_engine`, not whether inference continues afterward. Files saved before timestamp recording was introduced, and files whose timestamp could not be written, may not reflect the actual last successful read.
143
147
 
@@ -74,6 +74,8 @@ handle = open_engine(
74
74
 
75
75
  `my_model` represents the application's model package. On a valid cache hit, `open_engine` does not call `graph`; on a miss, it builds and saves a new engine. Raise `graph_version` when the graph code changes or when any other caller value that changes the engine contents but is not part of the cache identity changes, such as a plugin manifest or a build recipe. `open_engine` itself does not execute the engine or manage GPU buffers. Create an execution unit from its returned handle to enqueue inference. Saved engines live under `XDG_CACHE_HOME/tensoracle`, or `~/.cache/tensoracle` when `XDG_CACHE_HOME` is unset. On Windows, `~` is `%USERPROFILE%`. See the [complete cache example](https://github.com/sync-dev-org/tensoracle#save-and-reopen-an-engine) for the declaration and expected input and output types.
76
76
 
77
+ `workspace_limit` is excluded from the cache identity; `optimization_level` and `tf32` remain in the conditions. Changing only the workspace limit reuses a matching saved engine. Pass `force_rebuild=True` to `open_engine` to build again with the new limit. The saved header records the workspace limit used for that build, including `None`, as a reference outside the conditions used for matching. Cache reads leave this reference unchanged. `tensoracle cache list` shows the recorded limit.
78
+
77
79
  ### Manage saved engines
78
80
 
79
81
  Inspect saved engines and preview or perform cleanup with the installed command:
@@ -88,7 +90,7 @@ tensoracle cache prune --max-size 10GiB
88
90
 
89
91
  The commands use the same `XDG_CACHE_HOME` / home `.cache` directory as `open_engine`. They read only filesystem metadata, engine headers, and saved weight digest records. They do not read or checksum the plan body, load the original weights, deserialize engines, import TensorRT, or query a GPU. Listing recognizes the header format; it does not establish that an engine can be loaded.
90
92
 
91
- `cache list` prints one JSON object with `directory`, `entries`, and `total_size_bytes`. Each entry contains the file name, size in bytes, recorded time in UTC (`recorded_at`) and exact nanoseconds (`recorded_at_ns`), and the complete header conditions. This distinguishes engines with the same name and graph version but different weights or plugins. `weight_paths` associates each weight digest with paths from valid, checksum-verified `weight-*.json` records whose SHA-256 matches that digest; an empty `paths` array means unknown. These paths are references and do not establish which application uses an engine or whether the weight file still exists. Empty files have status `building or leftover`; unreadable or invalid headers have status `unrecognized` and a reason.
93
+ `cache list` prints one JSON object with `directory`, `entries`, and `total_size_bytes`. Each entry contains the file name, size in bytes, recorded time in UTC (`recorded_at`) and exact nanoseconds (`recorded_at_ns`), and the complete header conditions. This distinguishes engines with the same name and graph version but different weights or plugins. Each recognized entry also contains `workspace_limit`: a positive integer in bytes for an explicit build limit, JSON `null` when the build left the limit at TensorRT's default, or `"unknown"` when the header has no reference value. `weight_paths` associates each weight digest with paths from valid, checksum-verified `weight-*.json` records whose SHA-256 matches that digest; an empty `paths` array means unknown. These paths are references and do not establish which application uses an engine or whether the weight file still exists. Empty files have status `building or leftover`; unreadable or invalid headers have status `unrecognized` and a reason.
92
94
 
93
95
  An engine's `.engine` modification time records its initial save or the last successful `open_engine` cache read. Failed loads and these management commands leave that time unchanged. A timestamp write failure, including a read-only cache, does not prevent a successful load, and a disappeared file is never recreated to record its time. The record describes the time of `open_engine`, not whether inference continues afterward. Files saved before timestamp recording was introduced, and files whose timestamp could not be written, may not reflect the actual last successful read.
94
96
 
@@ -252,7 +254,7 @@ Engines with loops or conditionals, or layers or plugins that synchronize with t
252
254
 
253
255
  ### Build a standard graph
254
256
 
255
- Define `graph(network)` with TensorRT's Python network API and load model weights from safetensors. Describe each graph input with `InputShape.fixed` or `InputShape.ranged`, and pass an `EngineBuild` declaration to `build_serialized_engine`. `EngineBuild` declares input shapes and builder settings; the graph function determines precision through the tensor types it places in the network. The supported precisions are fp32 (with TF32 disabled), fp16, bf16, int8, and fp8. The [bf16, int8, and fp8 guide](precision.md) shows tensor casts, explicit Q/DQ layers, and scales supplied through safetensors; fp8 targets sm89 and sm120. `open_engine` uses the same declaration plus a name, graph version, weight files, and `EngineIO` to save and reuse a checked engine. Use a different `name` or `graph_version` for engines with different tensor types, such as fp32 and fp16. The [graph example](https://github.com/sync-dev-org/tensoracle#define-and-build-a-standard-graph) shows the exact calls.
257
+ Define `graph(network)` with TensorRT's Python network API and load model weights from safetensors. Describe each graph input with `InputShape.fixed` or `InputShape.ranged`, and pass an `EngineBuild` declaration to `build_serialized_engine`. `EngineBuild` declares input shapes and builder settings; the graph function determines precision through the tensor types it places in the network. `BuilderSettings.workspace_limit` defaults to `None`, leaving the workspace limit at TensorRT's default. To impose a limit, pass a positive integer in bytes, such as `workspace_limit=4 << 30` for 4 GiB. Zero, negative values, booleans, and noninteger values raise `ValueError`. The supported precisions are fp32 (with TF32 disabled), fp16, bf16, int8, and fp8. The [bf16, int8, and fp8 guide](precision.md) shows tensor casts, explicit Q/DQ layers, and scales supplied through safetensors; fp8 targets sm89 and sm120. `open_engine` uses the same declaration plus a name, graph version, weight files, and `EngineIO` to save and reuse a checked engine. Use a different `name` or `graph_version` for engines with different tensor types, such as fp32 and fp16. The [graph example](https://github.com/sync-dev-org/tensoracle#define-and-build-a-standard-graph) shows the exact calls.
256
258
 
257
259
  The application runs the engine and passes a CPU NumPy candidate output and a corresponding reference output to `compare_outputs`. The application chooses the acceptance limits. See [Compare output arrays](https://github.com/sync-dev-org/tensoracle#compare-output-arrays).
258
260
 
@@ -69,10 +69,16 @@ class InputShape:
69
69
  class BuilderSettings:
70
70
  """TensorRT builder options that affect the serialized engine."""
71
71
 
72
- workspace_limit: int = 1 << 30
72
+ workspace_limit: int | None = None
73
73
  optimization_level: int = 3
74
74
  tf32: bool = False
75
75
 
76
+ def __post_init__(self) -> None:
77
+ if self.workspace_limit is not None and (
78
+ type(self.workspace_limit) is not int or self.workspace_limit <= 0
79
+ ):
80
+ raise ValueError("workspace_limit must be a positive integer or None")
81
+
76
82
 
77
83
  @dataclass(frozen=True)
78
84
  class EngineBuild:
@@ -103,14 +109,12 @@ def _build_serialized_engine(
103
109
  ) -> tuple[bytes, tuple[str, ...]]:
104
110
  if declaration.settings.tf32:
105
111
  raise ValueError("TF32 cannot be enabled")
106
- if declaration.settings.workspace_limit <= 0:
107
- raise ValueError("workspace_limit must be positive")
108
112
  if not 0 <= declaration.settings.optimization_level <= 5:
109
113
  raise ValueError("optimization_level must be between 0 and 5")
110
114
 
111
115
  register_libraries(plugins)
112
116
 
113
- builder = network = config = profile = serialized = tensor = None
117
+ builder = network = config = profile = serialized = tensor = timing = None
114
118
  inputs = {}
115
119
  try:
116
120
  builder = trt.Builder(logger or trt.Logger(trt.Logger.ERROR))
@@ -125,9 +129,10 @@ def _build_serialized_engine(
125
129
  _warn_shape_tensor_outputs(output_names, stacklevel=3)
126
130
  config = builder.create_builder_config()
127
131
  config.clear_flag(trt.BuilderFlag.TF32)
128
- config.set_memory_pool_limit(
129
- trt.MemoryPoolType.WORKSPACE, declaration.settings.workspace_limit
130
- )
132
+ if declaration.settings.workspace_limit is not None:
133
+ config.set_memory_pool_limit(
134
+ trt.MemoryPoolType.WORKSPACE, declaration.settings.workspace_limit
135
+ )
131
136
  config.builder_optimization_level = declaration.settings.optimization_level
132
137
 
133
138
  inputs = {
@@ -156,12 +161,22 @@ def _build_serialized_engine(
156
161
  if profile is not None:
157
162
  config.add_optimization_profile(profile)
158
163
 
159
- serialized = builder.build_serialized_network(network, config)
160
- if serialized is None:
161
- raise RuntimeError("TensorRT failed to build the serialized engine")
164
+ timing = config.create_timing_cache(b"")
165
+ if timing is None:
166
+ raise RuntimeError("TensorRT failed to create a timing cache")
167
+ if not config.set_timing_cache(timing, ignore_mismatch=False):
168
+ raise RuntimeError("TensorRT rejected the timing cache")
169
+ # A cold build can serialize large zero-filled buffers. Rebuild with the
170
+ # populated in-memory cache, releasing the first plan before the second.
171
+ for _ in range(2):
172
+ serialized = None
173
+ serialized = builder.build_serialized_network(network, config)
174
+ if serialized is None:
175
+ raise RuntimeError("TensorRT failed to build the serialized engine")
162
176
  return bytes(serialized), output_names
163
177
  finally:
164
178
  # Tensor references also keep the network alive, including in exception frames.
165
179
  inputs.clear()
166
- del tensor, profile, serialized, config, network, builder
180
+ # TensorRT requires the timing cache to outlive the builder using it.
181
+ del tensor, profile, serialized, config, network, builder, timing
167
182
  release_host_memory()
@@ -7,7 +7,7 @@ import os
7
7
  import platform
8
8
  import tempfile
9
9
  from collections.abc import Callable, Mapping, Sequence
10
- from dataclasses import asdict, dataclass
10
+ from dataclasses import dataclass
11
11
  from dataclasses import field as dataclass_field
12
12
  from pathlib import Path
13
13
  from typing import TYPE_CHECKING
@@ -125,7 +125,11 @@ def open_engine(
125
125
  handle = _open_plan(
126
126
  plan, path, declaration, io, actual_logger, shape_tensor_outputs=shape_tensor_outputs
127
127
  )
128
- header = {"conditions": conditions, "plan_sha256": hashlib.sha256(plan).hexdigest()}
128
+ header = {
129
+ "conditions": conditions,
130
+ "plan_sha256": hashlib.sha256(plan).hexdigest(),
131
+ "workspace_limit": declaration.settings.workspace_limit,
132
+ }
129
133
  if shape_tensor_outputs:
130
134
  header["shape_tensor_outputs"] = list(shape_tensor_outputs)
131
135
  header_bytes = _json_bytes(header)
@@ -286,7 +290,10 @@ def _conditions(
286
290
  }
287
291
  for name, shape in declaration.inputs.items()
288
292
  },
289
- "builder_settings": asdict(declaration.settings),
293
+ "builder_settings": {
294
+ "optimization_level": declaration.settings.optimization_level,
295
+ "tf32": declaration.settings.tf32,
296
+ },
290
297
  "gpu": _gpu_identity(),
291
298
  "os": platform.system(),
292
299
  "architecture": platform.machine(),
@@ -58,6 +58,7 @@ def _inspect(path: Path) -> _Entry:
58
58
  raise OSError("file changed during inspection")
59
59
  details["status"] = "recognized"
60
60
  details["conditions"] = header["conditions"]
61
+ details["workspace_limit"] = header.get("workspace_limit", "unknown")
61
62
  except (OSError, CorruptEngineError) as error:
62
63
  details["reason"] = str(error)
63
64
  return _Entry(path, observed, details)
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '1.5.0'
22
- __version_tuple__ = version_tuple = (1, 5, 0)
21
+ __version__ = version = '1.6.1'
22
+ __version_tuple__ = version_tuple = (1, 6, 1)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -187,8 +187,23 @@ def test_tf32_request_is_rejected_before_graph_is_built():
187
187
 
188
188
 
189
189
  @pytest.mark.req("SP-TENSORACLE-013")
190
- def test_tf32_is_disabled_and_other_builder_settings_are_applied(monkeypatch):
191
- """The builder clears TF32 while applying the declared workspace and optimization level."""
190
+ @pytest.mark.req("SP-TENSORACLE-052")
191
+ @pytest.mark.parametrize(
192
+ "settings, workspace_calls",
193
+ [
194
+ (BuilderSettings(optimization_level=2), []),
195
+ (BuilderSettings(workspace_limit=None, optimization_level=2), []),
196
+ (
197
+ BuilderSettings(workspace_limit=4096, optimization_level=2),
198
+ [(trt.MemoryPoolType.WORKSPACE, 4096)],
199
+ ),
200
+ ],
201
+ ids=["default-workspace", "explicit-none", "positive-workspace"],
202
+ )
203
+ def test_tf32_is_disabled_and_other_builder_settings_are_applied(
204
+ monkeypatch, settings, workspace_calls
205
+ ):
206
+ """The builder clears TF32 and sets a workspace limit only for an explicit positive integer."""
192
207
  import tensoracle._build as module
193
208
 
194
209
  class Config:
@@ -196,13 +211,19 @@ def test_tf32_is_disabled_and_other_builder_settings_are_applied(monkeypatch):
196
211
 
197
212
  def __init__(self):
198
213
  self.flags = {trt.BuilderFlag.TF32}
199
- self.workspace = None
214
+ self.workspace_calls = []
200
215
 
201
216
  def clear_flag(self, flag):
202
217
  self.flags.discard(flag)
203
218
 
204
219
  def set_memory_pool_limit(self, pool, limit):
205
- self.workspace = (pool, limit)
220
+ self.workspace_calls.append((pool, limit))
221
+
222
+ def create_timing_cache(self, data):
223
+ return object()
224
+
225
+ def set_timing_cache(self, timing, ignore_mismatch):
226
+ return True
206
227
 
207
228
  class Network:
208
229
  num_inputs = 1
@@ -229,10 +250,20 @@ def test_tf32_is_disabled_and_other_builder_settings_are_applied(monkeypatch):
229
250
  monkeypatch.setattr(module.trt, "Builder", Builder)
230
251
  declaration = EngineBuild(
231
252
  inputs={"features": InputShape.fixed((1, 3, 2))},
232
- settings=BuilderSettings(workspace_limit=4096, optimization_level=2),
253
+ settings=settings,
233
254
  )
234
255
  assert build_serialized_engine(lambda network: None, declaration) == b"plan"
235
256
  config = observed[0].config
236
257
  assert trt.BuilderFlag.TF32 not in config.flags
237
- assert config.workspace == (trt.MemoryPoolType.WORKSPACE, 4096)
258
+ assert config.workspace_calls == workspace_calls
238
259
  assert config.builder_optimization_level == 2
260
+
261
+
262
+ @pytest.mark.req("SP-TENSORACLE-052")
263
+ @pytest.mark.parametrize("workspace_limit", [0, -1, True, False, 1.5, "4096"])
264
+ def test_invalid_workspace_limit_is_rejected_before_building(monkeypatch, workspace_limit):
265
+ """Zero, negatives, booleans, and nonintegers fail before creating a TensorRT builder."""
266
+ monkeypatch.setattr(trt, "Builder", lambda logger: pytest.fail("builder was created"))
267
+ with pytest.raises(ValueError, match="workspace_limit"):
268
+ declaration = EngineBuild({}, BuilderSettings(workspace_limit=workspace_limit))
269
+ build_serialized_engine(lambda network: pytest.fail("graph was built"), declaration)
@@ -68,6 +68,8 @@ def test_unavailable_host_memory_release_preserves_the_build_result(monkeypatch,
68
68
  create_builder_config=lambda: SimpleNamespace(
69
69
  clear_flag=lambda flag: None,
70
70
  set_memory_pool_limit=lambda pool, limit: None,
71
+ create_timing_cache=lambda data: object(),
72
+ set_timing_cache=lambda timing, ignore_mismatch: True,
71
73
  ),
72
74
  build_serialized_network=serialize,
73
75
  )
@@ -101,6 +101,8 @@ def test_save_large_built_plan_keeps_other_threads_running(tmp_path, monkeypatch
101
101
  create_builder_config=lambda: SimpleNamespace(
102
102
  clear_flag=lambda flag: None,
103
103
  set_memory_pool_limit=lambda pool, limit: None,
104
+ create_timing_cache=lambda data: object(),
105
+ set_timing_cache=lambda timing, ignore_mismatch: True,
104
106
  ),
105
107
  build_serialized_network=lambda network, config: plan,
106
108
  )
@@ -347,6 +349,7 @@ def test_corrupt_plan_and_changed_header_fail_before_deserialization(tmp_path, m
347
349
  @pytest.mark.gpu
348
350
  @pytest.mark.req("SP-TENSORACLE-002")
349
351
  @pytest.mark.req("SP-TENSORACLE-011")
352
+ @pytest.mark.req("SP-TENSORACLE-052")
350
353
  def test_force_rebuild_uses_changed_weight_contents_and_preserves_old_engine(tmp_path, monkeypatch):
351
354
  """A forced rebuild saves under the new digest without replacing the old engine."""
352
355
  monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache"))
@@ -378,7 +381,7 @@ def test_force_rebuild_uses_changed_weight_contents_and_preserves_old_engine(tmp
378
381
  offset = len(b"TENSORACLE\x01")
379
382
  old_header_size = int.from_bytes(old_engine[offset : offset + 8], "big")
380
383
  old_header = json.loads(old_engine[offset + 8 : offset + 8 + old_header_size])
381
- assert old_header.keys() == {"conditions", "plan_sha256"}
384
+ assert old_header.keys() == {"conditions", "plan_sha256", "workspace_limit"}
382
385
 
383
386
  before = weights.stat()
384
387
  changed = bytearray(weights.read_bytes())
@@ -497,7 +500,7 @@ def test_each_declared_condition_selects_a_distinct_cache_entry(tmp_path, monkey
497
500
 
498
501
  monkeypatch.setattr(trt, "get_plugin_registry", PluginRegistry)
499
502
 
500
- state = {"builds": 0, "graph_calls": 0, "shape": (1, 3, 2), "range": None}
503
+ state = {"graph_calls": 0, "shape": (1, 3, 2), "range": None}
501
504
 
502
505
  class FakeEngine:
503
506
  num_io_tensors = 2
@@ -545,6 +548,12 @@ def test_each_declared_condition_selects_a_distinct_cache_entry(tmp_path, monkey
545
548
  def add_optimization_profile(self, profile):
546
549
  pass
547
550
 
551
+ def create_timing_cache(self, data):
552
+ return object()
553
+
554
+ def set_timing_cache(self, timing, ignore_mismatch):
555
+ return True
556
+
548
557
  class FakeBuilder:
549
558
  def __init__(self, logger):
550
559
  pass
@@ -559,7 +568,6 @@ def test_each_declared_condition_selects_a_distinct_cache_entry(tmp_path, monkey
559
568
  return FakeProfile()
560
569
 
561
570
  def build_serialized_network(self, network, config):
562
- state["builds"] += 1
563
571
  return b"small plan"
564
572
 
565
573
  monkeypatch.setattr(trt, "Builder", FakeBuilder)
@@ -606,9 +614,9 @@ def test_each_declared_condition_selects_a_distinct_cache_entry(tmp_path, monkey
606
614
  )
607
615
 
608
616
  first = open_case()
609
- assert state["builds"] == 1
617
+ assert state["graph_calls"] == 1
610
618
  assert open_case(weight_files=(same_contents,)).path == first.path
611
- assert state["builds"] == 1
619
+ assert state["graph_calls"] == 1
612
620
  variants = [
613
621
  {"name": "decoder"},
614
622
  {"graph_version": "2"},
@@ -618,7 +626,6 @@ def test_each_declared_condition_selects_a_distinct_cache_entry(tmp_path, monkey
618
626
  {"shape": (1, 4, 2)},
619
627
  {"input_range": ((1, 1, 2), (1, 3, 2), (1, 5, 2))},
620
628
  {"input_range": ((1, 1, 2), (1, 3, 2), (1, 6, 2))},
621
- {"settings": tensoracle.BuilderSettings(workspace_limit=2048, optimization_level=0)},
622
629
  {"settings": tensoracle.BuilderSettings(optimization_level=1)},
623
630
  ]
624
631
  paths = {open_case(**variant).path for variant in variants}
@@ -636,12 +643,12 @@ def test_each_declared_condition_selects_a_distinct_cache_entry(tmp_path, monkey
636
643
  paths.add(open_case().path)
637
644
  assert first.path not in paths
638
645
  assert len(paths) == len(variants) + 4
639
- assert state["builds"] == len(paths) + 1
646
+ assert state["graph_calls"] == len(paths) + 1
640
647
  assert (
641
648
  open_case(graph=lambda network: pytest.fail("graph changed without version bump")).path
642
649
  == first.path
643
650
  )
644
- assert state["builds"] == len(paths) + 1
651
+ assert state["graph_calls"] == len(paths) + 1
645
652
 
646
653
 
647
654
  @pytest.mark.gpu
@@ -22,7 +22,9 @@ def saved_cache(tmp_path, monkeypatch):
22
22
  return directory
23
23
 
24
24
 
25
- def save_engine(directory, filename="one.engine", *, conditions=None, mtime=OLD_NS, size=None):
25
+ def save_engine(
26
+ directory, filename="one.engine", *, conditions=None, mtime=OLD_NS, size=None, metadata=None
27
+ ):
26
28
  if conditions is None:
27
29
  conditions = {
28
30
  "name": "encoder",
@@ -36,7 +38,9 @@ def save_engine(directory, filename="one.engine", *, conditions=None, mtime=OLD_
36
38
  "architecture": "x86_64",
37
39
  "tensorrt_version": "11.0",
38
40
  }
39
- header = json.dumps({"conditions": conditions, "plan_sha256": "0" * 64}).encode()
41
+ header = json.dumps(
42
+ {"conditions": conditions, "plan_sha256": "0" * 64, **(metadata or {})}
43
+ ).encode()
40
44
  path = directory / filename
41
45
  path.write_bytes(b"TENSORACLE\x01" + len(header).to_bytes(8, "big") + header + b"plan")
42
46
  if size is not None:
@@ -104,6 +108,26 @@ def test_list_distinguishes_all_conditions_and_preserves_recorded_time(saved_cac
104
108
  )
105
109
 
106
110
 
111
+ @pytest.mark.req("SP-TENSORACLE-043")
112
+ @pytest.mark.req("SP-TENSORACLE-052")
113
+ @pytest.mark.parametrize(
114
+ "metadata, expected",
115
+ [({"workspace_limit": None}, None), ({"workspace_limit": 4096}, 4096), ({}, "unknown")],
116
+ ids=["tensorrt-default", "explicit-limit", "missing-record"],
117
+ )
118
+ def test_list_shows_recorded_workspace_limit_or_unknown(saved_cache, capsys, metadata, expected):
119
+ """Listing distinguishes TensorRT's default, an explicit byte limit, and a missing record."""
120
+ path, conditions = save_engine(saved_cache, metadata=metadata)
121
+ contents = path.read_bytes()
122
+ status, (listing,), stderr = call_cli(capsys, "list")
123
+ assert status == 0 and not stderr
124
+ entry = listing["entries"][0]
125
+ assert entry["workspace_limit"] == expected
126
+ assert entry["conditions"] == conditions
127
+ assert path.read_bytes() == contents
128
+ assert path.stat().st_mtime_ns == OLD_NS
129
+
130
+
107
131
  @pytest.mark.req("SP-TENSORACLE-049")
108
132
  @pytest.mark.parametrize(
109
133
  "damage", ["missing", "json", "version", "type", "checksum", "digest", "mismatch", "valid"]
@@ -20,7 +20,10 @@ def open_cached(tmp_path, monkeypatch):
20
20
  builder = SimpleNamespace(
21
21
  create_network=lambda flags: SimpleNamespace(num_inputs=0, num_outputs=0),
22
22
  create_builder_config=lambda: SimpleNamespace(
23
- clear_flag=lambda flag: None, set_memory_pool_limit=lambda pool, limit: None
23
+ clear_flag=lambda flag: None,
24
+ set_memory_pool_limit=lambda pool, limit: None,
25
+ create_timing_cache=lambda data: object(),
26
+ set_timing_cache=lambda timing, ignore_mismatch: True,
24
27
  ),
25
28
  build_serialized_network=lambda network, config: b"TensorRT plan",
26
29
  )
@@ -39,19 +42,60 @@ def open_cached(tmp_path, monkeypatch):
39
42
  def graph(network):
40
43
  built.append(True)
41
44
 
42
- def open_engine(*, io=None):
45
+ def open_engine(*, io=None, settings=None, force_rebuild=False):
43
46
  return tensoracle.open_engine(
44
47
  graph,
45
- tensoracle.EngineBuild({}),
48
+ tensoracle.EngineBuild({}, settings or tensoracle.BuilderSettings()),
46
49
  name="timestamp",
47
50
  graph_version="1",
48
51
  weights=(weight,),
49
52
  io=io or tensoracle.EngineIO({}, {}),
53
+ force_rebuild=force_rebuild,
50
54
  )
51
55
 
52
56
  return open_engine, built
53
57
 
54
58
 
59
+ @pytest.mark.req("SP-TENSORACLE-052")
60
+ @pytest.mark.parametrize("workspace_limit", [None, 8192])
61
+ def test_workspace_limit_changes_reuse_saved_engine(open_cached, workspace_limit):
62
+ """Changing only the workspace limit reuses the same file without invoking the graph."""
63
+ open_engine, built = open_cached
64
+ first = open_engine(settings=tensoracle.BuilderSettings(workspace_limit=4096))
65
+ contents = first.path.read_bytes()
66
+ reopened = open_engine(settings=tensoracle.BuilderSettings(workspace_limit=workspace_limit))
67
+ assert reopened.path == first.path
68
+ assert built == [True]
69
+ assert first.path.read_bytes() == contents
70
+
71
+
72
+ @pytest.mark.req("SP-TENSORACLE-052")
73
+ @pytest.mark.req("SP-TENSORACLE-011")
74
+ @pytest.mark.parametrize("workspace_limit", [None, 4096])
75
+ def test_saved_engine_records_workspace_limit_and_updates_it_only_on_rebuild(
76
+ open_cached, workspace_limit
77
+ ):
78
+ """Headers record the actual build's workspace outside conditions and forced rebuilds replace it."""
79
+ open_engine, built = open_cached
80
+
81
+ def header(path):
82
+ with path.open("rb") as stream:
83
+ return cache._read_header(stream, path, path.stat().st_size)
84
+
85
+ first = open_engine(settings=tensoracle.BuilderSettings(workspace_limit=workspace_limit))
86
+ original = header(first.path)
87
+ assert original["workspace_limit"] == workspace_limit
88
+ assert original["conditions"]["builder_settings"] == {"optimization_level": 3, "tf32": False}
89
+ changed_limit = 8192 if workspace_limit is None else None
90
+ rebuilt = open_engine(
91
+ settings=tensoracle.BuilderSettings(workspace_limit=changed_limit), force_rebuild=True
92
+ )
93
+ assert rebuilt.path == first.path
94
+ assert header(rebuilt.path)["workspace_limit"] == changed_limit
95
+ assert header(rebuilt.path)["conditions"] == original["conditions"]
96
+ assert built == [True, True]
97
+
98
+
55
99
  @pytest.mark.req("SP-TENSORACLE-042")
56
100
  @pytest.mark.req("SP-TENSORACLE-048")
57
101
  @pytest.mark.req("SP-TENSORACLE-046")
@@ -1,15 +1,80 @@
1
- """Separate saved engines by the CUDA device properties used for compatibility."""
1
+ """Save compact engines and separate them by CUDA device compatibility."""
2
2
 
3
3
  import ctypes
4
+ import time
4
5
  from types import SimpleNamespace
5
6
 
7
+ import numpy as np
6
8
  import pytest
7
9
  import tensorrt as trt
10
+ from safetensors.numpy import save_file
8
11
 
9
12
  import tensoracle
10
13
  import tensoracle._cache as cache
11
14
 
12
15
 
16
+ @pytest.mark.gpu
17
+ @pytest.mark.req("SP-TENSORACLE-009")
18
+ def test_compact_saved_engine_executes_and_reopens_without_calling_graph(tmp_path, monkeypatch):
19
+ """A LeakyReLU engine stays below 1 MiB and computes correctly before and after reopening."""
20
+ import cupy as cp
21
+
22
+ monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache"))
23
+ weights = tmp_path / "weights.safetensors"
24
+ save_file({"slope": np.array([0.125], dtype=np.float16)}, weights)
25
+ shape = (1, 64, 135, 240)
26
+ graph_calls = []
27
+
28
+ def graph(network):
29
+ graph_calls.append(1)
30
+ features = network.add_input("features", trt.float16, shape)
31
+ layer = network.add_activation(features, trt.ActivationType.LEAKY_RELU)
32
+ layer.alpha = 0.125
33
+ result = layer.get_output(0)
34
+ result.name = "result"
35
+ network.mark_output(result)
36
+
37
+ def open_engine(callback):
38
+ return tensoracle.open_engine(
39
+ callback,
40
+ tensoracle.EngineBuild({"features": tensoracle.InputShape.fixed(shape)}),
41
+ name="compact-leaky-relu",
42
+ graph_version="1",
43
+ weights=(weights,),
44
+ io=tensoracle.EngineIO(
45
+ {"features": tensoracle.TensorSpec(trt.float16, shape)},
46
+ {"result": tensoracle.TensorSpec(trt.float16, shape)},
47
+ ),
48
+ )
49
+
50
+ started = time.perf_counter()
51
+ first = open_engine(graph)
52
+ elapsed = time.perf_counter() - started
53
+ size = first.path.stat().st_size
54
+ print(f"LeakyReLU saved engine: {size} bytes, build/open: {elapsed:.6f} s")
55
+ assert graph_calls == [1]
56
+ assert size < 1024 * 1024
57
+
58
+ def forbidden(network):
59
+ pytest.fail("saved engine invoked the graph callback")
60
+
61
+ reopened = open_engine(forbidden)
62
+ assert reopened.path == first.path
63
+ assert reopened.path.stat().st_size == size
64
+ assert graph_calls == [1]
65
+ host = np.linspace(-1, 1, np.prod(shape), dtype=np.float32).astype(np.float16).reshape(shape)
66
+ expected = np.where(host >= 0, host, host * np.float16(0.125))
67
+ value = cp.asarray(host)
68
+ for handle in (first, reopened):
69
+ output = cp.asarray(np.full_like(host, np.nan))
70
+ with handle.create_execution() as execution:
71
+ execution.enqueue(
72
+ {"features": value.data.ptr, "result": output.data.ptr},
73
+ stream=cp.cuda.get_current_stream().ptr,
74
+ )
75
+ np.testing.assert_array_equal(cp.asnumpy(output), expected)
76
+
77
+
13
78
  @pytest.fixture
14
79
  def cuda_driver(monkeypatch):
15
80
  properties = {
@@ -128,17 +193,18 @@ def test_changed_gpu_property_builds_a_separate_saved_engine(
128
193
  monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache"))
129
194
  builds = []
130
195
 
131
- def serialize(network, config):
196
+ def graph(network):
132
197
  builds.append(cuda_driver.device)
133
- return b"serialized engine"
134
198
 
135
199
  builder = SimpleNamespace(
136
200
  create_network=lambda flags: SimpleNamespace(num_inputs=0, num_outputs=0),
137
201
  create_builder_config=lambda: SimpleNamespace(
138
202
  clear_flag=lambda flag: None,
139
203
  set_memory_pool_limit=lambda pool, limit: None,
204
+ create_timing_cache=lambda data: object(),
205
+ set_timing_cache=lambda timing, ignore_mismatch: True,
140
206
  ),
141
- build_serialized_network=serialize,
207
+ build_serialized_network=lambda network, config: b"serialized engine",
142
208
  )
143
209
  monkeypatch.setattr(trt, "Builder", lambda logger: builder)
144
210
  monkeypatch.setattr(
@@ -153,7 +219,7 @@ def test_changed_gpu_property_builds_a_separate_saved_engine(
153
219
 
154
220
  def open_engine():
155
221
  return tensoracle.open_engine(
156
- lambda network: None,
222
+ graph,
157
223
  tensoracle.EngineBuild({}),
158
224
  name="encoder",
159
225
  graph_version="1",
@@ -297,6 +297,7 @@ def test_record_lives_in_user_cache_directory(tmp_path, monkeypatch, xdg):
297
297
 
298
298
  @pytest.mark.req("SP-TENSORACLE-002")
299
299
  @pytest.mark.req("SP-TENSORACLE-009")
300
+ @pytest.mark.req("SP-TENSORACLE-052")
300
301
  def test_engine_identity_matches_the_full_hash_calculation(tmp_path, monkeypatch):
301
302
  """Stored digests produce the same conditions and engine name as full file hashing."""
302
303
  root = tmp_path / "cache"
@@ -312,7 +313,7 @@ def test_engine_identity_matches_the_full_hash_calculation(tmp_path, monkeypatch
312
313
  "weights": [hashlib.sha256(weight.read_bytes()).hexdigest()],
313
314
  "plugins": [hashlib.sha256(plugin.read_bytes()).hexdigest()],
314
315
  "inputs": {"features": {"minimum": [1, 3, 2], "optimum": [1, 3, 2], "maximum": [1, 3, 2]}},
315
- "builder_settings": {"workspace_limit": 1 << 30, "optimization_level": 3, "tf32": False},
316
+ "builder_settings": {"optimization_level": 3, "tf32": False},
316
317
  "gpu": {"name": "GPU A"},
317
318
  "os": cache.platform.system(),
318
319
  "architecture": cache.platform.machine(),
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes