nvmath-python 1.0.0__cp314-cp314t-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (332) hide show
  1. nvmath/__init__.pxd +0 -0
  2. nvmath/__init__.py +45 -0
  3. nvmath/_internal/__init__.py +0 -0
  4. nvmath/_internal/attribute_ifc_factory.py +330 -0
  5. nvmath/_internal/layout.py +70 -0
  6. nvmath/_internal/templates.py +130 -0
  7. nvmath/_internal/threadsafe.py +106 -0
  8. nvmath/_internal/utils.py +43 -0
  9. nvmath/_internal/workspace.py +490 -0
  10. nvmath/_utils.py +147 -0
  11. nvmath/bindings/__init__.py +60 -0
  12. nvmath/bindings/_internal/__init__.pxd +0 -0
  13. nvmath/bindings/_internal/__init__.py +0 -0
  14. nvmath/bindings/_internal/common_types.pxd +31 -0
  15. nvmath/bindings/_internal/cublas.cp314t-win_amd64.pyd +0 -0
  16. nvmath/bindings/_internal/cublas.pxd +530 -0
  17. nvmath/bindings/_internal/cublasLt.cp314t-win_amd64.pyd +0 -0
  18. nvmath/bindings/_internal/cublasLt.pxd +59 -0
  19. nvmath/bindings/_internal/cublasMp.pxd +52 -0
  20. nvmath/bindings/_internal/cudss.cp314t-win_amd64.pyd +0 -0
  21. nvmath/bindings/_internal/cudss.pxd +54 -0
  22. nvmath/bindings/_internal/cufft.cp314t-win_amd64.pyd +0 -0
  23. nvmath/bindings/_internal/cufft.pxd +70 -0
  24. nvmath/bindings/_internal/cufftMp.pxd +77 -0
  25. nvmath/bindings/_internal/curand.cp314t-win_amd64.pyd +0 -0
  26. nvmath/bindings/_internal/curand.pxd +42 -0
  27. nvmath/bindings/_internal/cusolver.cp314t-win_amd64.pyd +0 -0
  28. nvmath/bindings/_internal/cusolver.pxd +15 -0
  29. nvmath/bindings/_internal/cusolverDn.cp314t-win_amd64.pyd +0 -0
  30. nvmath/bindings/_internal/cusolverDn.pxd +406 -0
  31. nvmath/bindings/_internal/cusolverMp.pxd +71 -0
  32. nvmath/bindings/_internal/cusolverSp.cp314t-win_amd64.pyd +0 -0
  33. nvmath/bindings/_internal/cusolverSp.pxd +75 -0
  34. nvmath/bindings/_internal/cusparse.cp314t-win_amd64.pyd +0 -0
  35. nvmath/bindings/_internal/cusparse.pxd +471 -0
  36. nvmath/bindings/_internal/cusparseLt.cp314t-win_amd64.pyd +0 -0
  37. nvmath/bindings/_internal/cusparseLt.pxd +48 -0
  38. nvmath/bindings/_internal/cutensor.cp314t-win_amd64.pyd +0 -0
  39. nvmath/bindings/_internal/cutensor.pxd +58 -0
  40. nvmath/bindings/_internal/mathdx.cp314t-win_amd64.pyd +0 -0
  41. nvmath/bindings/_internal/mathdx.pxd +116 -0
  42. nvmath/bindings/_internal/nvshmem.pxd +29 -0
  43. nvmath/bindings/_internal/utils.cp314t-win_amd64.pyd +0 -0
  44. nvmath/bindings/_internal/utils.pxd +174 -0
  45. nvmath/bindings/_internal/utils.pyi +10 -0
  46. nvmath/bindings/cublas.cp314t-win_amd64.pyd +0 -0
  47. nvmath/bindings/cublas.pxd +558 -0
  48. nvmath/bindings/cublas.pyi +812 -0
  49. nvmath/bindings/cublasLt.cp314t-win_amd64.pyd +0 -0
  50. nvmath/bindings/cublasLt.pxd +109 -0
  51. nvmath/bindings/cublasLt.pyi +1461 -0
  52. nvmath/bindings/cublasMp.pxd +85 -0
  53. nvmath/bindings/cublasMp.pyi +267 -0
  54. nvmath/bindings/cudss.cp314t-win_amd64.pyd +0 -0
  55. nvmath/bindings/cudss.pxd +98 -0
  56. nvmath/bindings/cudss.pyi +443 -0
  57. nvmath/bindings/cufft.cp314t-win_amd64.pyd +0 -0
  58. nvmath/bindings/cufft.pxd +118 -0
  59. nvmath/bindings/cufft.pyi +301 -0
  60. nvmath/bindings/cufftMp.pxd +124 -0
  61. nvmath/bindings/cufftMp.pyi +326 -0
  62. nvmath/bindings/curand.cp314t-win_amd64.pyd +0 -0
  63. nvmath/bindings/curand.pxd +71 -0
  64. nvmath/bindings/curand.pyi +189 -0
  65. nvmath/bindings/cusolver.cp314t-win_amd64.pyd +0 -0
  66. nvmath/bindings/cusolver.pxd +62 -0
  67. nvmath/bindings/cusolver.pyi +320 -0
  68. nvmath/bindings/cusolverDn.cp314t-win_amd64.pyd +0 -0
  69. nvmath/bindings/cusolverDn.pxd +430 -0
  70. nvmath/bindings/cusolverDn.pyi +422 -0
  71. nvmath/bindings/cusolverMp.pxd +98 -0
  72. nvmath/bindings/cusolverMp.pyi +114 -0
  73. nvmath/bindings/cusolverSp.cp314t-win_amd64.pyd +0 -0
  74. nvmath/bindings/cusolverSp.pxd +95 -0
  75. nvmath/bindings/cusolverSp.pyi +70 -0
  76. nvmath/bindings/cusparse.cp314t-win_amd64.pyd +0 -0
  77. nvmath/bindings/cusparse.pxd +546 -0
  78. nvmath/bindings/cusparse.pyi +1017 -0
  79. nvmath/bindings/cusparseLt.cp314t-win_amd64.pyd +0 -0
  80. nvmath/bindings/cusparseLt.pxd +99 -0
  81. nvmath/bindings/cusparseLt.pyi +252 -0
  82. nvmath/bindings/cutensor.cp314t-win_amd64.pyd +0 -0
  83. nvmath/bindings/cutensor.pxd +98 -0
  84. nvmath/bindings/cutensor.pyi +324 -0
  85. nvmath/bindings/cycublas.cp314t-win_amd64.pyd +0 -0
  86. nvmath/bindings/cycublas.pxd +664 -0
  87. nvmath/bindings/cycublasLt.cp314t-win_amd64.pyd +0 -0
  88. nvmath/bindings/cycublasLt.pxd +1045 -0
  89. nvmath/bindings/cycublasMp.pxd +171 -0
  90. nvmath/bindings/cycudss.cp314t-win_amd64.pyd +0 -0
  91. nvmath/bindings/cycudss.pxd +277 -0
  92. nvmath/bindings/cycufft.cp314t-win_amd64.pyd +0 -0
  93. nvmath/bindings/cycufft.pxd +333 -0
  94. nvmath/bindings/cycufftMp.pxd +342 -0
  95. nvmath/bindings/cycurand.cp314t-win_amd64.pyd +0 -0
  96. nvmath/bindings/cycurand.pxd +141 -0
  97. nvmath/bindings/cycusolver.cp314t-win_amd64.pyd +0 -0
  98. nvmath/bindings/cycusolver.pxd +137 -0
  99. nvmath/bindings/cycusolverDn.cp314t-win_amd64.pyd +0 -0
  100. nvmath/bindings/cycusolverDn.pxd +443 -0
  101. nvmath/bindings/cycusolverMp.pxd +107 -0
  102. nvmath/bindings/cycusolverSp.cp314t-win_amd64.pyd +0 -0
  103. nvmath/bindings/cycusolverSp.pxd +93 -0
  104. nvmath/bindings/cycusparse.cp314t-win_amd64.pyd +0 -0
  105. nvmath/bindings/cycusparse.pxd +679 -0
  106. nvmath/bindings/cycusparseLt.cp314t-win_amd64.pyd +0 -0
  107. nvmath/bindings/cycusparseLt.pxd +135 -0
  108. nvmath/bindings/cycutensor.cp314t-win_amd64.pyd +0 -0
  109. nvmath/bindings/cycutensor.pxd +189 -0
  110. nvmath/bindings/cymathdx.cp314t-win_amd64.pyd +0 -0
  111. nvmath/bindings/cymathdx.pxd +552 -0
  112. nvmath/bindings/cynvshmem.pxd +118 -0
  113. nvmath/bindings/mathdx.cp314t-win_amd64.pyd +0 -0
  114. nvmath/bindings/mathdx.pxd +182 -0
  115. nvmath/bindings/mathdx.pyi +1562 -0
  116. nvmath/bindings/nvpl/__init__.pxd +0 -0
  117. nvmath/bindings/nvpl/__init__.py +13 -0
  118. nvmath/bindings/nvpl/_internal/__init__.pxd +0 -0
  119. nvmath/bindings/nvpl/_internal/__init__.py +0 -0
  120. nvmath/bindings/nvpl/_internal/blas.cp314t-win_amd64.pyd +0 -0
  121. nvmath/bindings/nvpl/_internal/blas.pxd +237 -0
  122. nvmath/bindings/nvpl/_internal/fft.cp314t-win_amd64.pyd +0 -0
  123. nvmath/bindings/nvpl/_internal/fft.pxd +36 -0
  124. nvmath/bindings/nvpl/blas.cp314t-win_amd64.pyd +0 -0
  125. nvmath/bindings/nvpl/blas.pxd +131 -0
  126. nvmath/bindings/nvpl/blas.pyi +168 -0
  127. nvmath/bindings/nvpl/cyblas.cp314t-win_amd64.pyd +0 -0
  128. nvmath/bindings/nvpl/cyblas.pxd +280 -0
  129. nvmath/bindings/nvpl/cyfft.cp314t-win_amd64.pyd +0 -0
  130. nvmath/bindings/nvpl/cyfft.pxd +93 -0
  131. nvmath/bindings/nvpl/fft.cp314t-win_amd64.pyd +0 -0
  132. nvmath/bindings/nvpl/fft.pxd +100 -0
  133. nvmath/bindings/nvpl/fft.pyi +168 -0
  134. nvmath/bindings/nvshmem.pxd +54 -0
  135. nvmath/bindings/nvshmem.pyi +191 -0
  136. nvmath/device/__init__.py +38 -0
  137. nvmath/device/_deprecated.py +33 -0
  138. nvmath/device/common.py +315 -0
  139. nvmath/device/common_backend.py +131 -0
  140. nvmath/device/common_cuda.py +201 -0
  141. nvmath/device/common_numba.py +300 -0
  142. nvmath/device/common_numba_cuda_mlir.py +202 -0
  143. nvmath/device/common_opaque_tensor.py +201 -0
  144. nvmath/device/cublasdx.py +1606 -0
  145. nvmath/device/cublasdx_backend.py +860 -0
  146. nvmath/device/cublasdx_numba.py +1534 -0
  147. nvmath/device/cublasdx_numba_cuda_mlir.py +208 -0
  148. nvmath/device/cufftdx.py +373 -0
  149. nvmath/device/cufftdx_backend.py +220 -0
  150. nvmath/device/cufftdx_numba.py +140 -0
  151. nvmath/device/cufftdx_numba_cuda_mlir.py +79 -0
  152. nvmath/device/curand_kernel.py +9147 -0
  153. nvmath/device/cusolverdx.py +2708 -0
  154. nvmath/device/cusolverdx_backend.py +440 -0
  155. nvmath/device/cusolverdx_numba.py +567 -0
  156. nvmath/device/cusolverdx_numba_cuda_mlir.py +604 -0
  157. nvmath/device/cusolverdx_overload_backend.py +1029 -0
  158. nvmath/device/llvm_array.py +29 -0
  159. nvmath/device/random.py +441 -0
  160. nvmath/device/random_helpers.py +23 -0
  161. nvmath/device/random_states.py +187 -0
  162. nvmath/device/types.py +138 -0
  163. nvmath/device/vector_types_numba.py +259 -0
  164. nvmath/distributed/__init__.py +200 -0
  165. nvmath/distributed/_internal/__init__.py +0 -0
  166. nvmath/distributed/_internal/nccl.py +86 -0
  167. nvmath/distributed/_internal/nvshmem.py +307 -0
  168. nvmath/distributed/_internal/symmetric_memory.py +35 -0
  169. nvmath/distributed/_internal/tensor_ifc.py +70 -0
  170. nvmath/distributed/_internal/tensor_ifc_cupy.py +68 -0
  171. nvmath/distributed/_internal/tensor_ifc_host_device.py +172 -0
  172. nvmath/distributed/_internal/tensor_ifc_numpy.py +46 -0
  173. nvmath/distributed/_internal/tensor_ifc_torch.py +162 -0
  174. nvmath/distributed/_internal/tensor_wrapper.py +81 -0
  175. nvmath/distributed/_utils.py +167 -0
  176. nvmath/distributed/distribution/__init__.py +30 -0
  177. nvmath/distributed/distribution/_configuration.py +39 -0
  178. nvmath/distributed/distribution/distributions.py +1024 -0
  179. nvmath/distributed/distribution/redistribute.py +1284 -0
  180. nvmath/distributed/fft/__init__.py +7 -0
  181. nvmath/distributed/fft/_configuration.py +82 -0
  182. nvmath/distributed/fft/fft.py +2742 -0
  183. nvmath/distributed/linalg/__init__.py +22 -0
  184. nvmath/distributed/linalg/_internal/__init__.py +3 -0
  185. nvmath/distributed/linalg/_internal/epilog_protocol.py +586 -0
  186. nvmath/distributed/linalg/_internal/matmul_desc_ifc.py +28 -0
  187. nvmath/distributed/linalg/advanced/__init__.py +8 -0
  188. nvmath/distributed/linalg/advanced/_configuration.py +171 -0
  189. nvmath/distributed/linalg/advanced/matmulmod.py +3573 -0
  190. nvmath/distributed/linalg/generic/__init__.py +8 -0
  191. nvmath/distributed/linalg/generic/_caching.py +66 -0
  192. nvmath/distributed/linalg/generic/_configuration.py +61 -0
  193. nvmath/distributed/linalg/generic/_factorization.py +172 -0
  194. nvmath/distributed/linalg/generic/_initialization.py +966 -0
  195. nvmath/distributed/linalg/generic/_problem_spec.py +511 -0
  196. nvmath/distributed/linalg/generic/solvermod.py +1368 -0
  197. nvmath/distributed/process_group.py +408 -0
  198. nvmath/fft/__init__.py +7 -0
  199. nvmath/fft/_configuration.py +189 -0
  200. nvmath/fft/_exec_utils.py +82 -0
  201. nvmath/fft/_helpers.py +237 -0
  202. nvmath/fft/fft.py +3122 -0
  203. nvmath/internal/__init__.pxd +3 -0
  204. nvmath/internal/__init__.py +10 -0
  205. nvmath/internal/_bindings.cp314t-win_amd64.pyd +0 -0
  206. nvmath/internal/_bindings.pxd +18 -0
  207. nvmath/internal/_device_utils.py +45 -0
  208. nvmath/internal/_layout/__init__.pxd +3 -0
  209. nvmath/internal/_layout/__init__.py +7 -0
  210. nvmath/internal/_layout/_layout.cp314t-win_amd64.pyd +0 -0
  211. nvmath/internal/_layout/_layout.pxd +1303 -0
  212. nvmath/internal/_layout/_layout.pyi +1145 -0
  213. nvmath/internal/enum_utils.py +142 -0
  214. nvmath/internal/formatters.py +87 -0
  215. nvmath/internal/mem_limit.py +51 -0
  216. nvmath/internal/memory.cp314t-win_amd64.pyd +0 -0
  217. nvmath/internal/memory.pxd +13 -0
  218. nvmath/internal/memory.pyi +50 -0
  219. nvmath/internal/ndbuffer/__init__.pxd +3 -0
  220. nvmath/internal/ndbuffer/__init__.py +9 -0
  221. nvmath/internal/ndbuffer/_copy_kernel.cp314t-win_amd64.pyd +0 -0
  222. nvmath/internal/ndbuffer/_copy_kernel.pxd +10 -0
  223. nvmath/internal/ndbuffer/_jit.cp314t-win_amd64.pyd +0 -0
  224. nvmath/internal/ndbuffer/_jit.pxd +7 -0
  225. nvmath/internal/ndbuffer/_ndbuffer.cp314t-win_amd64.pyd +0 -0
  226. nvmath/internal/ndbuffer/_ndbuffer.pxd +40 -0
  227. nvmath/internal/ndbuffer/_ndbuffer.pyi +463 -0
  228. nvmath/internal/ndbuffer/copy_kernel/args.h +34 -0
  229. nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/array_view.h +52 -0
  230. nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/elementwise.h +68 -0
  231. nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/grid_indexer.h +69 -0
  232. nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/transposed.h +242 -0
  233. nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/type_utils.h +39 -0
  234. nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/utils.h +132 -0
  235. nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/vec.h +159 -0
  236. nvmath/internal/ndbuffer/copy_kernel/elementwise.h +53 -0
  237. nvmath/internal/ndbuffer/copy_kernel/transposed.h +58 -0
  238. nvmath/internal/package_ifc.py +168 -0
  239. nvmath/internal/package_ifc_cuda.py +57 -0
  240. nvmath/internal/package_ifc_cupy.py +67 -0
  241. nvmath/internal/package_ifc_torch.py +69 -0
  242. nvmath/internal/package_wrapper.py +14 -0
  243. nvmath/internal/tensor_ifc.py +179 -0
  244. nvmath/internal/tensor_ifc_cupy.py +234 -0
  245. nvmath/internal/tensor_ifc_ndbuffer.py +147 -0
  246. nvmath/internal/tensor_ifc_numpy.py +184 -0
  247. nvmath/internal/tensor_ifc_torch.py +178 -0
  248. nvmath/internal/tensor_wrapper.py +160 -0
  249. nvmath/internal/typemaps.py +113 -0
  250. nvmath/internal/utils.py +805 -0
  251. nvmath/linalg/__init__.py +56 -0
  252. nvmath/linalg/_internal/__init__.py +3 -0
  253. nvmath/linalg/_internal/algo_cap_ifc.py +82 -0
  254. nvmath/linalg/_internal/algo_config_ifc.py +43 -0
  255. nvmath/linalg/_internal/batch.py +234 -0
  256. nvmath/linalg/_internal/enum_to_tuples.py +64 -0
  257. nvmath/linalg/_internal/epilog_protocol.py +766 -0
  258. nvmath/linalg/_internal/layout.py +624 -0
  259. nvmath/linalg/_internal/matmul_desc_ifc.py +28 -0
  260. nvmath/linalg/_internal/matmul_pref_ifc.py +27 -0
  261. nvmath/linalg/_internal/matrix_layout_ifc.py +26 -0
  262. nvmath/linalg/_internal/solver_utils.py +432 -0
  263. nvmath/linalg/_internal/typemaps.py +144 -0
  264. nvmath/linalg/_internal/utils.py +157 -0
  265. nvmath/linalg/advanced/__init__.py +8 -0
  266. nvmath/linalg/advanced/_algorithmmod.py +170 -0
  267. nvmath/linalg/advanced/_configuration.py +351 -0
  268. nvmath/linalg/advanced/helpers/__init__.py +5 -0
  269. nvmath/linalg/advanced/helpers/matmul.py +1316 -0
  270. nvmath/linalg/advanced/matmulmod.py +3734 -0
  271. nvmath/linalg/generic/__init__.py +53 -0
  272. nvmath/linalg/generic/_configuration/__init__.py +39 -0
  273. nvmath/linalg/generic/_configuration/layout.py +263 -0
  274. nvmath/linalg/generic/_configuration/match.py +734 -0
  275. nvmath/linalg/generic/_configuration/qualifiers.py +493 -0
  276. nvmath/linalg/generic/_configuration/solver_configuration.py +59 -0
  277. nvmath/linalg/generic/_configuration/wrap.py +217 -0
  278. nvmath/linalg/generic/_dtype.py +15 -0
  279. nvmath/linalg/generic/matmulmod.py +2094 -0
  280. nvmath/linalg/generic/solvermod.py +1301 -0
  281. nvmath/memory.py +279 -0
  282. nvmath/sparse/__init__.py +38 -0
  283. nvmath/sparse/_internal/__init__.py +21 -0
  284. nvmath/sparse/_internal/common_utils.py +147 -0
  285. nvmath/sparse/_internal/cudss_config_ifc.py +702 -0
  286. nvmath/sparse/_internal/cudss_data_ifc.py +399 -0
  287. nvmath/sparse/_internal/cudss_utils.py +506 -0
  288. nvmath/sparse/_internal/cusparse_utils.py +382 -0
  289. nvmath/sparse/_internal/sparse_bsc_ifc.py +303 -0
  290. nvmath/sparse/_internal/sparse_bsr_ifc.py +305 -0
  291. nvmath/sparse/_internal/sparse_coo_ifc.py +256 -0
  292. nvmath/sparse/_internal/sparse_csc_ifc.py +268 -0
  293. nvmath/sparse/_internal/sparse_csr_ifc.py +288 -0
  294. nvmath/sparse/_internal/sparse_dia_ifc.py +242 -0
  295. nvmath/sparse/_internal/sparse_format_helpers.py +601 -0
  296. nvmath/sparse/_internal/sparse_tensor_ifc.py +133 -0
  297. nvmath/sparse/_internal/sparse_ust_ifc.py +141 -0
  298. nvmath/sparse/_internal/utils.py +56 -0
  299. nvmath/sparse/advanced/__init__.py +7 -0
  300. nvmath/sparse/advanced/_configuration.py +227 -0
  301. nvmath/sparse/advanced/direct_solver.py +2069 -0
  302. nvmath/sparse/generic/__init__.py +7 -0
  303. nvmath/sparse/generic/_configuration.py +129 -0
  304. nvmath/sparse/generic/_helpers.py +137 -0
  305. nvmath/sparse/generic/_thunks.py +21 -0
  306. nvmath/sparse/generic/matmulmod.py +2353 -0
  307. nvmath/sparse/ust/__init__.py +7 -0
  308. nvmath/sparse/ust/_converters.py +422 -0
  309. nvmath/sparse/ust/_cpp.py +28 -0
  310. nvmath/sparse/ust/_drawer.py +565 -0
  311. nvmath/sparse/ust/_emitter.py +1033 -0
  312. nvmath/sparse/ust/_jit.py +188 -0
  313. nvmath/sparse/ust/_kernel.py +282 -0
  314. nvmath/sparse/ust/_utils.py +149 -0
  315. nvmath/sparse/ust/interfaces/__init__.py +0 -0
  316. nvmath/sparse/ust/interfaces/torch_interface.py +476 -0
  317. nvmath/sparse/ust/tensor.py +1016 -0
  318. nvmath/sparse/ust/tensor_format.py +957 -0
  319. nvmath/tensor/__init__.py +6 -0
  320. nvmath/tensor/_configuration.py +120 -0
  321. nvmath/tensor/_internal/__init__.py +3 -0
  322. nvmath/tensor/_internal/cutensor_config_ifc.py +279 -0
  323. nvmath/tensor/_internal/cutensor_utils.py +230 -0
  324. nvmath/tensor/_internal/data.py +43 -0
  325. nvmath/tensor/_internal/einsum_parser.py +444 -0
  326. nvmath/tensor/_internal/typemaps.py +96 -0
  327. nvmath/tensor/contract.py +1900 -0
  328. nvmath_python-1.0.0.dist-info/METADATA +134 -0
  329. nvmath_python-1.0.0.dist-info/RECORD +332 -0
  330. nvmath_python-1.0.0.dist-info/WHEEL +5 -0
  331. nvmath_python-1.0.0.dist-info/licenses/LICENSE +177 -0
  332. nvmath_python-1.0.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,1303 @@
1
+ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
2
+ #
3
+ # SPDX-License-Identifier: Apache-2.0
4
+
5
+ """
6
+ StridedLayout can be used from Python and Cython code.
7
+ In Python, it feels customary for layout meta-data (shape/strides) to be
8
+ immutable (tuples), so the StridedLayout is immutable too
9
+ (to avoid unexpected modifications of a shared layout).
10
+ All transforming methods (``reshaped``, ``permuted``, etc.)
11
+ return a new instance if the transform is not a no-op.
12
+
13
+ In Cython, to reduce Python overhead (class instantiation, gil acquisition),
14
+ StridedLayout has extra cdef-ed methods (``reshape_into``, ``permute_into``, etc.)
15
+ that accept an instance of StridedLayout to store the result, that
16
+ can be self (in which case it will be modified in place). It is Cython
17
+ user responsibility to make sure externally provided StridedLayout instances
18
+ are not modified in-place if they may be used elsewhere (e.g. by calling
19
+ copy() method at the Cython function entry point).
20
+ """
21
+
22
+ cimport cpython
23
+ from cpython.object cimport PyObject
24
+
25
+ cimport cython
26
+ from cython.operator cimport dereference as deref
27
+
28
+ from libc.stdint cimport intptr_t, uint64_t, int64_t, int32_t
29
+ from libcpp cimport vector
30
+ from libcpp.cmath cimport abs as c_abs
31
+
32
+
33
+ ctypedef fused integer_t:
34
+ int64_t
35
+ int32_t
36
+
37
+
38
+ cdef extern from "../include/_layout.hpp" namespace "nvmath_strided_layout":
39
+
40
+ ctypedef uint64_t axes_mask_t
41
+ ctypedef int64_t extent_t
42
+ ctypedef int64_t stride_t
43
+ ctypedef int32_t axis_t
44
+ ctypedef vector.vector[stride_t] extents_strides_mem_t
45
+ ctypedef vector.vector[axis_t] axis_vec_t
46
+
47
+ int _popcount(axes_mask_t x) nogil
48
+ void _order_from_strides(axis_vec_t& indices, extent_t* extents, stride_t* strides, int ndim) except + nogil
49
+ void _swap(extents_strides_mem_t &a, extents_strides_mem_t &b) noexcept nogil
50
+ void _swap(int64_t*& a, int64_t*& b) noexcept nogil
51
+ void _swap(extent_t& a, extent_t& b) noexcept nogil
52
+ void _swap(int& a, int& b) noexcept nogil
53
+ void _swap(axis_vec_t &a, axis_vec_t &b) noexcept nogil
54
+
55
+
56
+ cdef extern from "../include/_layout.hpp":
57
+ cdef int STRIDED_LAYOUT_MAX_NDIM
58
+ cdef axes_mask_t STRIDED_LAYOUT_AXES_MASK_ALL
59
+
60
+
61
+ cdef extern from "../include/_layout_properties.hpp" namespace "nvmath_strided_layout":
62
+ ctypedef unsigned int property_mask_t
63
+
64
+ cdef enum OrderFlag:
65
+ ORDER_NONE
66
+ ORDER_C
67
+ ORDER_F
68
+ ORDER_PERM
69
+
70
+ enum BooleanProperty:
71
+ IS_CONTIGUOUS_C
72
+ IS_ABS_DENSE_C
73
+ IS_LEADING_DENSE_C
74
+ IS_ABS_LEADING_DENSE_C
75
+ IS_CONTIGUOUS_F
76
+ IS_ABS_DENSE_F
77
+ IS_LEADING_DENSE_F
78
+ IS_ABS_LEADING_DENSE_F
79
+ IS_CONTIGUOUS_ANY
80
+ IS_ABS_DENSE_ANY
81
+ IS_LEADING_DENSE_ANY
82
+ IS_ABS_LEADING_DENSE_ANY
83
+ HAS_STRIDE_ORDER_C
84
+ HAS_STRIDE_ORDER_F
85
+ IS_UNIQUE
86
+ IS_EXHAUSTIVE
87
+ HAS_NO_NEGATIVE_STRIDE
88
+
89
+ enum Property:
90
+ OFFSET_BOUNDS
91
+ MEMORY_RANGE_SIZE
92
+ SHAPE
93
+ STRIDES
94
+ STRIDES_IN_BYTES
95
+ STRIDE_ORDER
96
+ UNIT_EXTENTS
97
+ VOLUME
98
+ MIN_MAX_STRIDE
99
+
100
+ cdef bint _has_valid_property(property_mask_t has_props, int prop) noexcept nogil
101
+ cdef void _mark_property_valid(property_mask_t& has_props, int prop) noexcept nogil
102
+ cdef bint _boolean_property(property_mask_t bool_props, int prop) noexcept nogil
103
+ cdef void _set_boolean_property(property_mask_t& has_props, property_mask_t& bool_props, int prop, bint value) noexcept nogil
104
+ cdef int _dense_prop(OrderFlag order_flag, int allow_negative_strides, int allow_leading_dim_stride) noexcept nogil
105
+
106
+
107
+ cdef struct BaseLayout:
108
+ # A struct holding the shape and strides for the layout.
109
+ # The struct must be initialized before it is passed to any other APIs.
110
+ # Use ``init_base_layout`` for minimal initialization of the layout,
111
+ # it will set the ``shape`` and ``strides`` pointers to point to
112
+ # ndim contiguous integer arrays. After that you are responsible
113
+ # for filling out the ``ndim``, ``shape`` array and ``strides`` array.
114
+ # Alternatively, use ``init_base_layout_from_tuple`` or ``init_base_layout_from_ptr``
115
+ # (followed by zero_out_base_strides or set_base_dense_strides
116
+ # if no explicit strides are provided) to initialize and validate the layout.
117
+
118
+ # Note, extent_t and stride_t are both int64_t and we use a single _mem
119
+ # allocation for both (there's some overhead to small allocations but even
120
+ # more so, Cython adds overhead on handling very unlikely exceptions
121
+ # around the vector instantiation).
122
+ extents_strides_mem_t _mem
123
+ extent_t* shape
124
+ stride_t* strides
125
+ int ndim
126
+ int itemsize
127
+
128
+
129
+ @cython.final
130
+ cdef class StridedLayout:
131
+
132
+ # Definition
133
+ cdef:
134
+ BaseLayout base
135
+
136
+ # Lazy properties computed from the BaseLayout.
137
+ cdef:
138
+ # Set to 0 to invalidate all properties,
139
+ # whenever a defining attribute is changed
140
+ property_mask_t _prop_mask
141
+
142
+ # C and Python properties
143
+ property_mask_t _boolean_props
144
+ int64_t _memory_range_size
145
+ stride_t _min_offset
146
+ stride_t _max_offset
147
+ stride_t _min_stride
148
+ stride_t _max_stride
149
+ axes_mask_t _unit_extents_mask
150
+ int64_t _volume
151
+
152
+ # Python properties
153
+ tuple _py_shape
154
+ tuple _py_strides
155
+ tuple _py_strides_in_bytes
156
+ tuple _py_stride_order
157
+
158
+ # ==============================
159
+ # Initialization
160
+ # ==============================
161
+
162
+ cdef inline int init_from_ptr(StridedLayout self, int ndim, int itemsize, extent_t* shape, stride_t* strides, bint divide_strides=False) except -1 nogil:
163
+ self._prop_mask = 0
164
+ init_base_layout_from_ptr(self.base, ndim, itemsize, shape, strides, divide_strides)
165
+ if strides == NULL:
166
+ self._init_dense_strides(ORDER_C)
167
+ return 0
168
+
169
+ cdef inline int init_from_tuple(StridedLayout self, int itemsize, tuple shape, tuple strides, bint divide_strides=False) except -1:
170
+ self._prop_mask = 0
171
+ init_base_layout_from_tuple(self.base, itemsize, shape, strides, divide_strides)
172
+ if strides is None:
173
+ self._init_dense_strides(ORDER_C)
174
+ elif divide_strides:
175
+ self._py_strides_in_bytes = strides
176
+ mark_property_valid(self, STRIDES_IN_BYTES)
177
+ else:
178
+ self._py_strides = strides
179
+ mark_property_valid(self, STRIDES)
180
+ self._py_shape = shape
181
+ mark_property_valid(self, SHAPE)
182
+ return 0
183
+
184
+ cdef inline int init_dense_from_ptr(StridedLayout self, int ndim, int itemsize, extent_t* shape, OrderFlag order_flag, axis_vec_t* stride_order=NULL) except -1 nogil:
185
+ self._prop_mask = 0
186
+ init_base_layout_from_ptr(self.base, ndim, itemsize, shape, NULL)
187
+ self._init_dense_strides(order_flag, stride_order)
188
+ return 0
189
+
190
+ cdef inline int init_dense_from_tuple(StridedLayout self, int itemsize, tuple shape, object stride_order) except -1:
191
+ cdef axis_vec_t stride_order_vec
192
+ cdef OrderFlag order_flag = _stride_order2vec(stride_order_vec, stride_order)
193
+
194
+ if order_flag == ORDER_NONE:
195
+ raise ValueError(f"The stride_order must be 'C', 'F', or a permutation tuple. Got: {stride_order}")
196
+
197
+ self._prop_mask = 0
198
+ init_base_layout_from_tuple(self.base, itemsize, shape, None)
199
+ self._init_dense_strides(order_flag, &stride_order_vec)
200
+ self._py_shape = shape
201
+ mark_property_valid(self, SHAPE)
202
+ return 0
203
+
204
+ cdef inline int _init_dense_strides(StridedLayout self, OrderFlag order_flag, axis_vec_t* stride_order=NULL) except -1 nogil:
205
+ cdef stride_t volume
206
+ if order_flag == ORDER_C:
207
+ set_boolean_property(self, IS_CONTIGUOUS_C, True)
208
+ elif order_flag == ORDER_F:
209
+ set_boolean_property(self, IS_CONTIGUOUS_F, True)
210
+ elif order_flag == ORDER_PERM:
211
+ if stride_order == NULL:
212
+ raise ValueError("stride_order is required for ORDER_PERM")
213
+ set_boolean_property(self, IS_CONTIGUOUS_ANY, True)
214
+ else:
215
+ raise ValueError("The stride_order must be 'C', 'F', or a permutation.")
216
+ volume = set_base_dense_strides(self.base, order_flag, stride_order)
217
+ # computing dense strides gives us the volume for free
218
+ self._volume = volume
219
+ mark_property_valid(self, VOLUME)
220
+ return 0
221
+
222
+ cdef inline int make_dense(StridedLayout self, OrderFlag order_flag, axis_vec_t* stride_order=NULL) except -1 nogil:
223
+ if self.get_is_dense(order_flag, stride_order):
224
+ return 0
225
+
226
+ self._prop_mask = 0
227
+
228
+ if order_flag != ORDER_NONE:
229
+ self._init_dense_strides(order_flag, stride_order)
230
+ return 0
231
+
232
+ cdef axis_vec_t current_order_vec
233
+ self.get_stride_order(current_order_vec)
234
+ self._init_dense_strides(ORDER_PERM, &current_order_vec)
235
+ return 0
236
+
237
+ cdef inline StridedLayout copy(StridedLayout self):
238
+ cdef StridedLayout new_layout = StridedLayout.__new__(StridedLayout)
239
+ new_layout.init_from_ptr(self.base.ndim, self.base.itemsize, self.base.shape, self.base.strides)
240
+ return new_layout
241
+
242
+ cdef inline int reset_base(StridedLayout self, BaseLayout& base) except -1 nogil:
243
+ # Reset all memoized properties
244
+ self._prop_mask = 0
245
+
246
+ # Set new base
247
+ _swap_base_layout(self.base, base)
248
+ return 0
249
+
250
+ # ==============================
251
+ # Properties
252
+ # ==============================
253
+
254
+ cdef inline tuple get_shape_tuple(StridedLayout self):
255
+ if not has_valid_property(self, SHAPE):
256
+ self._py_shape = _carray_integer_t_to_tuple(self.base.shape, self.base.ndim)
257
+ mark_property_valid(self, SHAPE)
258
+ return self._py_shape
259
+
260
+ cdef inline tuple get_strides_tuple(StridedLayout self):
261
+ if not has_valid_property(self, STRIDES):
262
+ self._py_strides = _carray_integer_t_to_tuple(self.base.strides, self.base.ndim)
263
+ mark_property_valid(self, STRIDES)
264
+ return self._py_strides
265
+
266
+ cdef inline int get_strides_in_bytes(StridedLayout self, extents_strides_mem_t& strides_in_bytes) except -1 nogil:
267
+ cdef const stride_t* strides = self.base.strides
268
+ strides_in_bytes.resize(self.base.ndim)
269
+ for i in range(self.base.ndim):
270
+ strides_in_bytes[i] = _overflow_checked_mul(strides[i], self.base.itemsize)
271
+ return 0
272
+
273
+ cdef inline tuple get_strides_in_bytes_tuple(StridedLayout self):
274
+ if has_valid_property(self, STRIDES_IN_BYTES):
275
+ return self._py_strides_in_bytes
276
+ cdef extents_strides_mem_t strides_in_bytes
277
+ self.get_strides_in_bytes(strides_in_bytes)
278
+ self._py_strides_in_bytes = _carray_integer_t_to_tuple(strides_in_bytes.data(), strides_in_bytes.size())
279
+ mark_property_valid(self, STRIDES_IN_BYTES)
280
+ return self._py_strides_in_bytes
281
+
282
+ cdef inline int64_t get_volume(StridedLayout self) except -1 nogil:
283
+ if not has_valid_property(self, VOLUME):
284
+ self._volume = base_volume(self.base)
285
+ mark_property_valid(self, VOLUME)
286
+ return self._volume
287
+
288
+ cdef inline int64_t get_len(StridedLayout self) except -1 nogil:
289
+ if self.base.ndim == 0:
290
+ raise TypeError("The layout is a scalar, it has no length.")
291
+ return self.base.shape[0]
292
+
293
+ cdef inline int get_stride_order(StridedLayout self, axis_vec_t& stride_order) except -1 nogil:
294
+ _order_from_strides(stride_order, self.base.shape, self.base.strides, self.base.ndim)
295
+ return 0
296
+
297
+ cdef inline tuple get_stride_order_tuple(StridedLayout self):
298
+ if has_valid_property(self, STRIDE_ORDER):
299
+ return self._py_stride_order
300
+ cdef axis_vec_t stride_order
301
+ self.get_stride_order(stride_order)
302
+ self._py_stride_order = _carray_integer_t_to_tuple(stride_order.data(), stride_order.size())
303
+ mark_property_valid(self, STRIDE_ORDER)
304
+ return self._py_stride_order
305
+
306
+ cdef inline bint get_is_unique(StridedLayout self) except -1 nogil:
307
+ if has_valid_property(self, IS_UNIQUE):
308
+ return boolean_property(self, IS_UNIQUE)
309
+ cdef axis_vec_t stride_order
310
+ self.get_stride_order(stride_order)
311
+ cdef bint is_unique = set_boolean_property(self, IS_UNIQUE, base_is_unique(self.get_volume(), self.base, stride_order))
312
+ if is_unique and peek_boolean_property(self, IS_EXHAUSTIVE):
313
+ set_boolean_property(self, IS_ABS_DENSE_ANY, True)
314
+ return is_unique
315
+
316
+ cdef inline bint get_is_exhaustive(StridedLayout self) except -1 nogil:
317
+ if has_valid_property(self, IS_EXHAUSTIVE):
318
+ return boolean_property(self, IS_EXHAUSTIVE)
319
+ cdef axis_vec_t stride_order
320
+ self.get_stride_order(stride_order)
321
+ cdef bint is_exhaustive = set_boolean_property(self, IS_EXHAUSTIVE, base_is_exhaustive(self.get_volume(), self.base, stride_order))
322
+ if is_exhaustive and peek_boolean_property(self, IS_UNIQUE):
323
+ set_boolean_property(self, IS_ABS_DENSE_ANY, True)
324
+ return is_exhaustive
325
+
326
+ cdef inline bint get_is_contiguous_c(StridedLayout self) except -1 nogil:
327
+ return self.get_is_dense(ORDER_C, NULL)
328
+
329
+ cdef inline bint get_is_contiguous_f(StridedLayout self) except -1 nogil:
330
+ return self.get_is_dense(ORDER_F, NULL)
331
+
332
+ cdef inline bint get_is_contiguous_any(StridedLayout self) except -1 nogil:
333
+ return self.get_is_dense(ORDER_NONE, NULL)
334
+
335
+ cdef inline bint get_is_abs_dense_c(StridedLayout self) except -1 nogil:
336
+ return self.get_is_dense(ORDER_C, NULL, allow_negative_strides=True)
337
+
338
+ cdef inline bint get_is_abs_dense_f(StridedLayout self) except -1 nogil:
339
+ return self.get_is_dense(ORDER_F, NULL, allow_negative_strides=True)
340
+
341
+ cdef inline bint get_is_abs_dense_any(StridedLayout self) except -1 nogil:
342
+ return self.get_is_dense(ORDER_NONE, NULL, allow_negative_strides=True)
343
+
344
+ cdef inline bint get_is_dense(StridedLayout self, OrderFlag order_flag, axis_vec_t* stride_order_vec=NULL, bint allow_negative_strides=False, bint allow_leading_dim_stride=False) except -1 nogil:
345
+ cdef int prop = dense_prop(order_flag, allow_negative_strides, allow_leading_dim_stride)
346
+
347
+ # For arbitrary permutation, we can't use `c`, `f`, `any` cached flags, we need to compute it.
348
+ # But, if the answer is True, we can mark `any` to True.
349
+ if order_flag == ORDER_PERM:
350
+ if base_is_dense(self.get_volume(), self.base, order_flag, stride_order_vec, allow_negative_strides, allow_leading_dim_stride):
351
+ set_boolean_property(self, prop, True)
352
+ return True
353
+ return False
354
+
355
+ if has_valid_property(self, prop):
356
+ return boolean_property(self, prop)
357
+
358
+ cdef axis_vec_t stride_order
359
+ if order_flag == ORDER_NONE:
360
+ self.get_stride_order(stride_order)
361
+ order_flag = ORDER_PERM
362
+ return set_boolean_property(self, prop, base_is_dense(self.get_volume(), self.base, order_flag, &stride_order, allow_negative_strides, allow_leading_dim_stride))
363
+
364
+ cdef inline bint get_has_no_negative_stride(StridedLayout self) except -1 nogil:
365
+ # HAS_NO_NEGATIVE_STRIDE unfortunately has the negation in the name, but
366
+ # it's easier this way to capture implications (dense -> HAS_NO_NEGATIVE_STRIDE)
367
+ # than the (dense -> not HAS_NEGATIVE_STRIDE)
368
+ if has_valid_property(self, HAS_NO_NEGATIVE_STRIDE):
369
+ return boolean_property(self, HAS_NO_NEGATIVE_STRIDE)
370
+ return set_boolean_property(self, HAS_NO_NEGATIVE_STRIDE, base_has_no_negative_stride(self.get_volume(), self.base))
371
+
372
+ cdef inline stride_t get_min_stride(StridedLayout self) except? -1 nogil:
373
+ """
374
+ Minimal stride is the smallest wrt to absolute value (i.e. from -1, 2, -3)
375
+ the minimal is -1.
376
+ """
377
+ if has_valid_property(self, MIN_MAX_STRIDE):
378
+ return self._min_stride
379
+ base_layout_min_max_stride(self.get_volume(), self.base, self._min_stride, self._max_stride)
380
+ mark_property_valid(self, MIN_MAX_STRIDE)
381
+ return self._min_stride
382
+
383
+ cdef inline stride_t get_max_stride(StridedLayout self) except? -1 nogil:
384
+ """
385
+ Maximal stride is the largest wrt to absolute value,
386
+ from -1, 2, -3, the maximal is -3.
387
+ """
388
+ if has_valid_property(self, MIN_MAX_STRIDE):
389
+ return self._max_stride
390
+ base_layout_min_max_stride(self.get_volume(), self.base, self._min_stride, self._max_stride)
391
+ mark_property_valid(self, MIN_MAX_STRIDE)
392
+ return self._max_stride
393
+
394
+ cdef inline axes_mask_t get_unit_extents_mask(StridedLayout self) except -1 nogil:
395
+ if not has_valid_property(self, UNIT_EXTENTS):
396
+ self._unit_extents_mask = base_unit_extents_mask(self.base)
397
+ mark_property_valid(self, UNIT_EXTENTS)
398
+ return self._unit_extents_mask
399
+
400
+ cdef inline bint get_has_stride_order(StridedLayout self, OrderFlag order_flag, axis_vec_t* stride_order_vec=NULL) except -1 nogil:
401
+ if order_flag == ORDER_PERM:
402
+ return base_has_stride_order(self.get_volume(), self.base, ORDER_PERM, stride_order_vec)
403
+
404
+ cdef int prop
405
+ if order_flag == ORDER_C:
406
+ prop = HAS_STRIDE_ORDER_C
407
+ elif order_flag == ORDER_F:
408
+ prop = HAS_STRIDE_ORDER_F
409
+ else:
410
+ raise ValueError("The stride_order must be 'C', 'F', or a permutation.")
411
+ if has_valid_property(self, prop):
412
+ return boolean_property(self, prop)
413
+ return set_boolean_property(self, prop, base_has_stride_order(self.get_volume(), self.base, order_flag, NULL))
414
+
415
+ cdef inline int get_offset_bounds(StridedLayout self, stride_t& min_offset, stride_t& max_offset) except -1 nogil:
416
+ if has_valid_property(self, OFFSET_BOUNDS):
417
+ min_offset = self._min_offset
418
+ max_offset = self._max_offset
419
+ return 0
420
+ if self.get_volume() == 0:
421
+ self._min_offset = min_offset = 0
422
+ self._max_offset = max_offset = -1
423
+ mark_property_valid(self, OFFSET_BOUNDS)
424
+ return 0
425
+ cdef int ndim = self.base.ndim
426
+ cdef stride_t stride
427
+ cdef extent_t extent
428
+ min_offset = 0
429
+ max_offset = 0
430
+ for i in range(ndim):
431
+ stride = self.base.strides[i] # can be negative
432
+ extent = self.base.shape[i] # must be non-negative
433
+ if stride <= 0:
434
+ min_offset = _overflow_checked_sum(min_offset, _overflow_checked_mul(stride, (extent - 1)))
435
+ else:
436
+ max_offset = _overflow_checked_sum(max_offset, _overflow_checked_mul(stride, (extent - 1)))
437
+ self._min_offset = min_offset
438
+ self._max_offset = max_offset
439
+ mark_property_valid(self, OFFSET_BOUNDS)
440
+ return 0
441
+
442
+ cdef inline stride_t get_min_offset(StridedLayout self) except? -1 nogil:
443
+ if has_valid_property(self, OFFSET_BOUNDS):
444
+ return self._min_offset
445
+ cdef stride_t min_offset = 0
446
+ cdef stride_t max_offset = 0
447
+ self.get_offset_bounds(min_offset, max_offset)
448
+ return min_offset
449
+
450
+ cdef inline stride_t get_max_offset(StridedLayout self) except? -1 nogil:
451
+ if has_valid_property(self, OFFSET_BOUNDS):
452
+ return self._max_offset
453
+ cdef stride_t min_offset = 0
454
+ cdef stride_t max_offset = 0
455
+ self.get_offset_bounds(min_offset, max_offset)
456
+ return max_offset
457
+
458
+ cdef inline stride_t get_min_offset_in_bytes(StridedLayout self) except? -1 nogil:
459
+ return _overflow_checked_mul(self.get_min_offset(), self.base.itemsize)
460
+
461
+ cdef inline stride_t get_max_offset_in_bytes(StridedLayout self) except? -1 nogil:
462
+ return _overflow_checked_mul(self.get_max_offset(), self.base.itemsize)
463
+
464
+ cdef inline int64_t get_memory_range_size(StridedLayout self) except? -1 nogil:
465
+ if has_valid_property(self, MEMORY_RANGE_SIZE):
466
+ return self._memory_range_size
467
+ if peek_boolean_property(self, IS_ABS_DENSE_ANY):
468
+ self._memory_range_size = self.get_volume()
469
+ mark_property_valid(self, MEMORY_RANGE_SIZE)
470
+ return self._memory_range_size
471
+ cdef stride_t min_offset = 0
472
+ cdef stride_t max_offset = 0
473
+ self.get_offset_bounds(min_offset, max_offset)
474
+ cdef int64_t memory_range_size = _overflow_checked_diff(max_offset, min_offset)
475
+ self._memory_range_size = _overflow_checked_sum(memory_range_size, 1)
476
+ mark_property_valid(self, MEMORY_RANGE_SIZE)
477
+ return self._memory_range_size
478
+
479
+ cdef inline int64_t get_memory_range_size_in_bytes(StridedLayout self) except? -1 nogil:
480
+ return _overflow_checked_mul(self.get_memory_range_size(), self.base.itemsize)
481
+
482
+ cdef axes_mask_t get_flattened_axis_mask(StridedLayout self) except? -1 nogil
483
+ cdef int get_max_compatible_itemsize(StridedLayout self, int max_itemsize, intptr_t data_ptr, int axis=*) except -1 nogil
484
+
485
+ cdef inline bint get_is_equal(StridedLayout self, StridedLayout other) except -1 nogil:
486
+ if self is other:
487
+ return True
488
+ return base_layout_equal(self.base, other.base)
489
+
490
+ cdef inline bint get_is_almost_equal(StridedLayout self, StridedLayout other) except -1 nogil:
491
+ if self is other:
492
+ return True
493
+ return base_layout_almost_equal(self.base, other.base)
494
+
495
+ # ==============================
496
+ # Layout manipulation
497
+ # ==============================
498
+
499
+ cdef inline StridedLayout get_to_dense(StridedLayout self, OrderFlag order_flag, axis_vec_t* stride_order_vec=NULL):
500
+ if self.get_is_dense(order_flag, stride_order_vec):
501
+ return self
502
+
503
+ cdef axis_vec_t current_order_vec
504
+ if order_flag == ORDER_NONE:
505
+ self.get_stride_order(current_order_vec)
506
+ stride_order_vec = &current_order_vec
507
+ order_flag = ORDER_PERM
508
+
509
+ cdef StridedLayout new_layout = StridedLayout.__new__(StridedLayout)
510
+ new_layout.init_dense_from_ptr(
511
+ self.base.ndim,
512
+ self.base.itemsize,
513
+ self.base.shape,
514
+ order_flag,
515
+ stride_order_vec
516
+ )
517
+ return new_layout
518
+
519
+ cdef inline int to_dense_into(StridedLayout self, StridedLayout out_layout, OrderFlag order_flag, axis_vec_t* stride_order_vec=NULL) except -1 nogil:
520
+ if self is out_layout:
521
+ return self.make_dense(order_flag, stride_order_vec)
522
+
523
+ cdef axis_vec_t current_order_vec
524
+ if order_flag == ORDER_NONE:
525
+ self.get_stride_order(current_order_vec)
526
+ stride_order_vec = &current_order_vec
527
+ order_flag = ORDER_PERM
528
+
529
+ out_layout.init_dense_from_ptr(
530
+ self.base.ndim,
531
+ self.base.itemsize,
532
+ self.base.shape,
533
+ order_flag,
534
+ stride_order_vec
535
+ )
536
+ return 0
537
+
538
+ cdef StridedLayout get_sliced(StridedLayout self, stride_t &slice_offset, object slices)
539
+
540
+ cdef StridedLayout get_concat(StridedLayout self, StridedLayout other, axis_t axis=*)
541
+ cdef int concat_into(StridedLayout self, StridedLayout out_layout, const BaseLayout& other, axis_t axis=*) except -1 nogil
542
+ cdef StridedLayout get_sub(StridedLayout self, axis_t start_axis=*, axis_t end_axis=*, axis_vec_t* axes=*)
543
+ cdef int sub_into(StridedLayout self, StridedLayout out_layout, axis_t start_axis=*, axis_t end_axis=*, axis_vec_t* axes=*) except -1 nogil
544
+ cdef StridedLayout get_reshaped(StridedLayout self, BaseLayout& like)
545
+ cdef int reshape_into(StridedLayout self, StridedLayout out_layout, BaseLayout& like) except -1 nogil
546
+ cdef StridedLayout get_permuted(StridedLayout self, axis_vec_t& axis_order, bint inverse=*)
547
+ cdef int permute_into(StridedLayout self, StridedLayout out_layout, axis_vec_t& axis_order, bint inverse=*) except -1 nogil
548
+ cdef StridedLayout get_reversed(StridedLayout self)
549
+ cdef int reverse_into(StridedLayout self, StridedLayout out_layout) except -1 nogil
550
+ cdef StridedLayout get_transposed(StridedLayout self, axis_t axis_a, axis_t axis_b)
551
+ cdef int transpose_into(StridedLayout self, StridedLayout out_layout, axis_t axis_a, axis_t axis_b) except -1 nogil
552
+ cdef StridedLayout get_flattened(StridedLayout self, axes_mask_t axis_mask)
553
+ cdef int flatten_into(StridedLayout self, StridedLayout out_layout, axes_mask_t axis_mask) except -1 nogil
554
+ cdef StridedLayout get_squeezed(StridedLayout self, axes_mask_t axis_mask=*)
555
+ cdef int squeeze_into(StridedLayout self, StridedLayout out_layout, axes_mask_t axis_mask=*) except -1 nogil
556
+ cdef StridedLayout get_unsqueezed(StridedLayout self, axes_mask_t axis_mask)
557
+ cdef int unsqueeze_into(StridedLayout self, StridedLayout out_layout, axes_mask_t axis_mask) except -1 nogil
558
+ cdef StridedLayout get_broadcast(StridedLayout self, BaseLayout& like)
559
+ cdef int broadcast_into(StridedLayout self, StridedLayout out_layout, BaseLayout& like) except -1 nogil
560
+ cdef StridedLayout get_unbroadcast(StridedLayout self)
561
+ cdef int unbroadcast_into(StridedLayout self, StridedLayout out_layout) except -1 nogil
562
+ cdef StridedLayout get_packed(StridedLayout self, int itemsize, intptr_t data_ptr, int axis=*, bint keep_dim=*)
563
+ cdef int pack_into(StridedLayout self, StridedLayout out_layout, int itemsize, intptr_t data_ptr, int axis=*, bint keep_dim=*) except -1 nogil
564
+ cdef StridedLayout get_unpacked(StridedLayout self, int itemsize, int axis=*, bint add_dim=*)
565
+ cdef int unpack_into(StridedLayout self, StridedLayout out_layout, int itemsize, int axis=*, bint add_dim=*) except -1 nogil
566
+
567
+
568
+ cdef class SlicedLayout:
569
+
570
+ cdef:
571
+ BaseLayout base
572
+
573
+ readonly:
574
+ stride_t slice_offset
575
+
576
+ cdef:
577
+ # Logically, SlicedLayout is StridedLayout + slice_offset,
578
+ # but to keep SlicedLayout overhead minimal
579
+ # (one vs two Python class instantiations for each [] operator call),
580
+ # it is really defined in terms of BaseLayout + slice_offset, and the StridedLayout
581
+ # is instantiated on demand.
582
+ StridedLayout _layout
583
+
584
+ cdef inline int init_from_base(SlicedLayout self, BaseLayout& base, stride_t slice_offset, StridedLayout layout=None) except -1:
585
+ _swap_base_layout(self.base, base)
586
+ self.slice_offset = slice_offset
587
+ self._layout = layout
588
+ return 0
589
+
590
+ cdef inline stride_t get_slice_offset_in_bytes(SlicedLayout self) except? -1 nogil:
591
+ return _overflow_checked_mul(self.slice_offset, self.base.itemsize)
592
+
593
+ cdef StridedLayout get_layout(SlicedLayout self)
594
+ cdef SlicedLayout get_sliced(SlicedLayout self, object slices)
595
+
596
+
597
+ # ==============================
598
+ # IterAxis helpers
599
+ # ==============================
600
+
601
+
602
+ cdef struct IterAxisInCOrder:
603
+ int ndim
604
+
605
+
606
+ cdef struct IterAxisInFOrder:
607
+ int ndim
608
+
609
+
610
+ cdef struct IterAxisInPermOrder:
611
+ int ndim
612
+ axis_t* axis_order
613
+
614
+
615
+ ctypedef fused IterAxisType:
616
+ IterAxisInCOrder
617
+ IterAxisInFOrder
618
+ IterAxisInPermOrder
619
+
620
+
621
+ cdef union IterAxisHelper:
622
+ IterAxisInCOrder c
623
+ IterAxisInFOrder f
624
+ IterAxisInPermOrder perm
625
+
626
+
627
+ cdef inline int _setup_axis_iter(IterAxisHelper& axis_helper, int ndim, OrderFlag order_flag, axis_vec_t* axis_order_vec) except -1 nogil:
628
+ if order_flag == ORDER_C:
629
+ axis_helper.c.ndim = ndim
630
+ return 0
631
+ elif order_flag == ORDER_F:
632
+ axis_helper.f.ndim = ndim
633
+ return 0
634
+ if order_flag != ORDER_PERM:
635
+ raise ValueError(f"Invalid order flag: {order_flag}")
636
+ if axis_order_vec == NULL:
637
+ raise ValueError("order is required for ORDER_PERM")
638
+ cdef int axis_order_size = deref(axis_order_vec).size()
639
+ if ndim != axis_order_size:
640
+ raise ValueError(f"Permutation must have the same length as the number of dimensions, got {axis_order_size} for {ndim}D tensor.")
641
+ cdef axis_t* axis_order = deref(axis_order_vec).data()
642
+ cdef axes_mask_t visited_mask = 0
643
+ cdef axis_t axis
644
+ cdef axes_mask_t axis_mask_i
645
+ for i in range(ndim):
646
+ if not _normalize_axis(axis_order[i], ndim):
647
+ raise ValueError(f"Axis {axis_order[i]} out of range for {ndim}D tensor")
648
+ axis = axis_order[i]
649
+ axis_mask_i = _axis2mask(axis)
650
+ if visited_mask & axis_mask_i:
651
+ raise ValueError(f"Expected a permutation. Axis {axis} appears multiple times.")
652
+ visited_mask |= axis_mask_i
653
+ axis_helper.perm.axis_order = axis_order
654
+ axis_helper.perm.ndim = ndim
655
+ return 0
656
+
657
+
658
+ cdef inline int _get_axis(IterAxisType& axis_helper, int i) except? -2 nogil:
659
+ if IterAxisType is IterAxisInCOrder:
660
+ return i
661
+ elif IterAxisType is IterAxisInFOrder:
662
+ return axis_helper.ndim - 1 - i
663
+
664
+ assert IterAxisType is IterAxisInPermOrder
665
+ return axis_helper.axis_order[i]
666
+
667
+
668
+ # ==============================
669
+ # Base layout helpers
670
+ # ==============================
671
+
672
+
673
+ cdef inline int _validate_itemsize(int itemsize) except -1 nogil:
674
+ if itemsize <= 0:
675
+ raise ValueError("The itemsize must be positive")
676
+ if itemsize & (itemsize - 1):
677
+ raise ValueError("The itemsize must be a power of two")
678
+ return 0
679
+
680
+
681
+ cdef inline int init_base_layout(BaseLayout& layout, int ndim, int itemsize) except -1 nogil:
682
+ if 0 > ndim or ndim > STRIDED_LAYOUT_MAX_NDIM:
683
+ raise ValueError(f"Unsupported number of dimensions: {ndim}. Max supported ndim is {STRIDED_LAYOUT_MAX_NDIM}")
684
+ _validate_itemsize(itemsize)
685
+ layout._mem.resize(2 * ndim)
686
+ layout.shape = layout._mem.data()
687
+ layout.strides = layout._mem.data() + ndim
688
+ layout.ndim = ndim
689
+ layout.itemsize = itemsize
690
+ return 0
691
+
692
+
693
+ cdef inline int trim_base_layout(BaseLayout& layout, int ndim) except -1 nogil:
694
+ if ndim > layout.ndim:
695
+ raise AssertionError(f"Cannot trim layout to {ndim} dimensions, it has {layout.ndim} dimensions")
696
+ layout.ndim = ndim
697
+ return 0
698
+
699
+
700
+ cdef inline int init_base_layout_from_tuple(BaseLayout& base, int itemsize, tuple shape, tuple strides, bint divide_strides=False) except -1:
701
+ cdef int ndim = len(shape)
702
+ init_base_layout(base, ndim, itemsize)
703
+ for i in range(ndim):
704
+ base.shape[i] = shape[i]
705
+ validate_base_shape(base)
706
+
707
+ if strides is not None:
708
+ if len(strides) != ndim:
709
+ raise ValueError(f"Strides, if provided, must have the same length as shape. Shape has {ndim} dimensions, but strides has {len(strides)} elements.")
710
+ for i in range(ndim):
711
+ base.strides[i] = strides[i]
712
+ if divide_strides:
713
+ divide_base_strides(base)
714
+ return 0
715
+
716
+
717
+ cdef inline int init_base_layout_from_ptr(BaseLayout& base, int ndim, int itemsize, extent_t* shape, stride_t* strides, bint divide_strides=False) except -1 nogil:
718
+ init_base_layout(base, ndim, itemsize)
719
+ for i in range(ndim):
720
+ base.shape[i] = shape[i]
721
+ validate_base_shape(base)
722
+
723
+ if strides != NULL:
724
+ for i in range(ndim):
725
+ base.strides[i] = strides[i]
726
+ if divide_strides:
727
+ divide_base_strides(base)
728
+ return 0
729
+
730
+
731
+ cdef inline int validate_base_shape(BaseLayout& base) except -1 nogil:
732
+ for i in range(base.ndim):
733
+ if base.shape[i] < 0:
734
+ raise ValueError("Extents must be non-negative")
735
+ return 0
736
+
737
+
738
+ cdef inline void _swap_base_layout(BaseLayout& a, BaseLayout& b) noexcept nogil:
739
+ if &a == &b:
740
+ return
741
+ _swap(a._mem, b._mem)
742
+ _swap(a.shape, b.shape)
743
+ _swap(a.strides, b.strides)
744
+ _swap(a.ndim, b.ndim)
745
+ _swap(a.itemsize, b.itemsize)
746
+
747
+
748
+ cdef inline bint base_equal_shapes(BaseLayout& a, BaseLayout& b) noexcept nogil:
749
+ if a.ndim != b.ndim:
750
+ return False
751
+ for i in range(a.ndim):
752
+ if a.shape[i] != b.shape[i]:
753
+ return False
754
+ return True
755
+
756
+
757
+ cdef inline bint _base_equal_strides(BaseLayout& a, BaseLayout& b) noexcept nogil:
758
+ cdef stride_t* strides_a = a.strides
759
+ cdef stride_t* strides_b = b.strides
760
+ for i in range(a.ndim):
761
+ if strides_a[i] != strides_b[i]:
762
+ return False
763
+ return True
764
+
765
+
766
+ cdef inline bint base_equal_strides(BaseLayout& a, BaseLayout& b) noexcept nogil:
767
+ return a.ndim == b.ndim and _base_equal_strides(a, b)
768
+
769
+
770
+ cdef inline bint base_layout_equal(BaseLayout& a, BaseLayout& b) noexcept nogil:
771
+ return a.itemsize == b.itemsize and base_equal_shapes(a, b) and _base_equal_strides(a, b)
772
+
773
+
774
+ cdef inline bint base_layout_almost_equal(BaseLayout& a, BaseLayout& b) noexcept nogil:
775
+ if a.itemsize != b.itemsize:
776
+ return False
777
+ cdef int ndim = a.ndim
778
+ if ndim != b.ndim:
779
+ return False
780
+ cdef stride_t* strides_a = a.strides
781
+ cdef stride_t* strides_b = b.strides
782
+ cdef extent_t extent
783
+ for i in range(ndim):
784
+ extent = a.shape[i]
785
+ if b.shape[i] != extent:
786
+ return False
787
+ if extent != 1 and strides_a[i] != strides_b[i]:
788
+ return False
789
+ return True
790
+
791
+
792
+ @cython.overflowcheck(True)
793
+ cdef inline int64_t base_volume(BaseLayout& base) except? -1 nogil:
794
+ cdef int64_t vol = 1
795
+ for i in range(base.ndim):
796
+ vol *= base.shape[i]
797
+ return vol
798
+
799
+
800
+ cdef inline int divide_base_strides(BaseLayout& base) except -1 nogil:
801
+ cdef stride_t stride
802
+ cdef stride_t itemsize = base.itemsize
803
+ for i in range(base.ndim):
804
+ stride = base.strides[i] // itemsize
805
+ if stride * itemsize != base.strides[i]:
806
+ raise ValueError("strides must be divisible by itemsize")
807
+ base.strides[i] = stride
808
+ return 0
809
+
810
+
811
+ cdef inline void zero_out_base_strides(BaseLayout& base) noexcept nogil:
812
+ for i in range(base.ndim):
813
+ base.strides[i] = 0
814
+
815
+
816
+ cdef inline stride_t _set_base_dense_strides(IterAxisType& axis_iter, BaseLayout& base) except? -1 nogil:
817
+ cdef stride_t stride = 1
818
+ cdef int i = base.ndim - 1
819
+ while i >= 0:
820
+ axis = _get_axis(axis_iter, i)
821
+ base.strides[axis] = stride
822
+ stride = _overflow_checked_mul(stride, base.shape[axis])
823
+ i -= 1
824
+ if stride == 0:
825
+ zero_out_base_strides(base)
826
+ return stride
827
+
828
+
829
+ cdef inline stride_t set_base_dense_strides(BaseLayout& base, OrderFlag order_flag, axis_vec_t* stride_order) except? -1 nogil:
830
+ cdef IterAxisHelper axis_iter
831
+ _setup_axis_iter(axis_iter, base.ndim, order_flag, stride_order)
832
+ if order_flag == ORDER_C:
833
+ return _set_base_dense_strides(axis_iter.c, base)
834
+ elif order_flag == ORDER_F:
835
+ return _set_base_dense_strides(axis_iter.f, base)
836
+ elif order_flag == ORDER_PERM:
837
+ return _set_base_dense_strides(axis_iter.perm, base)
838
+ else:
839
+ raise ValueError(f"Invalid order flag: {order_flag}")
840
+
841
+
842
+ cdef inline bint _base_is_dense(IterAxisType& axis_iter, BaseLayout& base, bint allow_negative_strides, bint allow_leading_dim_stride) except -1 nogil:
843
+ cdef int j = base.ndim - 1
844
+ cdef stride_t expected_stride = 1
845
+ cdef stride_t current_stride
846
+ cdef extent_t extent
847
+ cdef axis_t axis
848
+ if allow_leading_dim_stride:
849
+ # Find first non-unit extent
850
+ while j >= 0:
851
+ axis = _get_axis(axis_iter, j)
852
+ if base.shape[axis] != 1:
853
+ expected_stride = base.strides[axis]
854
+ break
855
+ j -= 1
856
+ if expected_stride == 0:
857
+ return False
858
+ elif expected_stride < 0:
859
+ if allow_negative_strides:
860
+ expected_stride = -expected_stride
861
+ else:
862
+ return False
863
+ while j >= 0:
864
+ axis = _get_axis(axis_iter, j)
865
+ extent = base.shape[axis]
866
+ if extent != 1:
867
+ current_stride = base.strides[axis]
868
+ if allow_negative_strides:
869
+ current_stride = c_abs(current_stride)
870
+ if current_stride != expected_stride:
871
+ return False
872
+ expected_stride *= extent
873
+ j -= 1
874
+ return True
875
+
876
+
877
+ cdef inline bint base_is_dense(int64_t volume, BaseLayout& base, OrderFlag order_flag, axis_vec_t* stride_order, bint allow_negative_strides, bint allow_leading_dim_stride) except -1 nogil:
878
+ cdef int ndim = base.ndim
879
+ if volume == 0 or ndim < 1:
880
+ return True
881
+ cdef IterAxisHelper axis_iter
882
+ _setup_axis_iter(axis_iter, ndim, order_flag, stride_order)
883
+ if order_flag == ORDER_C:
884
+ return _base_is_dense(axis_iter.c, base, allow_negative_strides, allow_leading_dim_stride)
885
+ elif order_flag == ORDER_F:
886
+ return _base_is_dense(axis_iter.f, base, allow_negative_strides, allow_leading_dim_stride)
887
+ elif order_flag == ORDER_PERM:
888
+ return _base_is_dense(axis_iter.perm, base, allow_negative_strides, allow_leading_dim_stride)
889
+ else:
890
+ raise ValueError(f"Invalid order flag: {order_flag}")
891
+
892
+
893
+ cdef inline bint base_has_no_negative_stride(int64_t volume, BaseLayout& base) except -1 nogil:
894
+ if volume == 0:
895
+ return True
896
+ for i in range(base.ndim):
897
+ if base.shape[i] != 1 and base.strides[i] < 0:
898
+ return False
899
+ return True
900
+
901
+
902
+ cdef inline int base_layout_min_max_stride(int64_t volume, BaseLayout& base, stride_t& min_stride, stride_t& max_stride) except -1 nogil:
903
+ # There's no meaningful stride for empty or scalar layout.
904
+ # 0 could hint at broadcasting while 1 hints at dense.
905
+ # Chances are, some packages could complain about 0, so we pick 1.
906
+ # For volume 0, maybe more importantly, as we treat it as dense, the
907
+ # max_stride = 1 agrees with the offset_bounds = [0, -1]
908
+ # -1 = max_offset = (volume - 1) * max_stride = (0 - 1) * 1
909
+ min_stride = 1
910
+ max_stride = 1
911
+ if volume <= 1: # empty or effectively a scalar
912
+ return 0
913
+
914
+ cdef int i = 0
915
+ cdef extent_t extent
916
+ cdef stride_t stride
917
+ # find the first non-unit extent
918
+ while i < base.ndim:
919
+ extent = base.shape[i]
920
+ if extent != 1:
921
+ stride = base.strides[i]
922
+ min_stride = stride
923
+ max_stride = stride
924
+ i += 1
925
+ break
926
+ i += 1
927
+
928
+ # find the min and max stride
929
+ while i < base.ndim:
930
+ extent = base.shape[i]
931
+ if extent != 1:
932
+ stride = base.strides[i]
933
+ if c_abs(stride) < c_abs(min_stride):
934
+ min_stride = stride
935
+ if c_abs(stride) > c_abs(max_stride):
936
+ max_stride = stride
937
+ i += 1
938
+ return 0
939
+
940
+
941
+ cdef inline axes_mask_t base_unit_extents_mask(BaseLayout& base) except -1 nogil:
942
+ cdef axes_mask_t unit_extents_mask = 0
943
+ cdef int i = 0
944
+ while i < base.ndim:
945
+ if base.shape[i] == 1:
946
+ unit_extents_mask |= _axis2mask(i)
947
+ i += 1
948
+ return unit_extents_mask
949
+
950
+
951
+ cdef inline bint _base_has_stride_order(IterAxisType& axis_iter, BaseLayout& base) except -1 nogil:
952
+ cdef stride_t prev_stride = 0
953
+ cdef extent_t extent
954
+ cdef stride_t stride
955
+ cdef int i = base.ndim - 1
956
+ cdef axis_t axis
957
+ while i >= 0:
958
+ axis = _get_axis(axis_iter, i)
959
+ extent = base.shape[axis]
960
+ if extent != 1:
961
+ stride = c_abs(base.strides[axis])
962
+ if stride < prev_stride:
963
+ return False
964
+ prev_stride = stride
965
+ i -= 1
966
+ return True
967
+
968
+
969
+ cdef inline bint base_has_stride_order(int64_t volume, BaseLayout& base, OrderFlag order_flag, axis_vec_t* stride_order_vec) except -1 nogil:
970
+ if volume == 0:
971
+ return True
972
+ cdef IterAxisHelper axis_iter
973
+ _setup_axis_iter(axis_iter, base.ndim, order_flag, stride_order_vec)
974
+ if order_flag == ORDER_C:
975
+ return _base_has_stride_order(axis_iter.c, base)
976
+ elif order_flag == ORDER_F:
977
+ return _base_has_stride_order(axis_iter.f, base)
978
+ elif order_flag == ORDER_PERM:
979
+ return _base_has_stride_order(axis_iter.perm, base)
980
+ else:
981
+ raise ValueError(f"Invalid order flag: {order_flag}")
982
+
983
+
984
+ cdef inline bint base_is_unique(int64_t volume, BaseLayout& base, axis_vec_t& stride_order) except -1 nogil:
985
+ """
986
+ Check if every two valid indices map to different memory offsets in the
987
+ range ``[min_offset, max_offset]``.
988
+ Note, ``is_unique() and is_exhaustive() == is_abs_dense_any()``.
989
+
990
+ What do we check here?
991
+ Let's have o = stride_order and say we processed dims ``o[i], ..., o[ndim-1]``
992
+ Indices limited to those dimensions (i.e. index[o[j]] = 0 for j < i)
993
+ map indices uniquely (injective) to ``[cur_min_offset, cur_max_offset]``, where
994
+ ``cur_min_offset = sum((extent[o[j]] - 1) * strides[o[j]] for j in range(i, ndim) if strides[o[j]] < 0)``
995
+ ``cur_max_offset = sum((extent[o[j]] - 1) * strides[o[j]] for j in range(i, ndim) if strides[o[j]] >= 0)``
996
+ Now, if we consider the next dimension (``o[i - 1]``) and its stride ``s_next``,
997
+ we want to make sure that ``idx * s_next + [cur_min_offset, cur_max_offset]``
998
+ and ``(idx + 1) * s_next + [cur_min_offset, cur_max_offset]`` do not overlap.
999
+ We check that:
1000
+ * for ``s_next >= 0``, ``idx * s_next + cur_max_offset < (idx + 1) * s_next + cur_min_offset``
1001
+ * for ``s_next < 0``, ``(idx + 1) * s_next + cur_max_offset < idx * s_next + cur_min_offset``
1002
+ Those two are equivalent to abs(s_next) > cur_max_offset - cur_min_offset.
1003
+ Note, ``cur_max_offset - cur_min_offset = sum((extent[o[j]] - 1) * abs(strides[o[j]]) for j in range(i, ndim))``.
1004
+ """
1005
+ if volume == 0:
1006
+ return True
1007
+ cdef int64_t cur_range = 0
1008
+ cdef int i = base.ndim - 1
1009
+ cdef int64_t stride
1010
+ cdef axis_t axis
1011
+ cdef extent_t extent
1012
+ while i >= 0:
1013
+ axis = stride_order[i]
1014
+ extent = base.shape[axis]
1015
+ if extent != 1:
1016
+ stride = c_abs(base.strides[axis])
1017
+ if stride <= cur_range:
1018
+ return False
1019
+ cur_range = _overflow_checked_sum(cur_range, _overflow_checked_mul(stride, (extent - 1)))
1020
+ i -= 1
1021
+ return True
1022
+
1023
+
1024
+ cdef inline bint base_is_exhaustive(int64_t volume, BaseLayout& base, axis_vec_t& stride_order) except -1 nogil:
1025
+ """
1026
+ Check if there exists an index for every element in the range
1027
+ ``[min_offset, max_offset]``, i.e. the mapping from indices
1028
+ to memory offsets is onto/surjective.
1029
+ Note, ``is_unique() and is_exhaustive() == is_abs_dense_any()``.
1030
+
1031
+ What do we check here?
1032
+ Let's have o = stride_order and say we processed dims ``o[i], ..., o[ndim-1]``
1033
+ Indices limited to those dimensions (i.e. index[o[j]] = 0 for j < i)
1034
+ map indices exhaustively (onto/surjective) to ``[cur_min_offset, cur_max_offset]``, where
1035
+ ``cur_min_offset = sum((extent[o[j]] - 1) * strides[o[j]] for j in range(i, ndim) if strides[o[j]] < 0)``
1036
+ ``cur_max_offset = sum((extent[o[j]] - 1) * strides[o[j]] for j in range(i, ndim) if strides[o[j]] >= 0)``
1037
+ Now, if we consider the next dimension (``o[i - 1]``) and its stride ``s_next``,
1038
+ we want to make sure that ``idx * s_next + [cur_min_offset, cur_max_offset]``
1039
+ and ``(idx + 1) * s_next + [cur_min_offset, cur_max_offset]`` overlap or touch.
1040
+ We check that:
1041
+ * for ``s_next >= 0``, ``idx * s_next + cur_max_offset + 1 >= (idx + 1) * s_next + cur_min_offset``
1042
+ * for ``s_next < 0``, ``(idx + 1) * s_next + cur_max_offset + 1 >= idx * s_next + cur_min_offset``
1043
+ Those two are equivalent to abs(s_next) <= cur_max_offset - cur_min_offset + 1.
1044
+ Note, ``cur_max_offset - cur_min_offset = sum((extent[o[j]] - 1) * abs(strides[o[j]]) for j in range(i, ndim))``.
1045
+ """
1046
+ if volume == 0:
1047
+ return True
1048
+ cdef int64_t cur_range = 1
1049
+ cdef int i = base.ndim - 1
1050
+ cdef int64_t stride
1051
+ cdef axis_t axis
1052
+ cdef extent_t extent
1053
+ while i >= 0:
1054
+ axis = stride_order[i]
1055
+ extent = base.shape[axis]
1056
+ if extent != 1:
1057
+ stride = c_abs(base.strides[axis])
1058
+ if stride > cur_range:
1059
+ return False
1060
+ cur_range = _overflow_checked_sum(cur_range, _overflow_checked_mul(stride, (extent - 1)))
1061
+ i -= 1
1062
+ return True
1063
+
1064
+
1065
+ # ==============================
1066
+ # Strided layout helpers
1067
+ # ==============================
1068
+
1069
+ cdef inline int dense_prop(OrderFlag order_flag, bint allow_negative_strides, bint allow_leading_dim_stride) except -1 nogil:
1070
+ cdef int prop = _dense_prop(order_flag, allow_negative_strides, allow_leading_dim_stride)
1071
+ if prop < 0:
1072
+ raise ValueError(f"Invalid order_flag: {order_flag}")
1073
+ return prop
1074
+
1075
+
1076
+ cdef inline bint has_valid_property(StridedLayout self, int prop) except? -1 nogil:
1077
+ return _has_valid_property(self._prop_mask, prop)
1078
+
1079
+
1080
+ cdef inline int mark_property_valid(StridedLayout self, int prop) except? -1 nogil:
1081
+ _mark_property_valid(self._prop_mask, prop)
1082
+ return 0
1083
+
1084
+
1085
+ cdef inline bint boolean_property(StridedLayout self, int prop) except? -1 nogil:
1086
+ return _boolean_property(self._boolean_props, prop)
1087
+
1088
+
1089
+ cdef inline bint set_boolean_property(StridedLayout self, int prop, bint value) except? -1 nogil:
1090
+ _set_boolean_property(self._prop_mask, self._boolean_props, prop, value)
1091
+ return value
1092
+
1093
+
1094
+ cdef inline bint peek_boolean_property(StridedLayout self, int prop) except? -1 nogil:
1095
+ """
1096
+ True -> the property is True
1097
+ False -> the property is False or not set yet
1098
+ """
1099
+ return _has_valid_property(self._prop_mask, prop) and _boolean_property(self._boolean_props, prop)
1100
+
1101
+
1102
+ cdef inline dict get_boolean_flags(StridedLayout self):
1103
+ cdef dict properties = {
1104
+ "IS_CONTIGUOUS_C": IS_CONTIGUOUS_C,
1105
+ "IS_ABS_DENSE_C": IS_ABS_DENSE_C,
1106
+ "IS_LEADING_DENSE_C": IS_LEADING_DENSE_C,
1107
+ "IS_ABS_LEADING_DENSE_C": IS_ABS_LEADING_DENSE_C,
1108
+
1109
+ "IS_CONTIGUOUS_F": IS_CONTIGUOUS_F,
1110
+ "IS_ABS_DENSE_F": IS_ABS_DENSE_F,
1111
+ "IS_LEADING_DENSE_F": IS_LEADING_DENSE_F,
1112
+ "IS_ABS_LEADING_DENSE_F": IS_ABS_LEADING_DENSE_F,
1113
+
1114
+ "IS_CONTIGUOUS_ANY": IS_CONTIGUOUS_ANY,
1115
+ "IS_ABS_DENSE_ANY": IS_ABS_DENSE_ANY,
1116
+ "IS_LEADING_DENSE_ANY": IS_LEADING_DENSE_ANY,
1117
+ "IS_ABS_LEADING_DENSE_ANY": IS_ABS_LEADING_DENSE_ANY,
1118
+
1119
+ "HAS_STRIDE_ORDER_C": HAS_STRIDE_ORDER_C,
1120
+ "HAS_STRIDE_ORDER_F": HAS_STRIDE_ORDER_F,
1121
+ "IS_UNIQUE": IS_UNIQUE,
1122
+ "IS_EXHAUSTIVE": IS_EXHAUSTIVE,
1123
+ "HAS_NO_NEGATIVE_STRIDE": HAS_NO_NEGATIVE_STRIDE,
1124
+ }
1125
+
1126
+ return {name: boolean_property(self, prop) for name, prop in properties.items() if has_valid_property(self, prop)}
1127
+
1128
+ # ==============================
1129
+ # Conversion, validation and normalization helpers
1130
+ # ==============================
1131
+
1132
+ cdef inline tuple as_tuple(object obj):
1133
+ if obj is None or type(obj) is tuple:
1134
+ return obj
1135
+ elif isinstance(obj, int):
1136
+ return (obj,)
1137
+ else:
1138
+ return tuple(obj)
1139
+
1140
+
1141
+ cdef inline axes_mask_t _axis2mask(axis_t axis) noexcept nogil:
1142
+ return 1ULL << axis
1143
+
1144
+
1145
+ cdef inline axes_mask_t all_axes_mask(int ndim) noexcept nogil:
1146
+ if ndim == 0:
1147
+ return 0
1148
+ elif ndim == STRIDED_LAYOUT_MAX_NDIM:
1149
+ return STRIDED_LAYOUT_AXES_MASK_ALL
1150
+ return (1ULL << ndim) - 1ULL
1151
+
1152
+
1153
+ cdef inline OrderFlag _stride_order2vec(axis_vec_t& stride_order_vec, object stride_order) except? ORDER_NONE:
1154
+ if stride_order == 'C':
1155
+ return ORDER_C
1156
+ elif stride_order == 'F':
1157
+ return ORDER_F
1158
+ elif isinstance(stride_order, tuple | list):
1159
+ _tuple2axis_vec(stride_order_vec, stride_order)
1160
+ return ORDER_PERM
1161
+ return ORDER_NONE
1162
+
1163
+
1164
+ cdef inline int _tuple2axis_vec(axis_vec_t& vec, object t) except -1:
1165
+ cdef int ndim = len(t)
1166
+ vec.resize(ndim)
1167
+ for i in range(ndim):
1168
+ vec[i] = t[i]
1169
+ return 0
1170
+
1171
+
1172
+ cdef inline bint _normalize_axis(integer_t& axis, integer_t extent) except -1 nogil:
1173
+ if axis < -extent or axis >= extent:
1174
+ return False
1175
+ if axis < 0:
1176
+ axis += extent
1177
+ return True
1178
+
1179
+
1180
+ cdef inline axes_mask_t _tuple2axis_mask(int32_t ndim, object t) except? -1:
1181
+ # Returns mask such that mask & _axis2mask(axis) is True iff axis is in t
1182
+ cdef axes_mask_t mask = 0
1183
+ cdef axis_t axis
1184
+ cdef axes_mask_t axis_mask = 0
1185
+ if isinstance(t, int):
1186
+ axis = t
1187
+ if not _normalize_axis(axis, ndim):
1188
+ raise ValueError(f"Invalid axis: {axis} out of range for {ndim}D tensor")
1189
+ return _axis2mask(axis)
1190
+ for e in t:
1191
+ axis = e
1192
+ if not _normalize_axis(axis, ndim):
1193
+ raise ValueError(f"Invalid axis: {axis} out of range for {ndim}D tensor")
1194
+ axis_mask = _axis2mask(axis)
1195
+ if mask & axis_mask:
1196
+ raise ValueError(f"Axis {axis} appears multiple times.")
1197
+ mask |= axis_mask
1198
+ return mask
1199
+
1200
+
1201
+ cdef inline tuple _axes_mask2tuple(axes_mask_t mask):
1202
+ cdef int ndim = _popcount(mask)
1203
+ cdef axis_vec_t axes
1204
+ axes.resize(ndim)
1205
+ cdef int out_i = 0
1206
+ for i in range(STRIDED_LAYOUT_MAX_NDIM):
1207
+ if mask & _axis2mask(i):
1208
+ axes[out_i] = i
1209
+ out_i += 1
1210
+ return _carray_integer_t_to_tuple(axes.data(), ndim)
1211
+
1212
+
1213
+ cdef inline axes_mask_t axes_mask_from_range(int32_t ndim, int32_t start_axis, int32_t end_axis) except? -1 nogil:
1214
+ # Returns mask such that mask & _axis2mask(axis) is True iff axis is
1215
+ # in inclusive range [start_axis, end_axis]
1216
+ if start_axis == 0 and end_axis == -1:
1217
+ return all_axes_mask(ndim)
1218
+ if not _normalize_axis(start_axis, ndim):
1219
+ raise ValueError(f"Invalid start axis: {start_axis} out of range for {ndim}D tensor")
1220
+ if not _normalize_axis(end_axis, ndim):
1221
+ raise ValueError(f"Invalid end axis: {end_axis} out of range for {ndim}D tensor")
1222
+ cdef axes_mask_t axes_mask = all_axes_mask(ndim)
1223
+ axes_mask &= (STRIDED_LAYOUT_AXES_MASK_ALL << start_axis)
1224
+ axes_mask &= (STRIDED_LAYOUT_AXES_MASK_ALL >> (STRIDED_LAYOUT_MAX_NDIM - end_axis - 1))
1225
+ return axes_mask
1226
+
1227
+
1228
+ cdef inline axes_mask_t unsqueeze_to_ndim_mask(int32_t in_ndim, int32_t out_ndim, int32_t axis) except? -1 nogil:
1229
+ if in_ndim > out_ndim:
1230
+ raise ValueError(
1231
+ f"Invalid number of dimensions: {out_ndim}. It must be greater than or equal "
1232
+ f"to the current number of dimensions: {in_ndim}."
1233
+ )
1234
+ elif in_ndim == out_ndim:
1235
+ return 0
1236
+ elif out_ndim > STRIDED_LAYOUT_MAX_NDIM:
1237
+ raise ValueError(
1238
+ f"Invalid number of dimensions: {out_ndim}. It must be less than or equal "
1239
+ f"to {STRIDED_LAYOUT_MAX_NDIM}."
1240
+ )
1241
+ if not _normalize_axis(axis, in_ndim + 1):
1242
+ raise ValueError(f"Invalid axis: {axis} out of range for {in_ndim}D tensor")
1243
+ cdef int num_new_axes = out_ndim - in_ndim
1244
+ cdef axes_mask_t mask = all_axes_mask(axis + num_new_axes)
1245
+ cdef axes_mask_t left_mask = all_axes_mask(axis)
1246
+ return mask & (~left_mask)
1247
+
1248
+
1249
+ cdef inline axes_mask_t flattening_axes_mask_from_range(int ndim, int start_axis, int end_axis) except? -1 nogil:
1250
+ # Returns mask such that mask & _axis2mask(axis) is True iff axis is
1251
+ # in inclusive range [start_axis + 1, end_axis]
1252
+ # (As for flattening, the mask & _axis2mask(axis) means the axis
1253
+ # can be flattened with axis - 1)
1254
+ if start_axis == 0 and end_axis == -1:
1255
+ return all_axes_mask(ndim) & (~_axis2mask(0))
1256
+ if not _normalize_axis(start_axis, ndim):
1257
+ raise ValueError(f"Invalid start axis: {start_axis} out of range for {ndim}D tensor")
1258
+ if not _normalize_axis(end_axis, ndim):
1259
+ raise ValueError(f"Invalid end axis: {end_axis} out of range for {ndim}D tensor")
1260
+ if start_axis == end_axis:
1261
+ return 0
1262
+ cdef axes_mask_t axes_mask = all_axes_mask(ndim)
1263
+ axes_mask &= (STRIDED_LAYOUT_AXES_MASK_ALL << (start_axis + 1))
1264
+ axes_mask &= (STRIDED_LAYOUT_AXES_MASK_ALL >> (STRIDED_LAYOUT_MAX_NDIM - end_axis - 1))
1265
+ return axes_mask
1266
+
1267
+
1268
+ @cython.overflowcheck(True)
1269
+ cdef inline int64_t _overflow_checked_mul(int64_t a, int64_t b) except? -1 nogil:
1270
+ return a * b
1271
+
1272
+
1273
+ @cython.overflowcheck(True)
1274
+ cdef inline int64_t _overflow_checked_diff(int64_t a, int64_t b) except? -1 nogil:
1275
+ return a - b
1276
+
1277
+
1278
+ @cython.overflowcheck(True)
1279
+ cdef inline int64_t _overflow_checked_sum(int64_t a, int64_t b) except? -1 nogil:
1280
+ return a + b
1281
+
1282
+
1283
+ @cython.overflowcheck(True)
1284
+ cdef inline int64_t _overflow_checked_div_ceil(int64_t a, int64_t b) except? -1 nogil:
1285
+ return (a + b - 1) // b
1286
+
1287
+
1288
+ # https://github.com/NVIDIA/cuda-python/blob/main/cuda_core/cuda/core/_utils/cuda_utils.pxd
1289
+
1290
+ # Use Python C API directly instead of cython wrappers, as those end up incorrectly
1291
+ # decrefing the temporary Python int object after it is set as tuple element (resulting
1292
+ # in a double free when the tuple is garbage collected).
1293
+ cdef extern from "Python.h":
1294
+ PyObject *_PyLong_FromLongLong "PyLong_FromLongLong" (long long val) except NULL
1295
+ void _PyTuple_SET_ITEM "PyTuple_SET_ITEM" (object p, Py_ssize_t pos, PyObject *o)
1296
+
1297
+
1298
+ cdef inline tuple _carray_integer_t_to_tuple(const integer_t *ptr, int length):
1299
+ # Construct shape and strides tuples using the Python/C API for speed
1300
+ cdef tuple result = cpython.PyTuple_New(length)
1301
+ for i in range(length):
1302
+ _PyTuple_SET_ITEM(result, i, _PyLong_FromLongLong(ptr[i]))
1303
+ return result