nvmath-python 1.0.0__cp314-cp314t-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nvmath/__init__.pxd +0 -0
- nvmath/__init__.py +45 -0
- nvmath/_internal/__init__.py +0 -0
- nvmath/_internal/attribute_ifc_factory.py +330 -0
- nvmath/_internal/layout.py +70 -0
- nvmath/_internal/templates.py +130 -0
- nvmath/_internal/threadsafe.py +106 -0
- nvmath/_internal/utils.py +43 -0
- nvmath/_internal/workspace.py +490 -0
- nvmath/_utils.py +147 -0
- nvmath/bindings/__init__.py +60 -0
- nvmath/bindings/_internal/__init__.pxd +0 -0
- nvmath/bindings/_internal/__init__.py +0 -0
- nvmath/bindings/_internal/common_types.pxd +31 -0
- nvmath/bindings/_internal/cublas.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/cublas.pxd +530 -0
- nvmath/bindings/_internal/cublasLt.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/cublasLt.pxd +59 -0
- nvmath/bindings/_internal/cublasMp.pxd +52 -0
- nvmath/bindings/_internal/cudss.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/cudss.pxd +54 -0
- nvmath/bindings/_internal/cufft.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/cufft.pxd +70 -0
- nvmath/bindings/_internal/cufftMp.pxd +77 -0
- nvmath/bindings/_internal/curand.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/curand.pxd +42 -0
- nvmath/bindings/_internal/cusolver.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/cusolver.pxd +15 -0
- nvmath/bindings/_internal/cusolverDn.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/cusolverDn.pxd +406 -0
- nvmath/bindings/_internal/cusolverMp.pxd +71 -0
- nvmath/bindings/_internal/cusolverSp.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/cusolverSp.pxd +75 -0
- nvmath/bindings/_internal/cusparse.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/cusparse.pxd +471 -0
- nvmath/bindings/_internal/cusparseLt.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/cusparseLt.pxd +48 -0
- nvmath/bindings/_internal/cutensor.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/cutensor.pxd +58 -0
- nvmath/bindings/_internal/mathdx.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/mathdx.pxd +116 -0
- nvmath/bindings/_internal/nvshmem.pxd +29 -0
- nvmath/bindings/_internal/utils.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/_internal/utils.pxd +174 -0
- nvmath/bindings/_internal/utils.pyi +10 -0
- nvmath/bindings/cublas.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cublas.pxd +558 -0
- nvmath/bindings/cublas.pyi +812 -0
- nvmath/bindings/cublasLt.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cublasLt.pxd +109 -0
- nvmath/bindings/cublasLt.pyi +1461 -0
- nvmath/bindings/cublasMp.pxd +85 -0
- nvmath/bindings/cublasMp.pyi +267 -0
- nvmath/bindings/cudss.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cudss.pxd +98 -0
- nvmath/bindings/cudss.pyi +443 -0
- nvmath/bindings/cufft.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cufft.pxd +118 -0
- nvmath/bindings/cufft.pyi +301 -0
- nvmath/bindings/cufftMp.pxd +124 -0
- nvmath/bindings/cufftMp.pyi +326 -0
- nvmath/bindings/curand.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/curand.pxd +71 -0
- nvmath/bindings/curand.pyi +189 -0
- nvmath/bindings/cusolver.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cusolver.pxd +62 -0
- nvmath/bindings/cusolver.pyi +320 -0
- nvmath/bindings/cusolverDn.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cusolverDn.pxd +430 -0
- nvmath/bindings/cusolverDn.pyi +422 -0
- nvmath/bindings/cusolverMp.pxd +98 -0
- nvmath/bindings/cusolverMp.pyi +114 -0
- nvmath/bindings/cusolverSp.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cusolverSp.pxd +95 -0
- nvmath/bindings/cusolverSp.pyi +70 -0
- nvmath/bindings/cusparse.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cusparse.pxd +546 -0
- nvmath/bindings/cusparse.pyi +1017 -0
- nvmath/bindings/cusparseLt.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cusparseLt.pxd +99 -0
- nvmath/bindings/cusparseLt.pyi +252 -0
- nvmath/bindings/cutensor.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cutensor.pxd +98 -0
- nvmath/bindings/cutensor.pyi +324 -0
- nvmath/bindings/cycublas.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycublas.pxd +664 -0
- nvmath/bindings/cycublasLt.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycublasLt.pxd +1045 -0
- nvmath/bindings/cycublasMp.pxd +171 -0
- nvmath/bindings/cycudss.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycudss.pxd +277 -0
- nvmath/bindings/cycufft.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycufft.pxd +333 -0
- nvmath/bindings/cycufftMp.pxd +342 -0
- nvmath/bindings/cycurand.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycurand.pxd +141 -0
- nvmath/bindings/cycusolver.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycusolver.pxd +137 -0
- nvmath/bindings/cycusolverDn.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycusolverDn.pxd +443 -0
- nvmath/bindings/cycusolverMp.pxd +107 -0
- nvmath/bindings/cycusolverSp.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycusolverSp.pxd +93 -0
- nvmath/bindings/cycusparse.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycusparse.pxd +679 -0
- nvmath/bindings/cycusparseLt.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycusparseLt.pxd +135 -0
- nvmath/bindings/cycutensor.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cycutensor.pxd +189 -0
- nvmath/bindings/cymathdx.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/cymathdx.pxd +552 -0
- nvmath/bindings/cynvshmem.pxd +118 -0
- nvmath/bindings/mathdx.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/mathdx.pxd +182 -0
- nvmath/bindings/mathdx.pyi +1562 -0
- nvmath/bindings/nvpl/__init__.pxd +0 -0
- nvmath/bindings/nvpl/__init__.py +13 -0
- nvmath/bindings/nvpl/_internal/__init__.pxd +0 -0
- nvmath/bindings/nvpl/_internal/__init__.py +0 -0
- nvmath/bindings/nvpl/_internal/blas.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/nvpl/_internal/blas.pxd +237 -0
- nvmath/bindings/nvpl/_internal/fft.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/nvpl/_internal/fft.pxd +36 -0
- nvmath/bindings/nvpl/blas.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/nvpl/blas.pxd +131 -0
- nvmath/bindings/nvpl/blas.pyi +168 -0
- nvmath/bindings/nvpl/cyblas.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/nvpl/cyblas.pxd +280 -0
- nvmath/bindings/nvpl/cyfft.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/nvpl/cyfft.pxd +93 -0
- nvmath/bindings/nvpl/fft.cp314t-win_amd64.pyd +0 -0
- nvmath/bindings/nvpl/fft.pxd +100 -0
- nvmath/bindings/nvpl/fft.pyi +168 -0
- nvmath/bindings/nvshmem.pxd +54 -0
- nvmath/bindings/nvshmem.pyi +191 -0
- nvmath/device/__init__.py +38 -0
- nvmath/device/_deprecated.py +33 -0
- nvmath/device/common.py +315 -0
- nvmath/device/common_backend.py +131 -0
- nvmath/device/common_cuda.py +201 -0
- nvmath/device/common_numba.py +300 -0
- nvmath/device/common_numba_cuda_mlir.py +202 -0
- nvmath/device/common_opaque_tensor.py +201 -0
- nvmath/device/cublasdx.py +1606 -0
- nvmath/device/cublasdx_backend.py +860 -0
- nvmath/device/cublasdx_numba.py +1534 -0
- nvmath/device/cublasdx_numba_cuda_mlir.py +208 -0
- nvmath/device/cufftdx.py +373 -0
- nvmath/device/cufftdx_backend.py +220 -0
- nvmath/device/cufftdx_numba.py +140 -0
- nvmath/device/cufftdx_numba_cuda_mlir.py +79 -0
- nvmath/device/curand_kernel.py +9147 -0
- nvmath/device/cusolverdx.py +2708 -0
- nvmath/device/cusolverdx_backend.py +440 -0
- nvmath/device/cusolverdx_numba.py +567 -0
- nvmath/device/cusolverdx_numba_cuda_mlir.py +604 -0
- nvmath/device/cusolverdx_overload_backend.py +1029 -0
- nvmath/device/llvm_array.py +29 -0
- nvmath/device/random.py +441 -0
- nvmath/device/random_helpers.py +23 -0
- nvmath/device/random_states.py +187 -0
- nvmath/device/types.py +138 -0
- nvmath/device/vector_types_numba.py +259 -0
- nvmath/distributed/__init__.py +200 -0
- nvmath/distributed/_internal/__init__.py +0 -0
- nvmath/distributed/_internal/nccl.py +86 -0
- nvmath/distributed/_internal/nvshmem.py +307 -0
- nvmath/distributed/_internal/symmetric_memory.py +35 -0
- nvmath/distributed/_internal/tensor_ifc.py +70 -0
- nvmath/distributed/_internal/tensor_ifc_cupy.py +68 -0
- nvmath/distributed/_internal/tensor_ifc_host_device.py +172 -0
- nvmath/distributed/_internal/tensor_ifc_numpy.py +46 -0
- nvmath/distributed/_internal/tensor_ifc_torch.py +162 -0
- nvmath/distributed/_internal/tensor_wrapper.py +81 -0
- nvmath/distributed/_utils.py +167 -0
- nvmath/distributed/distribution/__init__.py +30 -0
- nvmath/distributed/distribution/_configuration.py +39 -0
- nvmath/distributed/distribution/distributions.py +1024 -0
- nvmath/distributed/distribution/redistribute.py +1284 -0
- nvmath/distributed/fft/__init__.py +7 -0
- nvmath/distributed/fft/_configuration.py +82 -0
- nvmath/distributed/fft/fft.py +2742 -0
- nvmath/distributed/linalg/__init__.py +22 -0
- nvmath/distributed/linalg/_internal/__init__.py +3 -0
- nvmath/distributed/linalg/_internal/epilog_protocol.py +586 -0
- nvmath/distributed/linalg/_internal/matmul_desc_ifc.py +28 -0
- nvmath/distributed/linalg/advanced/__init__.py +8 -0
- nvmath/distributed/linalg/advanced/_configuration.py +171 -0
- nvmath/distributed/linalg/advanced/matmulmod.py +3573 -0
- nvmath/distributed/linalg/generic/__init__.py +8 -0
- nvmath/distributed/linalg/generic/_caching.py +66 -0
- nvmath/distributed/linalg/generic/_configuration.py +61 -0
- nvmath/distributed/linalg/generic/_factorization.py +172 -0
- nvmath/distributed/linalg/generic/_initialization.py +966 -0
- nvmath/distributed/linalg/generic/_problem_spec.py +511 -0
- nvmath/distributed/linalg/generic/solvermod.py +1368 -0
- nvmath/distributed/process_group.py +408 -0
- nvmath/fft/__init__.py +7 -0
- nvmath/fft/_configuration.py +189 -0
- nvmath/fft/_exec_utils.py +82 -0
- nvmath/fft/_helpers.py +237 -0
- nvmath/fft/fft.py +3122 -0
- nvmath/internal/__init__.pxd +3 -0
- nvmath/internal/__init__.py +10 -0
- nvmath/internal/_bindings.cp314t-win_amd64.pyd +0 -0
- nvmath/internal/_bindings.pxd +18 -0
- nvmath/internal/_device_utils.py +45 -0
- nvmath/internal/_layout/__init__.pxd +3 -0
- nvmath/internal/_layout/__init__.py +7 -0
- nvmath/internal/_layout/_layout.cp314t-win_amd64.pyd +0 -0
- nvmath/internal/_layout/_layout.pxd +1303 -0
- nvmath/internal/_layout/_layout.pyi +1145 -0
- nvmath/internal/enum_utils.py +142 -0
- nvmath/internal/formatters.py +87 -0
- nvmath/internal/mem_limit.py +51 -0
- nvmath/internal/memory.cp314t-win_amd64.pyd +0 -0
- nvmath/internal/memory.pxd +13 -0
- nvmath/internal/memory.pyi +50 -0
- nvmath/internal/ndbuffer/__init__.pxd +3 -0
- nvmath/internal/ndbuffer/__init__.py +9 -0
- nvmath/internal/ndbuffer/_copy_kernel.cp314t-win_amd64.pyd +0 -0
- nvmath/internal/ndbuffer/_copy_kernel.pxd +10 -0
- nvmath/internal/ndbuffer/_jit.cp314t-win_amd64.pyd +0 -0
- nvmath/internal/ndbuffer/_jit.pxd +7 -0
- nvmath/internal/ndbuffer/_ndbuffer.cp314t-win_amd64.pyd +0 -0
- nvmath/internal/ndbuffer/_ndbuffer.pxd +40 -0
- nvmath/internal/ndbuffer/_ndbuffer.pyi +463 -0
- nvmath/internal/ndbuffer/copy_kernel/args.h +34 -0
- nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/array_view.h +52 -0
- nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/elementwise.h +68 -0
- nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/grid_indexer.h +69 -0
- nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/transposed.h +242 -0
- nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/type_utils.h +39 -0
- nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/utils.h +132 -0
- nvmath/internal/ndbuffer/copy_kernel/copy_kernel_impl/vec.h +159 -0
- nvmath/internal/ndbuffer/copy_kernel/elementwise.h +53 -0
- nvmath/internal/ndbuffer/copy_kernel/transposed.h +58 -0
- nvmath/internal/package_ifc.py +168 -0
- nvmath/internal/package_ifc_cuda.py +57 -0
- nvmath/internal/package_ifc_cupy.py +67 -0
- nvmath/internal/package_ifc_torch.py +69 -0
- nvmath/internal/package_wrapper.py +14 -0
- nvmath/internal/tensor_ifc.py +179 -0
- nvmath/internal/tensor_ifc_cupy.py +234 -0
- nvmath/internal/tensor_ifc_ndbuffer.py +147 -0
- nvmath/internal/tensor_ifc_numpy.py +184 -0
- nvmath/internal/tensor_ifc_torch.py +178 -0
- nvmath/internal/tensor_wrapper.py +160 -0
- nvmath/internal/typemaps.py +113 -0
- nvmath/internal/utils.py +805 -0
- nvmath/linalg/__init__.py +56 -0
- nvmath/linalg/_internal/__init__.py +3 -0
- nvmath/linalg/_internal/algo_cap_ifc.py +82 -0
- nvmath/linalg/_internal/algo_config_ifc.py +43 -0
- nvmath/linalg/_internal/batch.py +234 -0
- nvmath/linalg/_internal/enum_to_tuples.py +64 -0
- nvmath/linalg/_internal/epilog_protocol.py +766 -0
- nvmath/linalg/_internal/layout.py +624 -0
- nvmath/linalg/_internal/matmul_desc_ifc.py +28 -0
- nvmath/linalg/_internal/matmul_pref_ifc.py +27 -0
- nvmath/linalg/_internal/matrix_layout_ifc.py +26 -0
- nvmath/linalg/_internal/solver_utils.py +432 -0
- nvmath/linalg/_internal/typemaps.py +144 -0
- nvmath/linalg/_internal/utils.py +157 -0
- nvmath/linalg/advanced/__init__.py +8 -0
- nvmath/linalg/advanced/_algorithmmod.py +170 -0
- nvmath/linalg/advanced/_configuration.py +351 -0
- nvmath/linalg/advanced/helpers/__init__.py +5 -0
- nvmath/linalg/advanced/helpers/matmul.py +1316 -0
- nvmath/linalg/advanced/matmulmod.py +3734 -0
- nvmath/linalg/generic/__init__.py +53 -0
- nvmath/linalg/generic/_configuration/__init__.py +39 -0
- nvmath/linalg/generic/_configuration/layout.py +263 -0
- nvmath/linalg/generic/_configuration/match.py +734 -0
- nvmath/linalg/generic/_configuration/qualifiers.py +493 -0
- nvmath/linalg/generic/_configuration/solver_configuration.py +59 -0
- nvmath/linalg/generic/_configuration/wrap.py +217 -0
- nvmath/linalg/generic/_dtype.py +15 -0
- nvmath/linalg/generic/matmulmod.py +2094 -0
- nvmath/linalg/generic/solvermod.py +1301 -0
- nvmath/memory.py +279 -0
- nvmath/sparse/__init__.py +38 -0
- nvmath/sparse/_internal/__init__.py +21 -0
- nvmath/sparse/_internal/common_utils.py +147 -0
- nvmath/sparse/_internal/cudss_config_ifc.py +702 -0
- nvmath/sparse/_internal/cudss_data_ifc.py +399 -0
- nvmath/sparse/_internal/cudss_utils.py +506 -0
- nvmath/sparse/_internal/cusparse_utils.py +382 -0
- nvmath/sparse/_internal/sparse_bsc_ifc.py +303 -0
- nvmath/sparse/_internal/sparse_bsr_ifc.py +305 -0
- nvmath/sparse/_internal/sparse_coo_ifc.py +256 -0
- nvmath/sparse/_internal/sparse_csc_ifc.py +268 -0
- nvmath/sparse/_internal/sparse_csr_ifc.py +288 -0
- nvmath/sparse/_internal/sparse_dia_ifc.py +242 -0
- nvmath/sparse/_internal/sparse_format_helpers.py +601 -0
- nvmath/sparse/_internal/sparse_tensor_ifc.py +133 -0
- nvmath/sparse/_internal/sparse_ust_ifc.py +141 -0
- nvmath/sparse/_internal/utils.py +56 -0
- nvmath/sparse/advanced/__init__.py +7 -0
- nvmath/sparse/advanced/_configuration.py +227 -0
- nvmath/sparse/advanced/direct_solver.py +2069 -0
- nvmath/sparse/generic/__init__.py +7 -0
- nvmath/sparse/generic/_configuration.py +129 -0
- nvmath/sparse/generic/_helpers.py +137 -0
- nvmath/sparse/generic/_thunks.py +21 -0
- nvmath/sparse/generic/matmulmod.py +2353 -0
- nvmath/sparse/ust/__init__.py +7 -0
- nvmath/sparse/ust/_converters.py +422 -0
- nvmath/sparse/ust/_cpp.py +28 -0
- nvmath/sparse/ust/_drawer.py +565 -0
- nvmath/sparse/ust/_emitter.py +1033 -0
- nvmath/sparse/ust/_jit.py +188 -0
- nvmath/sparse/ust/_kernel.py +282 -0
- nvmath/sparse/ust/_utils.py +149 -0
- nvmath/sparse/ust/interfaces/__init__.py +0 -0
- nvmath/sparse/ust/interfaces/torch_interface.py +476 -0
- nvmath/sparse/ust/tensor.py +1016 -0
- nvmath/sparse/ust/tensor_format.py +957 -0
- nvmath/tensor/__init__.py +6 -0
- nvmath/tensor/_configuration.py +120 -0
- nvmath/tensor/_internal/__init__.py +3 -0
- nvmath/tensor/_internal/cutensor_config_ifc.py +279 -0
- nvmath/tensor/_internal/cutensor_utils.py +230 -0
- nvmath/tensor/_internal/data.py +43 -0
- nvmath/tensor/_internal/einsum_parser.py +444 -0
- nvmath/tensor/_internal/typemaps.py +96 -0
- nvmath/tensor/contract.py +1900 -0
- nvmath_python-1.0.0.dist-info/METADATA +134 -0
- nvmath_python-1.0.0.dist-info/RECORD +332 -0
- nvmath_python-1.0.0.dist-info/WHEEL +5 -0
- nvmath_python-1.0.0.dist-info/licenses/LICENSE +177 -0
- nvmath_python-1.0.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,1303 @@
|
|
|
1
|
+
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
"""
|
|
6
|
+
StridedLayout can be used from Python and Cython code.
|
|
7
|
+
In Python, it feels customary for layout meta-data (shape/strides) to be
|
|
8
|
+
immutable (tuples), so the StridedLayout is immutable too
|
|
9
|
+
(to avoid unexpected modifications of a shared layout).
|
|
10
|
+
All transforming methods (``reshaped``, ``permuted``, etc.)
|
|
11
|
+
return a new instance if the transform is not a no-op.
|
|
12
|
+
|
|
13
|
+
In Cython, to reduce Python overhead (class instantiation, gil acquisition),
|
|
14
|
+
StridedLayout has extra cdef-ed methods (``reshape_into``, ``permute_into``, etc.)
|
|
15
|
+
that accept an instance of StridedLayout to store the result, that
|
|
16
|
+
can be self (in which case it will be modified in place). It is Cython
|
|
17
|
+
user responsibility to make sure externally provided StridedLayout instances
|
|
18
|
+
are not modified in-place if they may be used elsewhere (e.g. by calling
|
|
19
|
+
copy() method at the Cython function entry point).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
cimport cpython
|
|
23
|
+
from cpython.object cimport PyObject
|
|
24
|
+
|
|
25
|
+
cimport cython
|
|
26
|
+
from cython.operator cimport dereference as deref
|
|
27
|
+
|
|
28
|
+
from libc.stdint cimport intptr_t, uint64_t, int64_t, int32_t
|
|
29
|
+
from libcpp cimport vector
|
|
30
|
+
from libcpp.cmath cimport abs as c_abs
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
ctypedef fused integer_t:
|
|
34
|
+
int64_t
|
|
35
|
+
int32_t
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
cdef extern from "../include/_layout.hpp" namespace "nvmath_strided_layout":
|
|
39
|
+
|
|
40
|
+
ctypedef uint64_t axes_mask_t
|
|
41
|
+
ctypedef int64_t extent_t
|
|
42
|
+
ctypedef int64_t stride_t
|
|
43
|
+
ctypedef int32_t axis_t
|
|
44
|
+
ctypedef vector.vector[stride_t] extents_strides_mem_t
|
|
45
|
+
ctypedef vector.vector[axis_t] axis_vec_t
|
|
46
|
+
|
|
47
|
+
int _popcount(axes_mask_t x) nogil
|
|
48
|
+
void _order_from_strides(axis_vec_t& indices, extent_t* extents, stride_t* strides, int ndim) except + nogil
|
|
49
|
+
void _swap(extents_strides_mem_t &a, extents_strides_mem_t &b) noexcept nogil
|
|
50
|
+
void _swap(int64_t*& a, int64_t*& b) noexcept nogil
|
|
51
|
+
void _swap(extent_t& a, extent_t& b) noexcept nogil
|
|
52
|
+
void _swap(int& a, int& b) noexcept nogil
|
|
53
|
+
void _swap(axis_vec_t &a, axis_vec_t &b) noexcept nogil
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
cdef extern from "../include/_layout.hpp":
|
|
57
|
+
cdef int STRIDED_LAYOUT_MAX_NDIM
|
|
58
|
+
cdef axes_mask_t STRIDED_LAYOUT_AXES_MASK_ALL
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
cdef extern from "../include/_layout_properties.hpp" namespace "nvmath_strided_layout":
|
|
62
|
+
ctypedef unsigned int property_mask_t
|
|
63
|
+
|
|
64
|
+
cdef enum OrderFlag:
|
|
65
|
+
ORDER_NONE
|
|
66
|
+
ORDER_C
|
|
67
|
+
ORDER_F
|
|
68
|
+
ORDER_PERM
|
|
69
|
+
|
|
70
|
+
enum BooleanProperty:
|
|
71
|
+
IS_CONTIGUOUS_C
|
|
72
|
+
IS_ABS_DENSE_C
|
|
73
|
+
IS_LEADING_DENSE_C
|
|
74
|
+
IS_ABS_LEADING_DENSE_C
|
|
75
|
+
IS_CONTIGUOUS_F
|
|
76
|
+
IS_ABS_DENSE_F
|
|
77
|
+
IS_LEADING_DENSE_F
|
|
78
|
+
IS_ABS_LEADING_DENSE_F
|
|
79
|
+
IS_CONTIGUOUS_ANY
|
|
80
|
+
IS_ABS_DENSE_ANY
|
|
81
|
+
IS_LEADING_DENSE_ANY
|
|
82
|
+
IS_ABS_LEADING_DENSE_ANY
|
|
83
|
+
HAS_STRIDE_ORDER_C
|
|
84
|
+
HAS_STRIDE_ORDER_F
|
|
85
|
+
IS_UNIQUE
|
|
86
|
+
IS_EXHAUSTIVE
|
|
87
|
+
HAS_NO_NEGATIVE_STRIDE
|
|
88
|
+
|
|
89
|
+
enum Property:
|
|
90
|
+
OFFSET_BOUNDS
|
|
91
|
+
MEMORY_RANGE_SIZE
|
|
92
|
+
SHAPE
|
|
93
|
+
STRIDES
|
|
94
|
+
STRIDES_IN_BYTES
|
|
95
|
+
STRIDE_ORDER
|
|
96
|
+
UNIT_EXTENTS
|
|
97
|
+
VOLUME
|
|
98
|
+
MIN_MAX_STRIDE
|
|
99
|
+
|
|
100
|
+
cdef bint _has_valid_property(property_mask_t has_props, int prop) noexcept nogil
|
|
101
|
+
cdef void _mark_property_valid(property_mask_t& has_props, int prop) noexcept nogil
|
|
102
|
+
cdef bint _boolean_property(property_mask_t bool_props, int prop) noexcept nogil
|
|
103
|
+
cdef void _set_boolean_property(property_mask_t& has_props, property_mask_t& bool_props, int prop, bint value) noexcept nogil
|
|
104
|
+
cdef int _dense_prop(OrderFlag order_flag, int allow_negative_strides, int allow_leading_dim_stride) noexcept nogil
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
cdef struct BaseLayout:
|
|
108
|
+
# A struct holding the shape and strides for the layout.
|
|
109
|
+
# The struct must be initialized before it is passed to any other APIs.
|
|
110
|
+
# Use ``init_base_layout`` for minimal initialization of the layout,
|
|
111
|
+
# it will set the ``shape`` and ``strides`` pointers to point to
|
|
112
|
+
# ndim contiguous integer arrays. After that you are responsible
|
|
113
|
+
# for filling out the ``ndim``, ``shape`` array and ``strides`` array.
|
|
114
|
+
# Alternatively, use ``init_base_layout_from_tuple`` or ``init_base_layout_from_ptr``
|
|
115
|
+
# (followed by zero_out_base_strides or set_base_dense_strides
|
|
116
|
+
# if no explicit strides are provided) to initialize and validate the layout.
|
|
117
|
+
|
|
118
|
+
# Note, extent_t and stride_t are both int64_t and we use a single _mem
|
|
119
|
+
# allocation for both (there's some overhead to small allocations but even
|
|
120
|
+
# more so, Cython adds overhead on handling very unlikely exceptions
|
|
121
|
+
# around the vector instantiation).
|
|
122
|
+
extents_strides_mem_t _mem
|
|
123
|
+
extent_t* shape
|
|
124
|
+
stride_t* strides
|
|
125
|
+
int ndim
|
|
126
|
+
int itemsize
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@cython.final
|
|
130
|
+
cdef class StridedLayout:
|
|
131
|
+
|
|
132
|
+
# Definition
|
|
133
|
+
cdef:
|
|
134
|
+
BaseLayout base
|
|
135
|
+
|
|
136
|
+
# Lazy properties computed from the BaseLayout.
|
|
137
|
+
cdef:
|
|
138
|
+
# Set to 0 to invalidate all properties,
|
|
139
|
+
# whenever a defining attribute is changed
|
|
140
|
+
property_mask_t _prop_mask
|
|
141
|
+
|
|
142
|
+
# C and Python properties
|
|
143
|
+
property_mask_t _boolean_props
|
|
144
|
+
int64_t _memory_range_size
|
|
145
|
+
stride_t _min_offset
|
|
146
|
+
stride_t _max_offset
|
|
147
|
+
stride_t _min_stride
|
|
148
|
+
stride_t _max_stride
|
|
149
|
+
axes_mask_t _unit_extents_mask
|
|
150
|
+
int64_t _volume
|
|
151
|
+
|
|
152
|
+
# Python properties
|
|
153
|
+
tuple _py_shape
|
|
154
|
+
tuple _py_strides
|
|
155
|
+
tuple _py_strides_in_bytes
|
|
156
|
+
tuple _py_stride_order
|
|
157
|
+
|
|
158
|
+
# ==============================
|
|
159
|
+
# Initialization
|
|
160
|
+
# ==============================
|
|
161
|
+
|
|
162
|
+
cdef inline int init_from_ptr(StridedLayout self, int ndim, int itemsize, extent_t* shape, stride_t* strides, bint divide_strides=False) except -1 nogil:
|
|
163
|
+
self._prop_mask = 0
|
|
164
|
+
init_base_layout_from_ptr(self.base, ndim, itemsize, shape, strides, divide_strides)
|
|
165
|
+
if strides == NULL:
|
|
166
|
+
self._init_dense_strides(ORDER_C)
|
|
167
|
+
return 0
|
|
168
|
+
|
|
169
|
+
cdef inline int init_from_tuple(StridedLayout self, int itemsize, tuple shape, tuple strides, bint divide_strides=False) except -1:
|
|
170
|
+
self._prop_mask = 0
|
|
171
|
+
init_base_layout_from_tuple(self.base, itemsize, shape, strides, divide_strides)
|
|
172
|
+
if strides is None:
|
|
173
|
+
self._init_dense_strides(ORDER_C)
|
|
174
|
+
elif divide_strides:
|
|
175
|
+
self._py_strides_in_bytes = strides
|
|
176
|
+
mark_property_valid(self, STRIDES_IN_BYTES)
|
|
177
|
+
else:
|
|
178
|
+
self._py_strides = strides
|
|
179
|
+
mark_property_valid(self, STRIDES)
|
|
180
|
+
self._py_shape = shape
|
|
181
|
+
mark_property_valid(self, SHAPE)
|
|
182
|
+
return 0
|
|
183
|
+
|
|
184
|
+
cdef inline int init_dense_from_ptr(StridedLayout self, int ndim, int itemsize, extent_t* shape, OrderFlag order_flag, axis_vec_t* stride_order=NULL) except -1 nogil:
|
|
185
|
+
self._prop_mask = 0
|
|
186
|
+
init_base_layout_from_ptr(self.base, ndim, itemsize, shape, NULL)
|
|
187
|
+
self._init_dense_strides(order_flag, stride_order)
|
|
188
|
+
return 0
|
|
189
|
+
|
|
190
|
+
cdef inline int init_dense_from_tuple(StridedLayout self, int itemsize, tuple shape, object stride_order) except -1:
|
|
191
|
+
cdef axis_vec_t stride_order_vec
|
|
192
|
+
cdef OrderFlag order_flag = _stride_order2vec(stride_order_vec, stride_order)
|
|
193
|
+
|
|
194
|
+
if order_flag == ORDER_NONE:
|
|
195
|
+
raise ValueError(f"The stride_order must be 'C', 'F', or a permutation tuple. Got: {stride_order}")
|
|
196
|
+
|
|
197
|
+
self._prop_mask = 0
|
|
198
|
+
init_base_layout_from_tuple(self.base, itemsize, shape, None)
|
|
199
|
+
self._init_dense_strides(order_flag, &stride_order_vec)
|
|
200
|
+
self._py_shape = shape
|
|
201
|
+
mark_property_valid(self, SHAPE)
|
|
202
|
+
return 0
|
|
203
|
+
|
|
204
|
+
cdef inline int _init_dense_strides(StridedLayout self, OrderFlag order_flag, axis_vec_t* stride_order=NULL) except -1 nogil:
|
|
205
|
+
cdef stride_t volume
|
|
206
|
+
if order_flag == ORDER_C:
|
|
207
|
+
set_boolean_property(self, IS_CONTIGUOUS_C, True)
|
|
208
|
+
elif order_flag == ORDER_F:
|
|
209
|
+
set_boolean_property(self, IS_CONTIGUOUS_F, True)
|
|
210
|
+
elif order_flag == ORDER_PERM:
|
|
211
|
+
if stride_order == NULL:
|
|
212
|
+
raise ValueError("stride_order is required for ORDER_PERM")
|
|
213
|
+
set_boolean_property(self, IS_CONTIGUOUS_ANY, True)
|
|
214
|
+
else:
|
|
215
|
+
raise ValueError("The stride_order must be 'C', 'F', or a permutation.")
|
|
216
|
+
volume = set_base_dense_strides(self.base, order_flag, stride_order)
|
|
217
|
+
# computing dense strides gives us the volume for free
|
|
218
|
+
self._volume = volume
|
|
219
|
+
mark_property_valid(self, VOLUME)
|
|
220
|
+
return 0
|
|
221
|
+
|
|
222
|
+
cdef inline int make_dense(StridedLayout self, OrderFlag order_flag, axis_vec_t* stride_order=NULL) except -1 nogil:
|
|
223
|
+
if self.get_is_dense(order_flag, stride_order):
|
|
224
|
+
return 0
|
|
225
|
+
|
|
226
|
+
self._prop_mask = 0
|
|
227
|
+
|
|
228
|
+
if order_flag != ORDER_NONE:
|
|
229
|
+
self._init_dense_strides(order_flag, stride_order)
|
|
230
|
+
return 0
|
|
231
|
+
|
|
232
|
+
cdef axis_vec_t current_order_vec
|
|
233
|
+
self.get_stride_order(current_order_vec)
|
|
234
|
+
self._init_dense_strides(ORDER_PERM, ¤t_order_vec)
|
|
235
|
+
return 0
|
|
236
|
+
|
|
237
|
+
cdef inline StridedLayout copy(StridedLayout self):
|
|
238
|
+
cdef StridedLayout new_layout = StridedLayout.__new__(StridedLayout)
|
|
239
|
+
new_layout.init_from_ptr(self.base.ndim, self.base.itemsize, self.base.shape, self.base.strides)
|
|
240
|
+
return new_layout
|
|
241
|
+
|
|
242
|
+
cdef inline int reset_base(StridedLayout self, BaseLayout& base) except -1 nogil:
|
|
243
|
+
# Reset all memoized properties
|
|
244
|
+
self._prop_mask = 0
|
|
245
|
+
|
|
246
|
+
# Set new base
|
|
247
|
+
_swap_base_layout(self.base, base)
|
|
248
|
+
return 0
|
|
249
|
+
|
|
250
|
+
# ==============================
|
|
251
|
+
# Properties
|
|
252
|
+
# ==============================
|
|
253
|
+
|
|
254
|
+
cdef inline tuple get_shape_tuple(StridedLayout self):
|
|
255
|
+
if not has_valid_property(self, SHAPE):
|
|
256
|
+
self._py_shape = _carray_integer_t_to_tuple(self.base.shape, self.base.ndim)
|
|
257
|
+
mark_property_valid(self, SHAPE)
|
|
258
|
+
return self._py_shape
|
|
259
|
+
|
|
260
|
+
cdef inline tuple get_strides_tuple(StridedLayout self):
|
|
261
|
+
if not has_valid_property(self, STRIDES):
|
|
262
|
+
self._py_strides = _carray_integer_t_to_tuple(self.base.strides, self.base.ndim)
|
|
263
|
+
mark_property_valid(self, STRIDES)
|
|
264
|
+
return self._py_strides
|
|
265
|
+
|
|
266
|
+
cdef inline int get_strides_in_bytes(StridedLayout self, extents_strides_mem_t& strides_in_bytes) except -1 nogil:
|
|
267
|
+
cdef const stride_t* strides = self.base.strides
|
|
268
|
+
strides_in_bytes.resize(self.base.ndim)
|
|
269
|
+
for i in range(self.base.ndim):
|
|
270
|
+
strides_in_bytes[i] = _overflow_checked_mul(strides[i], self.base.itemsize)
|
|
271
|
+
return 0
|
|
272
|
+
|
|
273
|
+
cdef inline tuple get_strides_in_bytes_tuple(StridedLayout self):
|
|
274
|
+
if has_valid_property(self, STRIDES_IN_BYTES):
|
|
275
|
+
return self._py_strides_in_bytes
|
|
276
|
+
cdef extents_strides_mem_t strides_in_bytes
|
|
277
|
+
self.get_strides_in_bytes(strides_in_bytes)
|
|
278
|
+
self._py_strides_in_bytes = _carray_integer_t_to_tuple(strides_in_bytes.data(), strides_in_bytes.size())
|
|
279
|
+
mark_property_valid(self, STRIDES_IN_BYTES)
|
|
280
|
+
return self._py_strides_in_bytes
|
|
281
|
+
|
|
282
|
+
cdef inline int64_t get_volume(StridedLayout self) except -1 nogil:
|
|
283
|
+
if not has_valid_property(self, VOLUME):
|
|
284
|
+
self._volume = base_volume(self.base)
|
|
285
|
+
mark_property_valid(self, VOLUME)
|
|
286
|
+
return self._volume
|
|
287
|
+
|
|
288
|
+
cdef inline int64_t get_len(StridedLayout self) except -1 nogil:
|
|
289
|
+
if self.base.ndim == 0:
|
|
290
|
+
raise TypeError("The layout is a scalar, it has no length.")
|
|
291
|
+
return self.base.shape[0]
|
|
292
|
+
|
|
293
|
+
cdef inline int get_stride_order(StridedLayout self, axis_vec_t& stride_order) except -1 nogil:
|
|
294
|
+
_order_from_strides(stride_order, self.base.shape, self.base.strides, self.base.ndim)
|
|
295
|
+
return 0
|
|
296
|
+
|
|
297
|
+
cdef inline tuple get_stride_order_tuple(StridedLayout self):
|
|
298
|
+
if has_valid_property(self, STRIDE_ORDER):
|
|
299
|
+
return self._py_stride_order
|
|
300
|
+
cdef axis_vec_t stride_order
|
|
301
|
+
self.get_stride_order(stride_order)
|
|
302
|
+
self._py_stride_order = _carray_integer_t_to_tuple(stride_order.data(), stride_order.size())
|
|
303
|
+
mark_property_valid(self, STRIDE_ORDER)
|
|
304
|
+
return self._py_stride_order
|
|
305
|
+
|
|
306
|
+
cdef inline bint get_is_unique(StridedLayout self) except -1 nogil:
|
|
307
|
+
if has_valid_property(self, IS_UNIQUE):
|
|
308
|
+
return boolean_property(self, IS_UNIQUE)
|
|
309
|
+
cdef axis_vec_t stride_order
|
|
310
|
+
self.get_stride_order(stride_order)
|
|
311
|
+
cdef bint is_unique = set_boolean_property(self, IS_UNIQUE, base_is_unique(self.get_volume(), self.base, stride_order))
|
|
312
|
+
if is_unique and peek_boolean_property(self, IS_EXHAUSTIVE):
|
|
313
|
+
set_boolean_property(self, IS_ABS_DENSE_ANY, True)
|
|
314
|
+
return is_unique
|
|
315
|
+
|
|
316
|
+
cdef inline bint get_is_exhaustive(StridedLayout self) except -1 nogil:
|
|
317
|
+
if has_valid_property(self, IS_EXHAUSTIVE):
|
|
318
|
+
return boolean_property(self, IS_EXHAUSTIVE)
|
|
319
|
+
cdef axis_vec_t stride_order
|
|
320
|
+
self.get_stride_order(stride_order)
|
|
321
|
+
cdef bint is_exhaustive = set_boolean_property(self, IS_EXHAUSTIVE, base_is_exhaustive(self.get_volume(), self.base, stride_order))
|
|
322
|
+
if is_exhaustive and peek_boolean_property(self, IS_UNIQUE):
|
|
323
|
+
set_boolean_property(self, IS_ABS_DENSE_ANY, True)
|
|
324
|
+
return is_exhaustive
|
|
325
|
+
|
|
326
|
+
cdef inline bint get_is_contiguous_c(StridedLayout self) except -1 nogil:
|
|
327
|
+
return self.get_is_dense(ORDER_C, NULL)
|
|
328
|
+
|
|
329
|
+
cdef inline bint get_is_contiguous_f(StridedLayout self) except -1 nogil:
|
|
330
|
+
return self.get_is_dense(ORDER_F, NULL)
|
|
331
|
+
|
|
332
|
+
cdef inline bint get_is_contiguous_any(StridedLayout self) except -1 nogil:
|
|
333
|
+
return self.get_is_dense(ORDER_NONE, NULL)
|
|
334
|
+
|
|
335
|
+
cdef inline bint get_is_abs_dense_c(StridedLayout self) except -1 nogil:
|
|
336
|
+
return self.get_is_dense(ORDER_C, NULL, allow_negative_strides=True)
|
|
337
|
+
|
|
338
|
+
cdef inline bint get_is_abs_dense_f(StridedLayout self) except -1 nogil:
|
|
339
|
+
return self.get_is_dense(ORDER_F, NULL, allow_negative_strides=True)
|
|
340
|
+
|
|
341
|
+
cdef inline bint get_is_abs_dense_any(StridedLayout self) except -1 nogil:
|
|
342
|
+
return self.get_is_dense(ORDER_NONE, NULL, allow_negative_strides=True)
|
|
343
|
+
|
|
344
|
+
cdef inline bint get_is_dense(StridedLayout self, OrderFlag order_flag, axis_vec_t* stride_order_vec=NULL, bint allow_negative_strides=False, bint allow_leading_dim_stride=False) except -1 nogil:
|
|
345
|
+
cdef int prop = dense_prop(order_flag, allow_negative_strides, allow_leading_dim_stride)
|
|
346
|
+
|
|
347
|
+
# For arbitrary permutation, we can't use `c`, `f`, `any` cached flags, we need to compute it.
|
|
348
|
+
# But, if the answer is True, we can mark `any` to True.
|
|
349
|
+
if order_flag == ORDER_PERM:
|
|
350
|
+
if base_is_dense(self.get_volume(), self.base, order_flag, stride_order_vec, allow_negative_strides, allow_leading_dim_stride):
|
|
351
|
+
set_boolean_property(self, prop, True)
|
|
352
|
+
return True
|
|
353
|
+
return False
|
|
354
|
+
|
|
355
|
+
if has_valid_property(self, prop):
|
|
356
|
+
return boolean_property(self, prop)
|
|
357
|
+
|
|
358
|
+
cdef axis_vec_t stride_order
|
|
359
|
+
if order_flag == ORDER_NONE:
|
|
360
|
+
self.get_stride_order(stride_order)
|
|
361
|
+
order_flag = ORDER_PERM
|
|
362
|
+
return set_boolean_property(self, prop, base_is_dense(self.get_volume(), self.base, order_flag, &stride_order, allow_negative_strides, allow_leading_dim_stride))
|
|
363
|
+
|
|
364
|
+
cdef inline bint get_has_no_negative_stride(StridedLayout self) except -1 nogil:
|
|
365
|
+
# HAS_NO_NEGATIVE_STRIDE unfortunately has the negation in the name, but
|
|
366
|
+
# it's easier this way to capture implications (dense -> HAS_NO_NEGATIVE_STRIDE)
|
|
367
|
+
# than the (dense -> not HAS_NEGATIVE_STRIDE)
|
|
368
|
+
if has_valid_property(self, HAS_NO_NEGATIVE_STRIDE):
|
|
369
|
+
return boolean_property(self, HAS_NO_NEGATIVE_STRIDE)
|
|
370
|
+
return set_boolean_property(self, HAS_NO_NEGATIVE_STRIDE, base_has_no_negative_stride(self.get_volume(), self.base))
|
|
371
|
+
|
|
372
|
+
cdef inline stride_t get_min_stride(StridedLayout self) except? -1 nogil:
|
|
373
|
+
"""
|
|
374
|
+
Minimal stride is the smallest wrt to absolute value (i.e. from -1, 2, -3)
|
|
375
|
+
the minimal is -1.
|
|
376
|
+
"""
|
|
377
|
+
if has_valid_property(self, MIN_MAX_STRIDE):
|
|
378
|
+
return self._min_stride
|
|
379
|
+
base_layout_min_max_stride(self.get_volume(), self.base, self._min_stride, self._max_stride)
|
|
380
|
+
mark_property_valid(self, MIN_MAX_STRIDE)
|
|
381
|
+
return self._min_stride
|
|
382
|
+
|
|
383
|
+
cdef inline stride_t get_max_stride(StridedLayout self) except? -1 nogil:
|
|
384
|
+
"""
|
|
385
|
+
Maximal stride is the largest wrt to absolute value,
|
|
386
|
+
from -1, 2, -3, the maximal is -3.
|
|
387
|
+
"""
|
|
388
|
+
if has_valid_property(self, MIN_MAX_STRIDE):
|
|
389
|
+
return self._max_stride
|
|
390
|
+
base_layout_min_max_stride(self.get_volume(), self.base, self._min_stride, self._max_stride)
|
|
391
|
+
mark_property_valid(self, MIN_MAX_STRIDE)
|
|
392
|
+
return self._max_stride
|
|
393
|
+
|
|
394
|
+
cdef inline axes_mask_t get_unit_extents_mask(StridedLayout self) except -1 nogil:
|
|
395
|
+
if not has_valid_property(self, UNIT_EXTENTS):
|
|
396
|
+
self._unit_extents_mask = base_unit_extents_mask(self.base)
|
|
397
|
+
mark_property_valid(self, UNIT_EXTENTS)
|
|
398
|
+
return self._unit_extents_mask
|
|
399
|
+
|
|
400
|
+
cdef inline bint get_has_stride_order(StridedLayout self, OrderFlag order_flag, axis_vec_t* stride_order_vec=NULL) except -1 nogil:
|
|
401
|
+
if order_flag == ORDER_PERM:
|
|
402
|
+
return base_has_stride_order(self.get_volume(), self.base, ORDER_PERM, stride_order_vec)
|
|
403
|
+
|
|
404
|
+
cdef int prop
|
|
405
|
+
if order_flag == ORDER_C:
|
|
406
|
+
prop = HAS_STRIDE_ORDER_C
|
|
407
|
+
elif order_flag == ORDER_F:
|
|
408
|
+
prop = HAS_STRIDE_ORDER_F
|
|
409
|
+
else:
|
|
410
|
+
raise ValueError("The stride_order must be 'C', 'F', or a permutation.")
|
|
411
|
+
if has_valid_property(self, prop):
|
|
412
|
+
return boolean_property(self, prop)
|
|
413
|
+
return set_boolean_property(self, prop, base_has_stride_order(self.get_volume(), self.base, order_flag, NULL))
|
|
414
|
+
|
|
415
|
+
cdef inline int get_offset_bounds(StridedLayout self, stride_t& min_offset, stride_t& max_offset) except -1 nogil:
|
|
416
|
+
if has_valid_property(self, OFFSET_BOUNDS):
|
|
417
|
+
min_offset = self._min_offset
|
|
418
|
+
max_offset = self._max_offset
|
|
419
|
+
return 0
|
|
420
|
+
if self.get_volume() == 0:
|
|
421
|
+
self._min_offset = min_offset = 0
|
|
422
|
+
self._max_offset = max_offset = -1
|
|
423
|
+
mark_property_valid(self, OFFSET_BOUNDS)
|
|
424
|
+
return 0
|
|
425
|
+
cdef int ndim = self.base.ndim
|
|
426
|
+
cdef stride_t stride
|
|
427
|
+
cdef extent_t extent
|
|
428
|
+
min_offset = 0
|
|
429
|
+
max_offset = 0
|
|
430
|
+
for i in range(ndim):
|
|
431
|
+
stride = self.base.strides[i] # can be negative
|
|
432
|
+
extent = self.base.shape[i] # must be non-negative
|
|
433
|
+
if stride <= 0:
|
|
434
|
+
min_offset = _overflow_checked_sum(min_offset, _overflow_checked_mul(stride, (extent - 1)))
|
|
435
|
+
else:
|
|
436
|
+
max_offset = _overflow_checked_sum(max_offset, _overflow_checked_mul(stride, (extent - 1)))
|
|
437
|
+
self._min_offset = min_offset
|
|
438
|
+
self._max_offset = max_offset
|
|
439
|
+
mark_property_valid(self, OFFSET_BOUNDS)
|
|
440
|
+
return 0
|
|
441
|
+
|
|
442
|
+
cdef inline stride_t get_min_offset(StridedLayout self) except? -1 nogil:
|
|
443
|
+
if has_valid_property(self, OFFSET_BOUNDS):
|
|
444
|
+
return self._min_offset
|
|
445
|
+
cdef stride_t min_offset = 0
|
|
446
|
+
cdef stride_t max_offset = 0
|
|
447
|
+
self.get_offset_bounds(min_offset, max_offset)
|
|
448
|
+
return min_offset
|
|
449
|
+
|
|
450
|
+
cdef inline stride_t get_max_offset(StridedLayout self) except? -1 nogil:
|
|
451
|
+
if has_valid_property(self, OFFSET_BOUNDS):
|
|
452
|
+
return self._max_offset
|
|
453
|
+
cdef stride_t min_offset = 0
|
|
454
|
+
cdef stride_t max_offset = 0
|
|
455
|
+
self.get_offset_bounds(min_offset, max_offset)
|
|
456
|
+
return max_offset
|
|
457
|
+
|
|
458
|
+
cdef inline stride_t get_min_offset_in_bytes(StridedLayout self) except? -1 nogil:
|
|
459
|
+
return _overflow_checked_mul(self.get_min_offset(), self.base.itemsize)
|
|
460
|
+
|
|
461
|
+
cdef inline stride_t get_max_offset_in_bytes(StridedLayout self) except? -1 nogil:
|
|
462
|
+
return _overflow_checked_mul(self.get_max_offset(), self.base.itemsize)
|
|
463
|
+
|
|
464
|
+
cdef inline int64_t get_memory_range_size(StridedLayout self) except? -1 nogil:
|
|
465
|
+
if has_valid_property(self, MEMORY_RANGE_SIZE):
|
|
466
|
+
return self._memory_range_size
|
|
467
|
+
if peek_boolean_property(self, IS_ABS_DENSE_ANY):
|
|
468
|
+
self._memory_range_size = self.get_volume()
|
|
469
|
+
mark_property_valid(self, MEMORY_RANGE_SIZE)
|
|
470
|
+
return self._memory_range_size
|
|
471
|
+
cdef stride_t min_offset = 0
|
|
472
|
+
cdef stride_t max_offset = 0
|
|
473
|
+
self.get_offset_bounds(min_offset, max_offset)
|
|
474
|
+
cdef int64_t memory_range_size = _overflow_checked_diff(max_offset, min_offset)
|
|
475
|
+
self._memory_range_size = _overflow_checked_sum(memory_range_size, 1)
|
|
476
|
+
mark_property_valid(self, MEMORY_RANGE_SIZE)
|
|
477
|
+
return self._memory_range_size
|
|
478
|
+
|
|
479
|
+
cdef inline int64_t get_memory_range_size_in_bytes(StridedLayout self) except? -1 nogil:
|
|
480
|
+
return _overflow_checked_mul(self.get_memory_range_size(), self.base.itemsize)
|
|
481
|
+
|
|
482
|
+
cdef axes_mask_t get_flattened_axis_mask(StridedLayout self) except? -1 nogil
|
|
483
|
+
cdef int get_max_compatible_itemsize(StridedLayout self, int max_itemsize, intptr_t data_ptr, int axis=*) except -1 nogil
|
|
484
|
+
|
|
485
|
+
cdef inline bint get_is_equal(StridedLayout self, StridedLayout other) except -1 nogil:
|
|
486
|
+
if self is other:
|
|
487
|
+
return True
|
|
488
|
+
return base_layout_equal(self.base, other.base)
|
|
489
|
+
|
|
490
|
+
cdef inline bint get_is_almost_equal(StridedLayout self, StridedLayout other) except -1 nogil:
|
|
491
|
+
if self is other:
|
|
492
|
+
return True
|
|
493
|
+
return base_layout_almost_equal(self.base, other.base)
|
|
494
|
+
|
|
495
|
+
# ==============================
|
|
496
|
+
# Layout manipulation
|
|
497
|
+
# ==============================
|
|
498
|
+
|
|
499
|
+
cdef inline StridedLayout get_to_dense(StridedLayout self, OrderFlag order_flag, axis_vec_t* stride_order_vec=NULL):
|
|
500
|
+
if self.get_is_dense(order_flag, stride_order_vec):
|
|
501
|
+
return self
|
|
502
|
+
|
|
503
|
+
cdef axis_vec_t current_order_vec
|
|
504
|
+
if order_flag == ORDER_NONE:
|
|
505
|
+
self.get_stride_order(current_order_vec)
|
|
506
|
+
stride_order_vec = ¤t_order_vec
|
|
507
|
+
order_flag = ORDER_PERM
|
|
508
|
+
|
|
509
|
+
cdef StridedLayout new_layout = StridedLayout.__new__(StridedLayout)
|
|
510
|
+
new_layout.init_dense_from_ptr(
|
|
511
|
+
self.base.ndim,
|
|
512
|
+
self.base.itemsize,
|
|
513
|
+
self.base.shape,
|
|
514
|
+
order_flag,
|
|
515
|
+
stride_order_vec
|
|
516
|
+
)
|
|
517
|
+
return new_layout
|
|
518
|
+
|
|
519
|
+
cdef inline int to_dense_into(StridedLayout self, StridedLayout out_layout, OrderFlag order_flag, axis_vec_t* stride_order_vec=NULL) except -1 nogil:
|
|
520
|
+
if self is out_layout:
|
|
521
|
+
return self.make_dense(order_flag, stride_order_vec)
|
|
522
|
+
|
|
523
|
+
cdef axis_vec_t current_order_vec
|
|
524
|
+
if order_flag == ORDER_NONE:
|
|
525
|
+
self.get_stride_order(current_order_vec)
|
|
526
|
+
stride_order_vec = ¤t_order_vec
|
|
527
|
+
order_flag = ORDER_PERM
|
|
528
|
+
|
|
529
|
+
out_layout.init_dense_from_ptr(
|
|
530
|
+
self.base.ndim,
|
|
531
|
+
self.base.itemsize,
|
|
532
|
+
self.base.shape,
|
|
533
|
+
order_flag,
|
|
534
|
+
stride_order_vec
|
|
535
|
+
)
|
|
536
|
+
return 0
|
|
537
|
+
|
|
538
|
+
cdef StridedLayout get_sliced(StridedLayout self, stride_t &slice_offset, object slices)
|
|
539
|
+
|
|
540
|
+
cdef StridedLayout get_concat(StridedLayout self, StridedLayout other, axis_t axis=*)
|
|
541
|
+
cdef int concat_into(StridedLayout self, StridedLayout out_layout, const BaseLayout& other, axis_t axis=*) except -1 nogil
|
|
542
|
+
cdef StridedLayout get_sub(StridedLayout self, axis_t start_axis=*, axis_t end_axis=*, axis_vec_t* axes=*)
|
|
543
|
+
cdef int sub_into(StridedLayout self, StridedLayout out_layout, axis_t start_axis=*, axis_t end_axis=*, axis_vec_t* axes=*) except -1 nogil
|
|
544
|
+
cdef StridedLayout get_reshaped(StridedLayout self, BaseLayout& like)
|
|
545
|
+
cdef int reshape_into(StridedLayout self, StridedLayout out_layout, BaseLayout& like) except -1 nogil
|
|
546
|
+
cdef StridedLayout get_permuted(StridedLayout self, axis_vec_t& axis_order, bint inverse=*)
|
|
547
|
+
cdef int permute_into(StridedLayout self, StridedLayout out_layout, axis_vec_t& axis_order, bint inverse=*) except -1 nogil
|
|
548
|
+
cdef StridedLayout get_reversed(StridedLayout self)
|
|
549
|
+
cdef int reverse_into(StridedLayout self, StridedLayout out_layout) except -1 nogil
|
|
550
|
+
cdef StridedLayout get_transposed(StridedLayout self, axis_t axis_a, axis_t axis_b)
|
|
551
|
+
cdef int transpose_into(StridedLayout self, StridedLayout out_layout, axis_t axis_a, axis_t axis_b) except -1 nogil
|
|
552
|
+
cdef StridedLayout get_flattened(StridedLayout self, axes_mask_t axis_mask)
|
|
553
|
+
cdef int flatten_into(StridedLayout self, StridedLayout out_layout, axes_mask_t axis_mask) except -1 nogil
|
|
554
|
+
cdef StridedLayout get_squeezed(StridedLayout self, axes_mask_t axis_mask=*)
|
|
555
|
+
cdef int squeeze_into(StridedLayout self, StridedLayout out_layout, axes_mask_t axis_mask=*) except -1 nogil
|
|
556
|
+
cdef StridedLayout get_unsqueezed(StridedLayout self, axes_mask_t axis_mask)
|
|
557
|
+
cdef int unsqueeze_into(StridedLayout self, StridedLayout out_layout, axes_mask_t axis_mask) except -1 nogil
|
|
558
|
+
cdef StridedLayout get_broadcast(StridedLayout self, BaseLayout& like)
|
|
559
|
+
cdef int broadcast_into(StridedLayout self, StridedLayout out_layout, BaseLayout& like) except -1 nogil
|
|
560
|
+
cdef StridedLayout get_unbroadcast(StridedLayout self)
|
|
561
|
+
cdef int unbroadcast_into(StridedLayout self, StridedLayout out_layout) except -1 nogil
|
|
562
|
+
cdef StridedLayout get_packed(StridedLayout self, int itemsize, intptr_t data_ptr, int axis=*, bint keep_dim=*)
|
|
563
|
+
cdef int pack_into(StridedLayout self, StridedLayout out_layout, int itemsize, intptr_t data_ptr, int axis=*, bint keep_dim=*) except -1 nogil
|
|
564
|
+
cdef StridedLayout get_unpacked(StridedLayout self, int itemsize, int axis=*, bint add_dim=*)
|
|
565
|
+
cdef int unpack_into(StridedLayout self, StridedLayout out_layout, int itemsize, int axis=*, bint add_dim=*) except -1 nogil
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
cdef class SlicedLayout:
|
|
569
|
+
|
|
570
|
+
cdef:
|
|
571
|
+
BaseLayout base
|
|
572
|
+
|
|
573
|
+
readonly:
|
|
574
|
+
stride_t slice_offset
|
|
575
|
+
|
|
576
|
+
cdef:
|
|
577
|
+
# Logically, SlicedLayout is StridedLayout + slice_offset,
|
|
578
|
+
# but to keep SlicedLayout overhead minimal
|
|
579
|
+
# (one vs two Python class instantiations for each [] operator call),
|
|
580
|
+
# it is really defined in terms of BaseLayout + slice_offset, and the StridedLayout
|
|
581
|
+
# is instantiated on demand.
|
|
582
|
+
StridedLayout _layout
|
|
583
|
+
|
|
584
|
+
cdef inline int init_from_base(SlicedLayout self, BaseLayout& base, stride_t slice_offset, StridedLayout layout=None) except -1:
|
|
585
|
+
_swap_base_layout(self.base, base)
|
|
586
|
+
self.slice_offset = slice_offset
|
|
587
|
+
self._layout = layout
|
|
588
|
+
return 0
|
|
589
|
+
|
|
590
|
+
cdef inline stride_t get_slice_offset_in_bytes(SlicedLayout self) except? -1 nogil:
|
|
591
|
+
return _overflow_checked_mul(self.slice_offset, self.base.itemsize)
|
|
592
|
+
|
|
593
|
+
cdef StridedLayout get_layout(SlicedLayout self)
|
|
594
|
+
cdef SlicedLayout get_sliced(SlicedLayout self, object slices)
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
# ==============================
|
|
598
|
+
# IterAxis helpers
|
|
599
|
+
# ==============================
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
cdef struct IterAxisInCOrder:
|
|
603
|
+
int ndim
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
cdef struct IterAxisInFOrder:
|
|
607
|
+
int ndim
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
cdef struct IterAxisInPermOrder:
|
|
611
|
+
int ndim
|
|
612
|
+
axis_t* axis_order
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
ctypedef fused IterAxisType:
|
|
616
|
+
IterAxisInCOrder
|
|
617
|
+
IterAxisInFOrder
|
|
618
|
+
IterAxisInPermOrder
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
cdef union IterAxisHelper:
|
|
622
|
+
IterAxisInCOrder c
|
|
623
|
+
IterAxisInFOrder f
|
|
624
|
+
IterAxisInPermOrder perm
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
cdef inline int _setup_axis_iter(IterAxisHelper& axis_helper, int ndim, OrderFlag order_flag, axis_vec_t* axis_order_vec) except -1 nogil:
|
|
628
|
+
if order_flag == ORDER_C:
|
|
629
|
+
axis_helper.c.ndim = ndim
|
|
630
|
+
return 0
|
|
631
|
+
elif order_flag == ORDER_F:
|
|
632
|
+
axis_helper.f.ndim = ndim
|
|
633
|
+
return 0
|
|
634
|
+
if order_flag != ORDER_PERM:
|
|
635
|
+
raise ValueError(f"Invalid order flag: {order_flag}")
|
|
636
|
+
if axis_order_vec == NULL:
|
|
637
|
+
raise ValueError("order is required for ORDER_PERM")
|
|
638
|
+
cdef int axis_order_size = deref(axis_order_vec).size()
|
|
639
|
+
if ndim != axis_order_size:
|
|
640
|
+
raise ValueError(f"Permutation must have the same length as the number of dimensions, got {axis_order_size} for {ndim}D tensor.")
|
|
641
|
+
cdef axis_t* axis_order = deref(axis_order_vec).data()
|
|
642
|
+
cdef axes_mask_t visited_mask = 0
|
|
643
|
+
cdef axis_t axis
|
|
644
|
+
cdef axes_mask_t axis_mask_i
|
|
645
|
+
for i in range(ndim):
|
|
646
|
+
if not _normalize_axis(axis_order[i], ndim):
|
|
647
|
+
raise ValueError(f"Axis {axis_order[i]} out of range for {ndim}D tensor")
|
|
648
|
+
axis = axis_order[i]
|
|
649
|
+
axis_mask_i = _axis2mask(axis)
|
|
650
|
+
if visited_mask & axis_mask_i:
|
|
651
|
+
raise ValueError(f"Expected a permutation. Axis {axis} appears multiple times.")
|
|
652
|
+
visited_mask |= axis_mask_i
|
|
653
|
+
axis_helper.perm.axis_order = axis_order
|
|
654
|
+
axis_helper.perm.ndim = ndim
|
|
655
|
+
return 0
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
cdef inline int _get_axis(IterAxisType& axis_helper, int i) except? -2 nogil:
|
|
659
|
+
if IterAxisType is IterAxisInCOrder:
|
|
660
|
+
return i
|
|
661
|
+
elif IterAxisType is IterAxisInFOrder:
|
|
662
|
+
return axis_helper.ndim - 1 - i
|
|
663
|
+
|
|
664
|
+
assert IterAxisType is IterAxisInPermOrder
|
|
665
|
+
return axis_helper.axis_order[i]
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
# ==============================
|
|
669
|
+
# Base layout helpers
|
|
670
|
+
# ==============================
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
cdef inline int _validate_itemsize(int itemsize) except -1 nogil:
|
|
674
|
+
if itemsize <= 0:
|
|
675
|
+
raise ValueError("The itemsize must be positive")
|
|
676
|
+
if itemsize & (itemsize - 1):
|
|
677
|
+
raise ValueError("The itemsize must be a power of two")
|
|
678
|
+
return 0
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
cdef inline int init_base_layout(BaseLayout& layout, int ndim, int itemsize) except -1 nogil:
|
|
682
|
+
if 0 > ndim or ndim > STRIDED_LAYOUT_MAX_NDIM:
|
|
683
|
+
raise ValueError(f"Unsupported number of dimensions: {ndim}. Max supported ndim is {STRIDED_LAYOUT_MAX_NDIM}")
|
|
684
|
+
_validate_itemsize(itemsize)
|
|
685
|
+
layout._mem.resize(2 * ndim)
|
|
686
|
+
layout.shape = layout._mem.data()
|
|
687
|
+
layout.strides = layout._mem.data() + ndim
|
|
688
|
+
layout.ndim = ndim
|
|
689
|
+
layout.itemsize = itemsize
|
|
690
|
+
return 0
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
cdef inline int trim_base_layout(BaseLayout& layout, int ndim) except -1 nogil:
|
|
694
|
+
if ndim > layout.ndim:
|
|
695
|
+
raise AssertionError(f"Cannot trim layout to {ndim} dimensions, it has {layout.ndim} dimensions")
|
|
696
|
+
layout.ndim = ndim
|
|
697
|
+
return 0
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
cdef inline int init_base_layout_from_tuple(BaseLayout& base, int itemsize, tuple shape, tuple strides, bint divide_strides=False) except -1:
|
|
701
|
+
cdef int ndim = len(shape)
|
|
702
|
+
init_base_layout(base, ndim, itemsize)
|
|
703
|
+
for i in range(ndim):
|
|
704
|
+
base.shape[i] = shape[i]
|
|
705
|
+
validate_base_shape(base)
|
|
706
|
+
|
|
707
|
+
if strides is not None:
|
|
708
|
+
if len(strides) != ndim:
|
|
709
|
+
raise ValueError(f"Strides, if provided, must have the same length as shape. Shape has {ndim} dimensions, but strides has {len(strides)} elements.")
|
|
710
|
+
for i in range(ndim):
|
|
711
|
+
base.strides[i] = strides[i]
|
|
712
|
+
if divide_strides:
|
|
713
|
+
divide_base_strides(base)
|
|
714
|
+
return 0
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
cdef inline int init_base_layout_from_ptr(BaseLayout& base, int ndim, int itemsize, extent_t* shape, stride_t* strides, bint divide_strides=False) except -1 nogil:
|
|
718
|
+
init_base_layout(base, ndim, itemsize)
|
|
719
|
+
for i in range(ndim):
|
|
720
|
+
base.shape[i] = shape[i]
|
|
721
|
+
validate_base_shape(base)
|
|
722
|
+
|
|
723
|
+
if strides != NULL:
|
|
724
|
+
for i in range(ndim):
|
|
725
|
+
base.strides[i] = strides[i]
|
|
726
|
+
if divide_strides:
|
|
727
|
+
divide_base_strides(base)
|
|
728
|
+
return 0
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
cdef inline int validate_base_shape(BaseLayout& base) except -1 nogil:
|
|
732
|
+
for i in range(base.ndim):
|
|
733
|
+
if base.shape[i] < 0:
|
|
734
|
+
raise ValueError("Extents must be non-negative")
|
|
735
|
+
return 0
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
cdef inline void _swap_base_layout(BaseLayout& a, BaseLayout& b) noexcept nogil:
|
|
739
|
+
if &a == &b:
|
|
740
|
+
return
|
|
741
|
+
_swap(a._mem, b._mem)
|
|
742
|
+
_swap(a.shape, b.shape)
|
|
743
|
+
_swap(a.strides, b.strides)
|
|
744
|
+
_swap(a.ndim, b.ndim)
|
|
745
|
+
_swap(a.itemsize, b.itemsize)
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
cdef inline bint base_equal_shapes(BaseLayout& a, BaseLayout& b) noexcept nogil:
|
|
749
|
+
if a.ndim != b.ndim:
|
|
750
|
+
return False
|
|
751
|
+
for i in range(a.ndim):
|
|
752
|
+
if a.shape[i] != b.shape[i]:
|
|
753
|
+
return False
|
|
754
|
+
return True
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
cdef inline bint _base_equal_strides(BaseLayout& a, BaseLayout& b) noexcept nogil:
|
|
758
|
+
cdef stride_t* strides_a = a.strides
|
|
759
|
+
cdef stride_t* strides_b = b.strides
|
|
760
|
+
for i in range(a.ndim):
|
|
761
|
+
if strides_a[i] != strides_b[i]:
|
|
762
|
+
return False
|
|
763
|
+
return True
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
cdef inline bint base_equal_strides(BaseLayout& a, BaseLayout& b) noexcept nogil:
|
|
767
|
+
return a.ndim == b.ndim and _base_equal_strides(a, b)
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
cdef inline bint base_layout_equal(BaseLayout& a, BaseLayout& b) noexcept nogil:
|
|
771
|
+
return a.itemsize == b.itemsize and base_equal_shapes(a, b) and _base_equal_strides(a, b)
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
cdef inline bint base_layout_almost_equal(BaseLayout& a, BaseLayout& b) noexcept nogil:
|
|
775
|
+
if a.itemsize != b.itemsize:
|
|
776
|
+
return False
|
|
777
|
+
cdef int ndim = a.ndim
|
|
778
|
+
if ndim != b.ndim:
|
|
779
|
+
return False
|
|
780
|
+
cdef stride_t* strides_a = a.strides
|
|
781
|
+
cdef stride_t* strides_b = b.strides
|
|
782
|
+
cdef extent_t extent
|
|
783
|
+
for i in range(ndim):
|
|
784
|
+
extent = a.shape[i]
|
|
785
|
+
if b.shape[i] != extent:
|
|
786
|
+
return False
|
|
787
|
+
if extent != 1 and strides_a[i] != strides_b[i]:
|
|
788
|
+
return False
|
|
789
|
+
return True
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
@cython.overflowcheck(True)
|
|
793
|
+
cdef inline int64_t base_volume(BaseLayout& base) except? -1 nogil:
|
|
794
|
+
cdef int64_t vol = 1
|
|
795
|
+
for i in range(base.ndim):
|
|
796
|
+
vol *= base.shape[i]
|
|
797
|
+
return vol
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
cdef inline int divide_base_strides(BaseLayout& base) except -1 nogil:
|
|
801
|
+
cdef stride_t stride
|
|
802
|
+
cdef stride_t itemsize = base.itemsize
|
|
803
|
+
for i in range(base.ndim):
|
|
804
|
+
stride = base.strides[i] // itemsize
|
|
805
|
+
if stride * itemsize != base.strides[i]:
|
|
806
|
+
raise ValueError("strides must be divisible by itemsize")
|
|
807
|
+
base.strides[i] = stride
|
|
808
|
+
return 0
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
cdef inline void zero_out_base_strides(BaseLayout& base) noexcept nogil:
|
|
812
|
+
for i in range(base.ndim):
|
|
813
|
+
base.strides[i] = 0
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
cdef inline stride_t _set_base_dense_strides(IterAxisType& axis_iter, BaseLayout& base) except? -1 nogil:
|
|
817
|
+
cdef stride_t stride = 1
|
|
818
|
+
cdef int i = base.ndim - 1
|
|
819
|
+
while i >= 0:
|
|
820
|
+
axis = _get_axis(axis_iter, i)
|
|
821
|
+
base.strides[axis] = stride
|
|
822
|
+
stride = _overflow_checked_mul(stride, base.shape[axis])
|
|
823
|
+
i -= 1
|
|
824
|
+
if stride == 0:
|
|
825
|
+
zero_out_base_strides(base)
|
|
826
|
+
return stride
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
cdef inline stride_t set_base_dense_strides(BaseLayout& base, OrderFlag order_flag, axis_vec_t* stride_order) except? -1 nogil:
|
|
830
|
+
cdef IterAxisHelper axis_iter
|
|
831
|
+
_setup_axis_iter(axis_iter, base.ndim, order_flag, stride_order)
|
|
832
|
+
if order_flag == ORDER_C:
|
|
833
|
+
return _set_base_dense_strides(axis_iter.c, base)
|
|
834
|
+
elif order_flag == ORDER_F:
|
|
835
|
+
return _set_base_dense_strides(axis_iter.f, base)
|
|
836
|
+
elif order_flag == ORDER_PERM:
|
|
837
|
+
return _set_base_dense_strides(axis_iter.perm, base)
|
|
838
|
+
else:
|
|
839
|
+
raise ValueError(f"Invalid order flag: {order_flag}")
|
|
840
|
+
|
|
841
|
+
|
|
842
|
+
cdef inline bint _base_is_dense(IterAxisType& axis_iter, BaseLayout& base, bint allow_negative_strides, bint allow_leading_dim_stride) except -1 nogil:
|
|
843
|
+
cdef int j = base.ndim - 1
|
|
844
|
+
cdef stride_t expected_stride = 1
|
|
845
|
+
cdef stride_t current_stride
|
|
846
|
+
cdef extent_t extent
|
|
847
|
+
cdef axis_t axis
|
|
848
|
+
if allow_leading_dim_stride:
|
|
849
|
+
# Find first non-unit extent
|
|
850
|
+
while j >= 0:
|
|
851
|
+
axis = _get_axis(axis_iter, j)
|
|
852
|
+
if base.shape[axis] != 1:
|
|
853
|
+
expected_stride = base.strides[axis]
|
|
854
|
+
break
|
|
855
|
+
j -= 1
|
|
856
|
+
if expected_stride == 0:
|
|
857
|
+
return False
|
|
858
|
+
elif expected_stride < 0:
|
|
859
|
+
if allow_negative_strides:
|
|
860
|
+
expected_stride = -expected_stride
|
|
861
|
+
else:
|
|
862
|
+
return False
|
|
863
|
+
while j >= 0:
|
|
864
|
+
axis = _get_axis(axis_iter, j)
|
|
865
|
+
extent = base.shape[axis]
|
|
866
|
+
if extent != 1:
|
|
867
|
+
current_stride = base.strides[axis]
|
|
868
|
+
if allow_negative_strides:
|
|
869
|
+
current_stride = c_abs(current_stride)
|
|
870
|
+
if current_stride != expected_stride:
|
|
871
|
+
return False
|
|
872
|
+
expected_stride *= extent
|
|
873
|
+
j -= 1
|
|
874
|
+
return True
|
|
875
|
+
|
|
876
|
+
|
|
877
|
+
cdef inline bint base_is_dense(int64_t volume, BaseLayout& base, OrderFlag order_flag, axis_vec_t* stride_order, bint allow_negative_strides, bint allow_leading_dim_stride) except -1 nogil:
|
|
878
|
+
cdef int ndim = base.ndim
|
|
879
|
+
if volume == 0 or ndim < 1:
|
|
880
|
+
return True
|
|
881
|
+
cdef IterAxisHelper axis_iter
|
|
882
|
+
_setup_axis_iter(axis_iter, ndim, order_flag, stride_order)
|
|
883
|
+
if order_flag == ORDER_C:
|
|
884
|
+
return _base_is_dense(axis_iter.c, base, allow_negative_strides, allow_leading_dim_stride)
|
|
885
|
+
elif order_flag == ORDER_F:
|
|
886
|
+
return _base_is_dense(axis_iter.f, base, allow_negative_strides, allow_leading_dim_stride)
|
|
887
|
+
elif order_flag == ORDER_PERM:
|
|
888
|
+
return _base_is_dense(axis_iter.perm, base, allow_negative_strides, allow_leading_dim_stride)
|
|
889
|
+
else:
|
|
890
|
+
raise ValueError(f"Invalid order flag: {order_flag}")
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
cdef inline bint base_has_no_negative_stride(int64_t volume, BaseLayout& base) except -1 nogil:
|
|
894
|
+
if volume == 0:
|
|
895
|
+
return True
|
|
896
|
+
for i in range(base.ndim):
|
|
897
|
+
if base.shape[i] != 1 and base.strides[i] < 0:
|
|
898
|
+
return False
|
|
899
|
+
return True
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
cdef inline int base_layout_min_max_stride(int64_t volume, BaseLayout& base, stride_t& min_stride, stride_t& max_stride) except -1 nogil:
|
|
903
|
+
# There's no meaningful stride for empty or scalar layout.
|
|
904
|
+
# 0 could hint at broadcasting while 1 hints at dense.
|
|
905
|
+
# Chances are, some packages could complain about 0, so we pick 1.
|
|
906
|
+
# For volume 0, maybe more importantly, as we treat it as dense, the
|
|
907
|
+
# max_stride = 1 agrees with the offset_bounds = [0, -1]
|
|
908
|
+
# -1 = max_offset = (volume - 1) * max_stride = (0 - 1) * 1
|
|
909
|
+
min_stride = 1
|
|
910
|
+
max_stride = 1
|
|
911
|
+
if volume <= 1: # empty or effectively a scalar
|
|
912
|
+
return 0
|
|
913
|
+
|
|
914
|
+
cdef int i = 0
|
|
915
|
+
cdef extent_t extent
|
|
916
|
+
cdef stride_t stride
|
|
917
|
+
# find the first non-unit extent
|
|
918
|
+
while i < base.ndim:
|
|
919
|
+
extent = base.shape[i]
|
|
920
|
+
if extent != 1:
|
|
921
|
+
stride = base.strides[i]
|
|
922
|
+
min_stride = stride
|
|
923
|
+
max_stride = stride
|
|
924
|
+
i += 1
|
|
925
|
+
break
|
|
926
|
+
i += 1
|
|
927
|
+
|
|
928
|
+
# find the min and max stride
|
|
929
|
+
while i < base.ndim:
|
|
930
|
+
extent = base.shape[i]
|
|
931
|
+
if extent != 1:
|
|
932
|
+
stride = base.strides[i]
|
|
933
|
+
if c_abs(stride) < c_abs(min_stride):
|
|
934
|
+
min_stride = stride
|
|
935
|
+
if c_abs(stride) > c_abs(max_stride):
|
|
936
|
+
max_stride = stride
|
|
937
|
+
i += 1
|
|
938
|
+
return 0
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
cdef inline axes_mask_t base_unit_extents_mask(BaseLayout& base) except -1 nogil:
|
|
942
|
+
cdef axes_mask_t unit_extents_mask = 0
|
|
943
|
+
cdef int i = 0
|
|
944
|
+
while i < base.ndim:
|
|
945
|
+
if base.shape[i] == 1:
|
|
946
|
+
unit_extents_mask |= _axis2mask(i)
|
|
947
|
+
i += 1
|
|
948
|
+
return unit_extents_mask
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
cdef inline bint _base_has_stride_order(IterAxisType& axis_iter, BaseLayout& base) except -1 nogil:
|
|
952
|
+
cdef stride_t prev_stride = 0
|
|
953
|
+
cdef extent_t extent
|
|
954
|
+
cdef stride_t stride
|
|
955
|
+
cdef int i = base.ndim - 1
|
|
956
|
+
cdef axis_t axis
|
|
957
|
+
while i >= 0:
|
|
958
|
+
axis = _get_axis(axis_iter, i)
|
|
959
|
+
extent = base.shape[axis]
|
|
960
|
+
if extent != 1:
|
|
961
|
+
stride = c_abs(base.strides[axis])
|
|
962
|
+
if stride < prev_stride:
|
|
963
|
+
return False
|
|
964
|
+
prev_stride = stride
|
|
965
|
+
i -= 1
|
|
966
|
+
return True
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
cdef inline bint base_has_stride_order(int64_t volume, BaseLayout& base, OrderFlag order_flag, axis_vec_t* stride_order_vec) except -1 nogil:
|
|
970
|
+
if volume == 0:
|
|
971
|
+
return True
|
|
972
|
+
cdef IterAxisHelper axis_iter
|
|
973
|
+
_setup_axis_iter(axis_iter, base.ndim, order_flag, stride_order_vec)
|
|
974
|
+
if order_flag == ORDER_C:
|
|
975
|
+
return _base_has_stride_order(axis_iter.c, base)
|
|
976
|
+
elif order_flag == ORDER_F:
|
|
977
|
+
return _base_has_stride_order(axis_iter.f, base)
|
|
978
|
+
elif order_flag == ORDER_PERM:
|
|
979
|
+
return _base_has_stride_order(axis_iter.perm, base)
|
|
980
|
+
else:
|
|
981
|
+
raise ValueError(f"Invalid order flag: {order_flag}")
|
|
982
|
+
|
|
983
|
+
|
|
984
|
+
cdef inline bint base_is_unique(int64_t volume, BaseLayout& base, axis_vec_t& stride_order) except -1 nogil:
|
|
985
|
+
"""
|
|
986
|
+
Check if every two valid indices map to different memory offsets in the
|
|
987
|
+
range ``[min_offset, max_offset]``.
|
|
988
|
+
Note, ``is_unique() and is_exhaustive() == is_abs_dense_any()``.
|
|
989
|
+
|
|
990
|
+
What do we check here?
|
|
991
|
+
Let's have o = stride_order and say we processed dims ``o[i], ..., o[ndim-1]``
|
|
992
|
+
Indices limited to those dimensions (i.e. index[o[j]] = 0 for j < i)
|
|
993
|
+
map indices uniquely (injective) to ``[cur_min_offset, cur_max_offset]``, where
|
|
994
|
+
``cur_min_offset = sum((extent[o[j]] - 1) * strides[o[j]] for j in range(i, ndim) if strides[o[j]] < 0)``
|
|
995
|
+
``cur_max_offset = sum((extent[o[j]] - 1) * strides[o[j]] for j in range(i, ndim) if strides[o[j]] >= 0)``
|
|
996
|
+
Now, if we consider the next dimension (``o[i - 1]``) and its stride ``s_next``,
|
|
997
|
+
we want to make sure that ``idx * s_next + [cur_min_offset, cur_max_offset]``
|
|
998
|
+
and ``(idx + 1) * s_next + [cur_min_offset, cur_max_offset]`` do not overlap.
|
|
999
|
+
We check that:
|
|
1000
|
+
* for ``s_next >= 0``, ``idx * s_next + cur_max_offset < (idx + 1) * s_next + cur_min_offset``
|
|
1001
|
+
* for ``s_next < 0``, ``(idx + 1) * s_next + cur_max_offset < idx * s_next + cur_min_offset``
|
|
1002
|
+
Those two are equivalent to abs(s_next) > cur_max_offset - cur_min_offset.
|
|
1003
|
+
Note, ``cur_max_offset - cur_min_offset = sum((extent[o[j]] - 1) * abs(strides[o[j]]) for j in range(i, ndim))``.
|
|
1004
|
+
"""
|
|
1005
|
+
if volume == 0:
|
|
1006
|
+
return True
|
|
1007
|
+
cdef int64_t cur_range = 0
|
|
1008
|
+
cdef int i = base.ndim - 1
|
|
1009
|
+
cdef int64_t stride
|
|
1010
|
+
cdef axis_t axis
|
|
1011
|
+
cdef extent_t extent
|
|
1012
|
+
while i >= 0:
|
|
1013
|
+
axis = stride_order[i]
|
|
1014
|
+
extent = base.shape[axis]
|
|
1015
|
+
if extent != 1:
|
|
1016
|
+
stride = c_abs(base.strides[axis])
|
|
1017
|
+
if stride <= cur_range:
|
|
1018
|
+
return False
|
|
1019
|
+
cur_range = _overflow_checked_sum(cur_range, _overflow_checked_mul(stride, (extent - 1)))
|
|
1020
|
+
i -= 1
|
|
1021
|
+
return True
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
cdef inline bint base_is_exhaustive(int64_t volume, BaseLayout& base, axis_vec_t& stride_order) except -1 nogil:
|
|
1025
|
+
"""
|
|
1026
|
+
Check if there exists an index for every element in the range
|
|
1027
|
+
``[min_offset, max_offset]``, i.e. the mapping from indices
|
|
1028
|
+
to memory offsets is onto/surjective.
|
|
1029
|
+
Note, ``is_unique() and is_exhaustive() == is_abs_dense_any()``.
|
|
1030
|
+
|
|
1031
|
+
What do we check here?
|
|
1032
|
+
Let's have o = stride_order and say we processed dims ``o[i], ..., o[ndim-1]``
|
|
1033
|
+
Indices limited to those dimensions (i.e. index[o[j]] = 0 for j < i)
|
|
1034
|
+
map indices exhaustively (onto/surjective) to ``[cur_min_offset, cur_max_offset]``, where
|
|
1035
|
+
``cur_min_offset = sum((extent[o[j]] - 1) * strides[o[j]] for j in range(i, ndim) if strides[o[j]] < 0)``
|
|
1036
|
+
``cur_max_offset = sum((extent[o[j]] - 1) * strides[o[j]] for j in range(i, ndim) if strides[o[j]] >= 0)``
|
|
1037
|
+
Now, if we consider the next dimension (``o[i - 1]``) and its stride ``s_next``,
|
|
1038
|
+
we want to make sure that ``idx * s_next + [cur_min_offset, cur_max_offset]``
|
|
1039
|
+
and ``(idx + 1) * s_next + [cur_min_offset, cur_max_offset]`` overlap or touch.
|
|
1040
|
+
We check that:
|
|
1041
|
+
* for ``s_next >= 0``, ``idx * s_next + cur_max_offset + 1 >= (idx + 1) * s_next + cur_min_offset``
|
|
1042
|
+
* for ``s_next < 0``, ``(idx + 1) * s_next + cur_max_offset + 1 >= idx * s_next + cur_min_offset``
|
|
1043
|
+
Those two are equivalent to abs(s_next) <= cur_max_offset - cur_min_offset + 1.
|
|
1044
|
+
Note, ``cur_max_offset - cur_min_offset = sum((extent[o[j]] - 1) * abs(strides[o[j]]) for j in range(i, ndim))``.
|
|
1045
|
+
"""
|
|
1046
|
+
if volume == 0:
|
|
1047
|
+
return True
|
|
1048
|
+
cdef int64_t cur_range = 1
|
|
1049
|
+
cdef int i = base.ndim - 1
|
|
1050
|
+
cdef int64_t stride
|
|
1051
|
+
cdef axis_t axis
|
|
1052
|
+
cdef extent_t extent
|
|
1053
|
+
while i >= 0:
|
|
1054
|
+
axis = stride_order[i]
|
|
1055
|
+
extent = base.shape[axis]
|
|
1056
|
+
if extent != 1:
|
|
1057
|
+
stride = c_abs(base.strides[axis])
|
|
1058
|
+
if stride > cur_range:
|
|
1059
|
+
return False
|
|
1060
|
+
cur_range = _overflow_checked_sum(cur_range, _overflow_checked_mul(stride, (extent - 1)))
|
|
1061
|
+
i -= 1
|
|
1062
|
+
return True
|
|
1063
|
+
|
|
1064
|
+
|
|
1065
|
+
# ==============================
|
|
1066
|
+
# Strided layout helpers
|
|
1067
|
+
# ==============================
|
|
1068
|
+
|
|
1069
|
+
cdef inline int dense_prop(OrderFlag order_flag, bint allow_negative_strides, bint allow_leading_dim_stride) except -1 nogil:
|
|
1070
|
+
cdef int prop = _dense_prop(order_flag, allow_negative_strides, allow_leading_dim_stride)
|
|
1071
|
+
if prop < 0:
|
|
1072
|
+
raise ValueError(f"Invalid order_flag: {order_flag}")
|
|
1073
|
+
return prop
|
|
1074
|
+
|
|
1075
|
+
|
|
1076
|
+
cdef inline bint has_valid_property(StridedLayout self, int prop) except? -1 nogil:
|
|
1077
|
+
return _has_valid_property(self._prop_mask, prop)
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
cdef inline int mark_property_valid(StridedLayout self, int prop) except? -1 nogil:
|
|
1081
|
+
_mark_property_valid(self._prop_mask, prop)
|
|
1082
|
+
return 0
|
|
1083
|
+
|
|
1084
|
+
|
|
1085
|
+
cdef inline bint boolean_property(StridedLayout self, int prop) except? -1 nogil:
|
|
1086
|
+
return _boolean_property(self._boolean_props, prop)
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
cdef inline bint set_boolean_property(StridedLayout self, int prop, bint value) except? -1 nogil:
|
|
1090
|
+
_set_boolean_property(self._prop_mask, self._boolean_props, prop, value)
|
|
1091
|
+
return value
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
cdef inline bint peek_boolean_property(StridedLayout self, int prop) except? -1 nogil:
|
|
1095
|
+
"""
|
|
1096
|
+
True -> the property is True
|
|
1097
|
+
False -> the property is False or not set yet
|
|
1098
|
+
"""
|
|
1099
|
+
return _has_valid_property(self._prop_mask, prop) and _boolean_property(self._boolean_props, prop)
|
|
1100
|
+
|
|
1101
|
+
|
|
1102
|
+
cdef inline dict get_boolean_flags(StridedLayout self):
|
|
1103
|
+
cdef dict properties = {
|
|
1104
|
+
"IS_CONTIGUOUS_C": IS_CONTIGUOUS_C,
|
|
1105
|
+
"IS_ABS_DENSE_C": IS_ABS_DENSE_C,
|
|
1106
|
+
"IS_LEADING_DENSE_C": IS_LEADING_DENSE_C,
|
|
1107
|
+
"IS_ABS_LEADING_DENSE_C": IS_ABS_LEADING_DENSE_C,
|
|
1108
|
+
|
|
1109
|
+
"IS_CONTIGUOUS_F": IS_CONTIGUOUS_F,
|
|
1110
|
+
"IS_ABS_DENSE_F": IS_ABS_DENSE_F,
|
|
1111
|
+
"IS_LEADING_DENSE_F": IS_LEADING_DENSE_F,
|
|
1112
|
+
"IS_ABS_LEADING_DENSE_F": IS_ABS_LEADING_DENSE_F,
|
|
1113
|
+
|
|
1114
|
+
"IS_CONTIGUOUS_ANY": IS_CONTIGUOUS_ANY,
|
|
1115
|
+
"IS_ABS_DENSE_ANY": IS_ABS_DENSE_ANY,
|
|
1116
|
+
"IS_LEADING_DENSE_ANY": IS_LEADING_DENSE_ANY,
|
|
1117
|
+
"IS_ABS_LEADING_DENSE_ANY": IS_ABS_LEADING_DENSE_ANY,
|
|
1118
|
+
|
|
1119
|
+
"HAS_STRIDE_ORDER_C": HAS_STRIDE_ORDER_C,
|
|
1120
|
+
"HAS_STRIDE_ORDER_F": HAS_STRIDE_ORDER_F,
|
|
1121
|
+
"IS_UNIQUE": IS_UNIQUE,
|
|
1122
|
+
"IS_EXHAUSTIVE": IS_EXHAUSTIVE,
|
|
1123
|
+
"HAS_NO_NEGATIVE_STRIDE": HAS_NO_NEGATIVE_STRIDE,
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
return {name: boolean_property(self, prop) for name, prop in properties.items() if has_valid_property(self, prop)}
|
|
1127
|
+
|
|
1128
|
+
# ==============================
|
|
1129
|
+
# Conversion, validation and normalization helpers
|
|
1130
|
+
# ==============================
|
|
1131
|
+
|
|
1132
|
+
cdef inline tuple as_tuple(object obj):
|
|
1133
|
+
if obj is None or type(obj) is tuple:
|
|
1134
|
+
return obj
|
|
1135
|
+
elif isinstance(obj, int):
|
|
1136
|
+
return (obj,)
|
|
1137
|
+
else:
|
|
1138
|
+
return tuple(obj)
|
|
1139
|
+
|
|
1140
|
+
|
|
1141
|
+
cdef inline axes_mask_t _axis2mask(axis_t axis) noexcept nogil:
|
|
1142
|
+
return 1ULL << axis
|
|
1143
|
+
|
|
1144
|
+
|
|
1145
|
+
cdef inline axes_mask_t all_axes_mask(int ndim) noexcept nogil:
|
|
1146
|
+
if ndim == 0:
|
|
1147
|
+
return 0
|
|
1148
|
+
elif ndim == STRIDED_LAYOUT_MAX_NDIM:
|
|
1149
|
+
return STRIDED_LAYOUT_AXES_MASK_ALL
|
|
1150
|
+
return (1ULL << ndim) - 1ULL
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
cdef inline OrderFlag _stride_order2vec(axis_vec_t& stride_order_vec, object stride_order) except? ORDER_NONE:
|
|
1154
|
+
if stride_order == 'C':
|
|
1155
|
+
return ORDER_C
|
|
1156
|
+
elif stride_order == 'F':
|
|
1157
|
+
return ORDER_F
|
|
1158
|
+
elif isinstance(stride_order, tuple | list):
|
|
1159
|
+
_tuple2axis_vec(stride_order_vec, stride_order)
|
|
1160
|
+
return ORDER_PERM
|
|
1161
|
+
return ORDER_NONE
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
cdef inline int _tuple2axis_vec(axis_vec_t& vec, object t) except -1:
|
|
1165
|
+
cdef int ndim = len(t)
|
|
1166
|
+
vec.resize(ndim)
|
|
1167
|
+
for i in range(ndim):
|
|
1168
|
+
vec[i] = t[i]
|
|
1169
|
+
return 0
|
|
1170
|
+
|
|
1171
|
+
|
|
1172
|
+
cdef inline bint _normalize_axis(integer_t& axis, integer_t extent) except -1 nogil:
|
|
1173
|
+
if axis < -extent or axis >= extent:
|
|
1174
|
+
return False
|
|
1175
|
+
if axis < 0:
|
|
1176
|
+
axis += extent
|
|
1177
|
+
return True
|
|
1178
|
+
|
|
1179
|
+
|
|
1180
|
+
cdef inline axes_mask_t _tuple2axis_mask(int32_t ndim, object t) except? -1:
|
|
1181
|
+
# Returns mask such that mask & _axis2mask(axis) is True iff axis is in t
|
|
1182
|
+
cdef axes_mask_t mask = 0
|
|
1183
|
+
cdef axis_t axis
|
|
1184
|
+
cdef axes_mask_t axis_mask = 0
|
|
1185
|
+
if isinstance(t, int):
|
|
1186
|
+
axis = t
|
|
1187
|
+
if not _normalize_axis(axis, ndim):
|
|
1188
|
+
raise ValueError(f"Invalid axis: {axis} out of range for {ndim}D tensor")
|
|
1189
|
+
return _axis2mask(axis)
|
|
1190
|
+
for e in t:
|
|
1191
|
+
axis = e
|
|
1192
|
+
if not _normalize_axis(axis, ndim):
|
|
1193
|
+
raise ValueError(f"Invalid axis: {axis} out of range for {ndim}D tensor")
|
|
1194
|
+
axis_mask = _axis2mask(axis)
|
|
1195
|
+
if mask & axis_mask:
|
|
1196
|
+
raise ValueError(f"Axis {axis} appears multiple times.")
|
|
1197
|
+
mask |= axis_mask
|
|
1198
|
+
return mask
|
|
1199
|
+
|
|
1200
|
+
|
|
1201
|
+
cdef inline tuple _axes_mask2tuple(axes_mask_t mask):
|
|
1202
|
+
cdef int ndim = _popcount(mask)
|
|
1203
|
+
cdef axis_vec_t axes
|
|
1204
|
+
axes.resize(ndim)
|
|
1205
|
+
cdef int out_i = 0
|
|
1206
|
+
for i in range(STRIDED_LAYOUT_MAX_NDIM):
|
|
1207
|
+
if mask & _axis2mask(i):
|
|
1208
|
+
axes[out_i] = i
|
|
1209
|
+
out_i += 1
|
|
1210
|
+
return _carray_integer_t_to_tuple(axes.data(), ndim)
|
|
1211
|
+
|
|
1212
|
+
|
|
1213
|
+
cdef inline axes_mask_t axes_mask_from_range(int32_t ndim, int32_t start_axis, int32_t end_axis) except? -1 nogil:
|
|
1214
|
+
# Returns mask such that mask & _axis2mask(axis) is True iff axis is
|
|
1215
|
+
# in inclusive range [start_axis, end_axis]
|
|
1216
|
+
if start_axis == 0 and end_axis == -1:
|
|
1217
|
+
return all_axes_mask(ndim)
|
|
1218
|
+
if not _normalize_axis(start_axis, ndim):
|
|
1219
|
+
raise ValueError(f"Invalid start axis: {start_axis} out of range for {ndim}D tensor")
|
|
1220
|
+
if not _normalize_axis(end_axis, ndim):
|
|
1221
|
+
raise ValueError(f"Invalid end axis: {end_axis} out of range for {ndim}D tensor")
|
|
1222
|
+
cdef axes_mask_t axes_mask = all_axes_mask(ndim)
|
|
1223
|
+
axes_mask &= (STRIDED_LAYOUT_AXES_MASK_ALL << start_axis)
|
|
1224
|
+
axes_mask &= (STRIDED_LAYOUT_AXES_MASK_ALL >> (STRIDED_LAYOUT_MAX_NDIM - end_axis - 1))
|
|
1225
|
+
return axes_mask
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
cdef inline axes_mask_t unsqueeze_to_ndim_mask(int32_t in_ndim, int32_t out_ndim, int32_t axis) except? -1 nogil:
|
|
1229
|
+
if in_ndim > out_ndim:
|
|
1230
|
+
raise ValueError(
|
|
1231
|
+
f"Invalid number of dimensions: {out_ndim}. It must be greater than or equal "
|
|
1232
|
+
f"to the current number of dimensions: {in_ndim}."
|
|
1233
|
+
)
|
|
1234
|
+
elif in_ndim == out_ndim:
|
|
1235
|
+
return 0
|
|
1236
|
+
elif out_ndim > STRIDED_LAYOUT_MAX_NDIM:
|
|
1237
|
+
raise ValueError(
|
|
1238
|
+
f"Invalid number of dimensions: {out_ndim}. It must be less than or equal "
|
|
1239
|
+
f"to {STRIDED_LAYOUT_MAX_NDIM}."
|
|
1240
|
+
)
|
|
1241
|
+
if not _normalize_axis(axis, in_ndim + 1):
|
|
1242
|
+
raise ValueError(f"Invalid axis: {axis} out of range for {in_ndim}D tensor")
|
|
1243
|
+
cdef int num_new_axes = out_ndim - in_ndim
|
|
1244
|
+
cdef axes_mask_t mask = all_axes_mask(axis + num_new_axes)
|
|
1245
|
+
cdef axes_mask_t left_mask = all_axes_mask(axis)
|
|
1246
|
+
return mask & (~left_mask)
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
cdef inline axes_mask_t flattening_axes_mask_from_range(int ndim, int start_axis, int end_axis) except? -1 nogil:
|
|
1250
|
+
# Returns mask such that mask & _axis2mask(axis) is True iff axis is
|
|
1251
|
+
# in inclusive range [start_axis + 1, end_axis]
|
|
1252
|
+
# (As for flattening, the mask & _axis2mask(axis) means the axis
|
|
1253
|
+
# can be flattened with axis - 1)
|
|
1254
|
+
if start_axis == 0 and end_axis == -1:
|
|
1255
|
+
return all_axes_mask(ndim) & (~_axis2mask(0))
|
|
1256
|
+
if not _normalize_axis(start_axis, ndim):
|
|
1257
|
+
raise ValueError(f"Invalid start axis: {start_axis} out of range for {ndim}D tensor")
|
|
1258
|
+
if not _normalize_axis(end_axis, ndim):
|
|
1259
|
+
raise ValueError(f"Invalid end axis: {end_axis} out of range for {ndim}D tensor")
|
|
1260
|
+
if start_axis == end_axis:
|
|
1261
|
+
return 0
|
|
1262
|
+
cdef axes_mask_t axes_mask = all_axes_mask(ndim)
|
|
1263
|
+
axes_mask &= (STRIDED_LAYOUT_AXES_MASK_ALL << (start_axis + 1))
|
|
1264
|
+
axes_mask &= (STRIDED_LAYOUT_AXES_MASK_ALL >> (STRIDED_LAYOUT_MAX_NDIM - end_axis - 1))
|
|
1265
|
+
return axes_mask
|
|
1266
|
+
|
|
1267
|
+
|
|
1268
|
+
@cython.overflowcheck(True)
|
|
1269
|
+
cdef inline int64_t _overflow_checked_mul(int64_t a, int64_t b) except? -1 nogil:
|
|
1270
|
+
return a * b
|
|
1271
|
+
|
|
1272
|
+
|
|
1273
|
+
@cython.overflowcheck(True)
|
|
1274
|
+
cdef inline int64_t _overflow_checked_diff(int64_t a, int64_t b) except? -1 nogil:
|
|
1275
|
+
return a - b
|
|
1276
|
+
|
|
1277
|
+
|
|
1278
|
+
@cython.overflowcheck(True)
|
|
1279
|
+
cdef inline int64_t _overflow_checked_sum(int64_t a, int64_t b) except? -1 nogil:
|
|
1280
|
+
return a + b
|
|
1281
|
+
|
|
1282
|
+
|
|
1283
|
+
@cython.overflowcheck(True)
|
|
1284
|
+
cdef inline int64_t _overflow_checked_div_ceil(int64_t a, int64_t b) except? -1 nogil:
|
|
1285
|
+
return (a + b - 1) // b
|
|
1286
|
+
|
|
1287
|
+
|
|
1288
|
+
# https://github.com/NVIDIA/cuda-python/blob/main/cuda_core/cuda/core/_utils/cuda_utils.pxd
|
|
1289
|
+
|
|
1290
|
+
# Use Python C API directly instead of cython wrappers, as those end up incorrectly
|
|
1291
|
+
# decrefing the temporary Python int object after it is set as tuple element (resulting
|
|
1292
|
+
# in a double free when the tuple is garbage collected).
|
|
1293
|
+
cdef extern from "Python.h":
|
|
1294
|
+
PyObject *_PyLong_FromLongLong "PyLong_FromLongLong" (long long val) except NULL
|
|
1295
|
+
void _PyTuple_SET_ITEM "PyTuple_SET_ITEM" (object p, Py_ssize_t pos, PyObject *o)
|
|
1296
|
+
|
|
1297
|
+
|
|
1298
|
+
cdef inline tuple _carray_integer_t_to_tuple(const integer_t *ptr, int length):
|
|
1299
|
+
# Construct shape and strides tuples using the Python/C API for speed
|
|
1300
|
+
cdef tuple result = cpython.PyTuple_New(length)
|
|
1301
|
+
for i in range(length):
|
|
1302
|
+
_PyTuple_SET_ITEM(result, i, _PyLong_FromLongLong(ptr[i]))
|
|
1303
|
+
return result
|