torchvulkan 0.2.0a0__tar.gz → 0.2.0a2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/PKG-INFO +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/backend.cpp +43 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/matmul.cpp +7 -0
- torchvulkan-0.2.0a2/src/api/registration.cpp +36 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/tensor/transfer.cpp +16 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/core/indexing.slang +13 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/arg_reduce.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/binary.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/cast.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/compare.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/copy.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/ctc.slang +7 -3
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/distribution.slang +2 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/dropout.slang +4 -2
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/embedding_backward.slang +2 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/fft.slang +3 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/fill.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/grid_sample.slang +3 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/index_gather.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/integer_binary.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/ldl.slang +3 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/multinomial.slang +2 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/nllloss.slang +4 -3
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/pool.slang +2 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/random.slang +3 -2
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/reduce.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/reduce_subgroup.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/resample.slang +3 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/resample_u8.slang +2 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/scan.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/scan_arg.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/scatter.slang +2 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/searchsorted.slang +2 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/sort.slang +2 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/special.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/triangular_solve.slang +3 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/unary.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/where.slang +1 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/allocator.cpp +4 -1
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/device_context.cpp +19 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/device_context.h +8 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/shader.cpp +4 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/shader.h +3 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/vulkan_context.cpp +7 -2
- torchvulkan-0.2.0a2/torchvulkan/__init__.py +157 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/torchvulkan/_version.py +3 -3
- torchvulkan-0.2.0a0/src/api/registration.cpp +0 -15
- torchvulkan-0.2.0a0/torchvulkan/__init__.py +0 -89
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.github/actions/setup-toolchain/action.yml +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.github/scripts/fetch-moltenvk-macos.sh +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.github/scripts/install-toolchain-linux.sh +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.github/workflows/release.yaml +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.github/workflows/test_torchvulkan.yml +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.gitignore +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/CMakeLists.txt +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/LICENSE +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/README.md +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/benchmark/benchmark.py +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/examples/llm.py +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/examples/mnist.py +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/extern/vma/vk_mem_alloc.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/extern/volk/volk.c +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/extern/volk/volk.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/licenses/MoltenVK-LICENSE +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/media/benchmark-cuda.png +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/media/benchmark-mps.png +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/pyproject.toml +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/requirements.txt +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/scripts/build_shaders.py +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/scripts/config.py +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/scripts/pack_shaders.py +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/decompositions.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/guard.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/hook.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/binary.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/bitwise.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/compare.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/complex.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/conv.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/ctc.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/distributions.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/dropout.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/fft.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/fill.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/grid_sample.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/helpers.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/indexing.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/internal.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/linalg.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/nllloss.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/nn.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/pool.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/random.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/reduce.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/sampling.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/scan.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/scatter.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/search.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/shape.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/sort.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/special.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/unary.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/upsample.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/tensor/allocation.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/tensor/cast.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/tensor/views.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/CMakeLists.txt +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/core/cast.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/core/dtype.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/core/philox.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/core/special_functions.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/cholesky.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/conv.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/eig.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/householder.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/jacobi.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/lu.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/matmul_coop.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/matmul_simd.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/unary_ops.slang +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/allocator.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/builders.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/cache.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/cache.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/dispatch.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/dispatch.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/generator.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/generator.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/memory.cpp +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/memory.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/vulkan_context.h +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/tests/test_pytorch_ops.py +0 -0
- {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/tox.ini +0 -0
|
@@ -57,10 +57,53 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
|
|
|
57
57
|
for (DeviceContext* device : VulkanContext::Instance().devices) device->shader_manager->savePipelineCache();
|
|
58
58
|
});
|
|
59
59
|
|
|
60
|
+
m.def("_amp_supported_dtypes", [](int64_t device_index) {
|
|
61
|
+
DeviceContext* device = VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index));
|
|
62
|
+
std::vector<py::object> dtypes;
|
|
63
|
+
if (device->support_float16) dtypes.push_back(py::reinterpret_borrow<py::object>(reinterpret_cast<PyObject*>(torch::getTHPDtype(at::kHalf))));
|
|
64
|
+
if (device->support_bfloat16) dtypes.push_back(py::reinterpret_borrow<py::object>(reinterpret_cast<PyObject*>(torch::getTHPDtype(at::kBFloat16))));
|
|
65
|
+
return dtypes;
|
|
66
|
+
});
|
|
67
|
+
|
|
60
68
|
m.def("_memory_usage", [](int64_t device_index) -> uint64_t {
|
|
61
69
|
return VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index))->memoryUsage();
|
|
62
70
|
});
|
|
63
71
|
|
|
72
|
+
m.def("_memory_allocated", [](int64_t device_index) -> uint64_t {
|
|
73
|
+
return VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index))->allocated_bytes.load();
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
m.def("_max_memory_allocated", [](int64_t device_index) -> uint64_t {
|
|
77
|
+
return VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index))->peak_allocated_bytes.load();
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
m.def("_reset_peak_memory_stats", [](int64_t device_index) {
|
|
81
|
+
DeviceContext* device = VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index));
|
|
82
|
+
device->peak_allocated_bytes.store(device->allocated_bytes.load());
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
m.def("_is_initialized", []() -> bool {
|
|
86
|
+
return VulkanContext::IsInitialized();
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
m.def("_lazy_init", []() {
|
|
90
|
+
VulkanContext::Instance();
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
m.def("_device_properties", [](int64_t device_index) {
|
|
94
|
+
DeviceContext* device = VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index));
|
|
95
|
+
const VkPhysicalDeviceProperties& props = device->properties;
|
|
96
|
+
py::dict info;
|
|
97
|
+
info["name"] = std::string(props.deviceName);
|
|
98
|
+
info["total_memory"] = device->memoryTotal();
|
|
99
|
+
info["vendor_id"] = props.vendorID;
|
|
100
|
+
info["device_id"] = props.deviceID;
|
|
101
|
+
info["api_version"] = std::to_string(VK_API_VERSION_MAJOR(props.apiVersion)) + "." + std::to_string(VK_API_VERSION_MINOR(props.apiVersion)) + "." + std::to_string(VK_API_VERSION_PATCH(props.apiVersion));
|
|
102
|
+
info["driver_version"] = props.driverVersion;
|
|
103
|
+
info["subgroup_size"] = device->subgroup_size;
|
|
104
|
+
return info;
|
|
105
|
+
});
|
|
106
|
+
|
|
64
107
|
m.def("_validation_error_count", []() -> uint64_t {
|
|
65
108
|
return VulkanContext::ValidationErrorCount();
|
|
66
109
|
});
|
|
@@ -236,6 +236,10 @@ at::Tensor dispatch_matmul_simd_shader(
|
|
|
236
236
|
std::vector<at::Tensor> readTensors = {ops.self_b, ops.other_b};
|
|
237
237
|
if (has_bias) readTensors.push_back(ops.bias_b);
|
|
238
238
|
|
|
239
|
+
// a 1D grid past the limit would be wrapped into rows by dispatchShader, which the tiles don't index for
|
|
240
|
+
TORCH_CHECK((N + TILE_N - 1) / TILE_N <= device->properties.limits.maxComputeWorkGroupCount[0],
|
|
241
|
+
"torchvulkan [ERROR]: matmul with N = ", N, " needs more column tiles than the device allows.");
|
|
242
|
+
|
|
239
243
|
// the batch is the grid's z dimension, so it goes in chunks of the device's limit
|
|
240
244
|
const int64_t max_batch = device->properties.limits.maxComputeWorkGroupCount[2];
|
|
241
245
|
for (int64_t b0 = 0; b0 < B; b0 += max_batch)
|
|
@@ -406,6 +410,9 @@ at::Tensor dispatch_matmul_coop_shader(
|
|
|
406
410
|
.push(static_cast<uint32_t>(params.use_shared_memory));
|
|
407
411
|
SpecializationArgs specialization = spd.build();
|
|
408
412
|
|
|
413
|
+
TORCH_CHECK(N_padded / tile_n <= device->properties.limits.maxComputeWorkGroupCount[0],
|
|
414
|
+
"torchvulkan [ERROR]: matmul with N = ", N, " needs more column tiles than the device allows.");
|
|
415
|
+
|
|
409
416
|
const int64_t max_batch = device->properties.limits.maxComputeWorkGroupCount[2];
|
|
410
417
|
for (int64_t b0 = 0; b0 < B; b0 += max_batch)
|
|
411
418
|
{
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
#include <torch/extension.h>
|
|
2
|
+
#include <ATen/autocast_mode.h>
|
|
3
|
+
#include <ATen/native/CPUFallback.h>
|
|
4
|
+
#include "vulkan/vulkan_context.h"
|
|
5
|
+
|
|
6
|
+
void vulkan_cpu_fallback(const c10::OperatorHandle& op, torch::jit::Stack* stack)
|
|
7
|
+
{
|
|
8
|
+
VulkanContext& context = VulkanContext::Instance();
|
|
9
|
+
if (context.isStrict()) TORCH_CHECK(false, "torchvulkan [NOT IMPLEMENTED]: Silent fallback detected for operation: ", op.schema().operator_name(), ". Set TORCHVULKAN_STRICT=0 to disable strict mode.");
|
|
10
|
+
TORCH_WARN_ONCE("torchvulkan [NOT IMPLEMENTED]: Silent fallback detected for operation: ", op.schema().operator_name(), ". Set TORCHVULKAN_STRICT=1 to enable strict mode.");
|
|
11
|
+
at::native::cpu_fallback(op, stack);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
TORCH_LIBRARY_IMPL(_, PrivateUse1, m) {
|
|
15
|
+
m.fallback(torch::CppFunction::makeFromBoxedFunction<&vulkan_cpu_fallback>());
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
TORCH_LIBRARY_IMPL(_, AutocastPrivateUse1, m) {
|
|
19
|
+
m.fallback(torch::CppFunction::makeFallthrough());
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
namespace at::autocast {
|
|
23
|
+
|
|
24
|
+
TORCH_LIBRARY_IMPL(aten, AutocastPrivateUse1, m) {
|
|
25
|
+
#define _KERNEL_PRIVATEUSEONE_LOW_PRECISION_FP(...) KERNEL_PRIVATEUSEONE(__VA_ARGS__, lower_precision_fp)
|
|
26
|
+
AT_FORALL_LOWER_PRECISION_FP(_KERNEL_PRIVATEUSEONE_LOW_PRECISION_FP)
|
|
27
|
+
#define _KERNEL_PRIVATEUSEONE_FP32(...) KERNEL_PRIVATEUSEONE(__VA_ARGS__, fp32)
|
|
28
|
+
AT_FORALL_FP32(_KERNEL_PRIVATEUSEONE_FP32)
|
|
29
|
+
#define _KERNEL_PRIVATEUSEONE_FP32_SET_OPT_DTYPE(...) KERNEL_PRIVATEUSEONE(__VA_ARGS__, fp32_set_opt_dtype)
|
|
30
|
+
AT_FORALL_FP32_SET_OPT_DTYPE(_KERNEL_PRIVATEUSEONE_FP32_SET_OPT_DTYPE)
|
|
31
|
+
AT_FORALL_DIFFERENT_REDISPATCH_SIGNATURE(KERNEL_DIFFERENT_REDISPATCH_SIGNATURE_PRIVATEUSEONE)
|
|
32
|
+
#define _KERNEL_PRIVATEUSEONE_PROMOTE(...) KERNEL_PRIVATEUSEONE(__VA_ARGS__, promote)
|
|
33
|
+
AT_FORALL_PROMOTE(_KERNEL_PRIVATEUSEONE_PROMOTE)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
}
|
|
@@ -155,6 +155,14 @@ at::Tensor torchvulkan::copy_vulkan(
|
|
|
155
155
|
bool /* non_blocking */)
|
|
156
156
|
{
|
|
157
157
|
if (dst.numel() == 0) return dst;
|
|
158
|
+
|
|
159
|
+
// as at::native::copy_, copying a tensor onto its own memory is a no-op (and vkCmdCopyBuffer forbids overlapping regions)
|
|
160
|
+
if (self.is_alias_of(dst) && self.storage_offset() == dst.storage_offset() && self.sizes().equals(dst.sizes()) &&
|
|
161
|
+
self.strides().equals(dst.strides()) && self.scalar_type() == dst.scalar_type() &&
|
|
162
|
+
self.is_conj() == dst.is_conj() && self.is_neg() == dst.is_neg()) {
|
|
163
|
+
return dst;
|
|
164
|
+
}
|
|
165
|
+
|
|
158
166
|
TORCH_CHECK(!dst.is_conj() && !dst.is_neg(), "torchvulkan [NOT IMPLEMENTED]: Copying into a conjugated or negated destination is not yet supported.");
|
|
159
167
|
|
|
160
168
|
// a lazily conjugated or negated source copies its stored values, then the copy applies the pending op
|
|
@@ -218,6 +226,13 @@ at::Scalar local_scalar_dense_vulkan(const at::Tensor& self)
|
|
|
218
226
|
return self.cpu().item();
|
|
219
227
|
}
|
|
220
228
|
|
|
229
|
+
// without this, Module.to("vulkan") replaces each Parameter instead of setting .data, which unties shared weights
|
|
230
|
+
bool has_compatible_shallow_copy_type_vulkan(const at::Tensor& self, const at::Tensor& from)
|
|
231
|
+
{
|
|
232
|
+
auto dense = [](const at::Tensor& t) { return t.layout() == at::kStrided && (t.is_cpu() || t.is_privateuseone()); };
|
|
233
|
+
return self.unsafeGetTensorImpl()->has_compatible_shallow_copy_type(from.key_set()) || (dense(self) && dense(from));
|
|
234
|
+
}
|
|
235
|
+
|
|
221
236
|
} // namespace
|
|
222
237
|
|
|
223
238
|
TORCH_LIBRARY_IMPL(aten, PrivateUse1, m) {
|
|
@@ -225,4 +240,5 @@ TORCH_LIBRARY_IMPL(aten, PrivateUse1, m) {
|
|
|
225
240
|
m.impl("_copy_from_and_resize", ©_from_and_resize_vulkan);
|
|
226
241
|
m.impl("copy_", ©_vulkan_);
|
|
227
242
|
m.impl("_local_scalar_dense", &local_scalar_dense_vulkan);
|
|
243
|
+
m.impl("_has_compatible_shallow_copy_type", &has_compatible_shallow_copy_type_vulkan);
|
|
228
244
|
}
|
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
module indexing;
|
|
2
2
|
|
|
3
|
+
// largest power of two under the maximum vulkan workgroup size of 65535
|
|
4
|
+
public static const uint GROUP_ROW = 32768;
|
|
5
|
+
|
|
6
|
+
public uint flat_group(uint3 group)
|
|
7
|
+
{
|
|
8
|
+
return group.y * GROUP_ROW + group.x;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
public uint flat_thread(uint3 gi, uint workgroupSize)
|
|
12
|
+
{
|
|
13
|
+
return gi.y * GROUP_ROW * workgroupSize + gi.x;
|
|
14
|
+
}
|
|
15
|
+
|
|
3
16
|
/*
|
|
4
17
|
Turns a linear element index into coordinates, one dimension at a time:
|
|
5
18
|
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
module ctc;
|
|
2
2
|
|
|
3
|
+
import core.indexing;
|
|
4
|
+
|
|
3
5
|
[[vk::constant_id(0)]] public const uint MODE = 0;
|
|
4
6
|
[[vk::constant_id(1)]] public const uint WG_x = 0;
|
|
5
7
|
|
|
@@ -197,7 +199,9 @@ void ctc_main<T : __BuiltinFloatingPointType>(
|
|
|
197
199
|
uint3 gi : SV_DispatchThreadID,
|
|
198
200
|
uniform CtcPushConstants pcs)
|
|
199
201
|
{
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
202
|
+
uint b = flat_group(group);
|
|
203
|
+
if (MODE != 2 && b >= pcs.batch) return;
|
|
204
|
+
if (MODE == 0) alphaKernel<T>(b, thread.x, pcs);
|
|
205
|
+
else if (MODE == 1) betaKernel<T>(b, thread.x, pcs);
|
|
206
|
+
else gradKernel<T>(uint64_t(flat_thread(gi, WG_x)), pcs);
|
|
203
207
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module distribution;
|
|
2
2
|
|
|
3
3
|
__exported import core.philox;
|
|
4
|
+
import core.indexing;
|
|
4
5
|
|
|
5
6
|
[[vk::constant_id(0)]] public const uint OP_TYPE = 0;
|
|
6
7
|
[[vk::constant_id(1)]] public const uint WG_x = 0;
|
|
@@ -213,7 +214,7 @@ public void baseDistributionKernel<T : IRandomDtype, Op : IDistributionOp<T.Real
|
|
|
213
214
|
{
|
|
214
215
|
typealias R = T.Real;
|
|
215
216
|
|
|
216
|
-
uint index = gi
|
|
217
|
+
uint index = flat_thread(gi, WG_x);
|
|
217
218
|
if (index >= pcs.numel) return;
|
|
218
219
|
|
|
219
220
|
var stream = RandomStream<R>(pcs.seed, pcs.offset, index);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module dropout;
|
|
2
2
|
|
|
3
3
|
__exported import core.philox;
|
|
4
|
+
import core.indexing;
|
|
4
5
|
|
|
5
6
|
[[vk::constant_id(0)]] public const uint WG_x = 0;
|
|
6
7
|
|
|
@@ -24,10 +25,11 @@ void dropout_main<T : IRandomDtype, let VEC_SIZE : int>(
|
|
|
24
25
|
{
|
|
25
26
|
typealias R = T.Real;
|
|
26
27
|
|
|
27
|
-
uint
|
|
28
|
+
uint stream_id = flat_thread(gi, WG_x);
|
|
29
|
+
uint first = stream_id * VEC_SIZE;
|
|
28
30
|
if (first >= pcs.numel) return;
|
|
29
31
|
|
|
30
|
-
var stream = RandomStream<R>(pcs.seed, pcs.offset,
|
|
32
|
+
var stream = RandomStream<R>(pcs.seed, pcs.offset, stream_id);
|
|
31
33
|
R keepProb = R.fromBits(pcs.keep_prob);
|
|
32
34
|
R scale = R.fromBits(pcs.scale);
|
|
33
35
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module embedding_backward;
|
|
2
2
|
|
|
3
3
|
__exported import core.dtype;
|
|
4
|
+
import core.indexing;
|
|
4
5
|
|
|
5
6
|
[[vk::constant_id(0)]] public const uint WG_x = 0;
|
|
6
7
|
|
|
@@ -19,7 +20,7 @@ public struct EmbeddingBackwardPushConstants
|
|
|
19
20
|
|
|
20
21
|
public void embeddingBackwardKernel<T : ITorchVulkan>(uint3 gi, EmbeddingBackwardPushConstants pcs)
|
|
21
22
|
{
|
|
22
|
-
uint idx = gi
|
|
23
|
+
uint idx = flat_thread(gi, WG_x);
|
|
23
24
|
if (idx >= pcs.numel_out) return;
|
|
24
25
|
|
|
25
26
|
T* grad = (T*)pcs.grad;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
module fft;
|
|
2
2
|
|
|
3
|
+
import core.indexing;
|
|
4
|
+
|
|
3
5
|
[[vk::constant_id(0)]] public const uint MODE = 0;
|
|
4
6
|
[[vk::constant_id(1)]] public const uint INVERSE = 0;
|
|
5
7
|
[[vk::constant_id(2)]] public const uint WG_x = 0;
|
|
@@ -110,5 +112,5 @@ void fft_main<T : __BuiltinFloatingPointType>(
|
|
|
110
112
|
uint3 gi : SV_DispatchThreadID,
|
|
111
113
|
uniform FftPushConstants pcs)
|
|
112
114
|
{
|
|
113
|
-
fftKernel<T>(uint64_t(gi
|
|
115
|
+
fftKernel<T>(uint64_t(flat_thread(gi, WG_x)), pcs);
|
|
114
116
|
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
module grid_sample;
|
|
2
2
|
|
|
3
|
+
import core.indexing;
|
|
4
|
+
|
|
3
5
|
[[vk::constant_id(0)]] public const uint MODE = 0;
|
|
4
6
|
[[vk::constant_id(1)]] public const uint DIMS = 2;
|
|
5
7
|
[[vk::constant_id(2)]] public const uint INTERPOLATION = 0;
|
|
@@ -258,7 +260,7 @@ void inputGradKernel<T : __BuiltinFloatingPointType>(uint64_t idx, Geometry g, G
|
|
|
258
260
|
|
|
259
261
|
void gridSampleKernel<T : __BuiltinFloatingPointType>(uint3 gi, GridSamplePushConstants pcs)
|
|
260
262
|
{
|
|
261
|
-
uint64_t idx = uint64_t(gi
|
|
263
|
+
uint64_t idx = uint64_t(flat_thread(gi, WG_x));
|
|
262
264
|
if (idx >= pcs.count) return;
|
|
263
265
|
Geometry g = loadGeometry(pcs.metadata);
|
|
264
266
|
if (MODE == 3)
|
|
@@ -85,7 +85,7 @@ T evalIntegerBinary<T : __BuiltinIntegerType>(T a, T b)
|
|
|
85
85
|
|
|
86
86
|
public void integerBinaryKernel<T : __BuiltinIntegerType>(uint3 gi, IntegerBinaryPushConstants<T> pcs)
|
|
87
87
|
{
|
|
88
|
-
uint idx = gi
|
|
88
|
+
uint idx = flat_thread(gi, WG_x);
|
|
89
89
|
if (idx >= pcs.numel) return;
|
|
90
90
|
|
|
91
91
|
T* tensor_a = (T*)pcs.tensor_a;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
module ldl;
|
|
2
2
|
|
|
3
|
+
import core.indexing;
|
|
4
|
+
|
|
3
5
|
[[vk::constant_id(0)]] public const uint MODE = 0;
|
|
4
6
|
[[vk::constant_id(1)]] public const uint WG_x = 0;
|
|
5
7
|
|
|
@@ -245,7 +247,7 @@ void ldl_main<T : __BuiltinFloatingPointType>(
|
|
|
245
247
|
uint3 gi : SV_DispatchThreadID,
|
|
246
248
|
uniform LdlPushConstants pcs)
|
|
247
249
|
{
|
|
248
|
-
uint64_t idx = uint64_t(gi
|
|
250
|
+
uint64_t idx = uint64_t(flat_thread(gi, WG_x));
|
|
249
251
|
if (idx >= pcs.count) return;
|
|
250
252
|
if (MODE == 0) factor<T>(int64_t(idx), pcs);
|
|
251
253
|
else solve<T>(int64_t(idx) / int64_t(pcs.k), int(int64_t(idx) % int64_t(pcs.k)), pcs);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module multinomial;
|
|
2
2
|
|
|
3
3
|
__exported import core.philox;
|
|
4
|
+
import core.indexing;
|
|
4
5
|
|
|
5
6
|
[[vk::constant_id(0)]] public const uint WG_x = 0;
|
|
6
7
|
|
|
@@ -21,7 +22,7 @@ void multinomial_main<R : __BuiltinFloatingPointType & IRandomReal>(
|
|
|
21
22
|
uint3 gi : SV_DispatchThreadID,
|
|
22
23
|
uniform MultinomialPushConstants pcs)
|
|
23
24
|
{
|
|
24
|
-
uint index = gi
|
|
25
|
+
uint index = flat_thread(gi, WG_x);
|
|
25
26
|
if (index >= pcs.numel) return;
|
|
26
27
|
|
|
27
28
|
R* cdf = (R*)pcs.cdf + (index / pcs.samples) * pcs.categories;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module nllloss;
|
|
2
2
|
|
|
3
3
|
__exported import core.dtype;
|
|
4
|
+
import core.indexing;
|
|
4
5
|
|
|
5
6
|
[[vk::constant_id(0)]] public const uint MODE = 0;
|
|
6
7
|
[[vk::constant_id(1)]] public const uint WG_x = 0;
|
|
@@ -71,9 +72,9 @@ public void dispatchNllLoss<T : ITorchVulkan>(uint3 gi, NllLossPushConstants pcs
|
|
|
71
72
|
|
|
72
73
|
switch (MODE)
|
|
73
74
|
{
|
|
74
|
-
case 0: ForwardGatherOp.apply<T>(gi
|
|
75
|
-
case 1: BackwardScatterOp.apply<T>(gi
|
|
76
|
-
case 2: WeightGatherOp.apply<T>(gi
|
|
75
|
+
case 0: ForwardGatherOp.apply<T>(flat_thread(gi, WG_x), tensor_in, target, tensor_out, pcs); break;
|
|
76
|
+
case 1: BackwardScatterOp.apply<T>(flat_thread(gi, WG_x), tensor_in, target, tensor_out, pcs); break;
|
|
77
|
+
case 2: WeightGatherOp.apply<T>(flat_thread(gi, WG_x), tensor_in, target, tensor_out, pcs); break;
|
|
77
78
|
default: break;
|
|
78
79
|
}
|
|
79
80
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module pool;
|
|
2
2
|
|
|
3
3
|
__exported import core.dtype;
|
|
4
|
+
import core.indexing;
|
|
4
5
|
|
|
5
6
|
[[vk::constant_id(0)]] public const uint MODE = 0;
|
|
6
7
|
[[vk::constant_id(1)]] public const uint WG_x = 0;
|
|
@@ -273,7 +274,7 @@ void poolBackward<T : ITorchVulkan>(uint64_t idx, PoolPushConstants pcs, Geometr
|
|
|
273
274
|
|
|
274
275
|
void poolKernel<T : ITorchVulkan>(uint3 gi, PoolPushConstants pcs)
|
|
275
276
|
{
|
|
276
|
-
uint64_t idx = uint64_t(gi
|
|
277
|
+
uint64_t idx = uint64_t(flat_thread(gi, WG_x));
|
|
277
278
|
if (idx >= pcs.count) return;
|
|
278
279
|
Geometry g = loadGeometry(pcs.metadata);
|
|
279
280
|
if (MODE == 5 || MODE == 6) poolBackward<T>(idx, pcs, g);
|
|
@@ -134,10 +134,11 @@ public struct RandomIntOp<T : IRandomDtype> : IRandomOp<T>
|
|
|
134
134
|
|
|
135
135
|
public void baseRandomKernel<T : IRandomDtype, Op : IRandomOp<T>, let VEC_SIZE : int>(uint3 gi, RandomPushConstants pcs)
|
|
136
136
|
{
|
|
137
|
-
uint
|
|
137
|
+
uint stream_id = flat_thread(gi, WG_x);
|
|
138
|
+
uint first = stream_id * VEC_SIZE;
|
|
138
139
|
if (first >= pcs.numel) return;
|
|
139
140
|
|
|
140
|
-
var stream = RandomStream<T.Real>(pcs.seed, pcs.offset,
|
|
141
|
+
var stream = RandomStream<T.Real>(pcs.seed, pcs.offset, stream_id);
|
|
141
142
|
T* tensor = (T*)pcs.tensor;
|
|
142
143
|
|
|
143
144
|
for (int k = 0; k < VEC_SIZE; k++)
|
|
@@ -108,5 +108,5 @@ void reduce_subgroup_main<T : __BuiltinFloatingPointType>(
|
|
|
108
108
|
uint3 group_thread_id : SV_GroupThreadID,
|
|
109
109
|
uniform SubgroupReducePushConstants pcs)
|
|
110
110
|
{
|
|
111
|
-
dispatchSubgroupReduce<T>(group_id
|
|
111
|
+
dispatchSubgroupReduce<T>(flat_group(group_id), group_thread_id.x, pcs);
|
|
112
112
|
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
module resample;
|
|
2
2
|
|
|
3
|
+
import core.indexing;
|
|
4
|
+
|
|
3
5
|
[[vk::constant_id(0)]] public const uint MODE = 0;
|
|
4
6
|
[[vk::constant_id(1)]] public const uint BACKWARD = 0;
|
|
5
7
|
[[vk::constant_id(2)]] public const uint ALIGN_CORNERS = 0;
|
|
@@ -154,7 +156,7 @@ Taps<T> computeTaps<T : __BuiltinFloatingPointType>(int o, int in_length, int ou
|
|
|
154
156
|
|
|
155
157
|
void resampleKernel<T : __BuiltinFloatingPointType>(uint3 gi, ResamplePushConstants<T> pcs)
|
|
156
158
|
{
|
|
157
|
-
uint64_t idx = uint64_t(gi
|
|
159
|
+
uint64_t idx = uint64_t(flat_thread(gi, WG_x));
|
|
158
160
|
if (idx >= pcs.count) return;
|
|
159
161
|
|
|
160
162
|
T* input = (T*)pcs.input;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module resample_u8;
|
|
2
2
|
|
|
3
3
|
__exported import core.dtype;
|
|
4
|
+
import core.indexing;
|
|
4
5
|
|
|
5
6
|
[[vk::constant_id(0)]] public const uint WG_x = 0;
|
|
6
7
|
|
|
@@ -49,5 +50,5 @@ void resample_u8_main<T : ITorchVulkan>(
|
|
|
49
50
|
uint3 gi : SV_DispatchThreadID,
|
|
50
51
|
uniform ResampleU8PushConstants pcs)
|
|
51
52
|
{
|
|
52
|
-
resampleU8Kernel<T>(uint64_t(gi
|
|
53
|
+
resampleU8Kernel<T>(uint64_t(flat_thread(gi, WG_x)), pcs);
|
|
53
54
|
}
|
|
@@ -135,5 +135,5 @@ void scan_main<T : ITorchVulkan>(
|
|
|
135
135
|
uniform ScanPushConstants pcs)
|
|
136
136
|
{
|
|
137
137
|
static groupshared T smem[SCAN_SMEM_SIZE];
|
|
138
|
-
dispatchScan<T, SCAN_SMEM_SIZE>(group_id
|
|
138
|
+
dispatchScan<T, SCAN_SMEM_SIZE>(flat_group(group_id), group_thread_id.x, smem, pcs);
|
|
139
139
|
}
|
|
@@ -136,7 +136,7 @@ void scan_arg_main<T : ITorchVulkan>(
|
|
|
136
136
|
static groupshared uint smem_idx[SCAN_ARG_SMEM_SIZE];
|
|
137
137
|
|
|
138
138
|
scanArgKernel<T, SCAN_ARG_SMEM_SIZE>(
|
|
139
|
-
group_id
|
|
139
|
+
flat_group(group_id), group_thread_id.x, smem_val, smem_idx,
|
|
140
140
|
pcs.tensor_in, pcs.tensor_val, pcs.tensor_idx, pcs.metadata,
|
|
141
141
|
pcs.num_lines, pcs.scan_size, pcs.scan_stride_in, pcs.scan_stride_out
|
|
142
142
|
);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module scatter;
|
|
2
2
|
|
|
3
3
|
__exported import core.dtype;
|
|
4
|
+
import core.indexing;
|
|
4
5
|
|
|
5
6
|
[[vk::constant_id(0)]] public const uint MODE = 0;
|
|
6
7
|
[[vk::constant_id(1)]] public const uint INCLUDE_SELF = 0;
|
|
@@ -55,7 +56,7 @@ T identity<T : ITorchVulkan>()
|
|
|
55
56
|
|
|
56
57
|
public void scatterKernel<T : ITorchVulkan>(uint3 gi, ScatterPushConstants pcs)
|
|
57
58
|
{
|
|
58
|
-
uint64_t i = uint64_t(gi
|
|
59
|
+
uint64_t i = uint64_t(flat_thread(gi, WG_x));
|
|
59
60
|
if (i >= pcs.count) return;
|
|
60
61
|
|
|
61
62
|
T* out = (T*)pcs.out;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module searchsorted;
|
|
2
2
|
|
|
3
3
|
__exported import core.dtype;
|
|
4
|
+
import core.indexing;
|
|
4
5
|
|
|
5
6
|
[[vk::constant_id(0)]] public const uint RIGHT = 0;
|
|
6
7
|
[[vk::constant_id(1)]] public const uint HAS_SORTER = 0;
|
|
@@ -22,7 +23,7 @@ public struct SearchsortedPushConstants
|
|
|
22
23
|
// torch's lower and upper bound, so NaNs land past the end
|
|
23
24
|
public void searchsortedKernel<T : ITorchVulkan>(uint3 gi, SearchsortedPushConstants pcs)
|
|
24
25
|
{
|
|
25
|
-
uint64_t idx = uint64_t(gi
|
|
26
|
+
uint64_t idx = uint64_t(flat_thread(gi, WG_x));
|
|
26
27
|
if (idx >= pcs.count) return;
|
|
27
28
|
|
|
28
29
|
T* sorted = (T*)pcs.sorted;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module sort;
|
|
2
2
|
|
|
3
3
|
__exported import core.dtype;
|
|
4
|
+
import core.indexing;
|
|
4
5
|
|
|
5
6
|
[[vk::constant_id(0)]] public const uint MODE = 0;
|
|
6
7
|
[[vk::constant_id(1)]] public const uint DESCENDING = 0;
|
|
@@ -49,7 +50,7 @@ bool sortsBefore<T : ITorchVulkan>(T x, int64_t px, T y, int64_t py, uint n)
|
|
|
49
50
|
|
|
50
51
|
public void sortKernel<T : ITorchVulkan>(uint3 gi, SortPushConstants pcs)
|
|
51
52
|
{
|
|
52
|
-
uint idx = gi
|
|
53
|
+
uint idx = flat_thread(gi, WG_x);
|
|
53
54
|
if (uint64_t(idx) >= pcs.threads) return;
|
|
54
55
|
|
|
55
56
|
T* keys_out = (T*)pcs.keys_out;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
module triangular_solve;
|
|
2
2
|
|
|
3
|
+
import core.indexing;
|
|
4
|
+
|
|
3
5
|
[[vk::constant_id(0)]] public const uint UPPER = 0;
|
|
4
6
|
[[vk::constant_id(1)]] public const uint UNIT_DIAGONAL = 0;
|
|
5
7
|
[[vk::constant_id(2)]] public const uint WG_x = 0;
|
|
@@ -50,5 +52,5 @@ void triangular_solve_main<T : __BuiltinFloatingPointType>(
|
|
|
50
52
|
uint3 gi : SV_DispatchThreadID,
|
|
51
53
|
uniform TriangularSolvePushConstants pcs)
|
|
52
54
|
{
|
|
53
|
-
triangularSolveKernel<T>(uint64_t(gi
|
|
55
|
+
triangularSolveKernel<T>(uint64_t(flat_thread(gi, WG_x)), pcs);
|
|
54
56
|
}
|
|
@@ -11,7 +11,9 @@ c10::DataPtr VulkanAllocator::allocate(size_t nbytes)
|
|
|
11
11
|
c10::Device device(c10::DeviceType::PrivateUse1, VulkanContext::CurrentDevice());
|
|
12
12
|
if (nbytes == 0) return {nullptr, nullptr, &VulkanAllocator::deleter, device};
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
DeviceContext* context = VulkanContext::Instance().CurrentDeviceContext();
|
|
15
|
+
VulkanBuffer* buffer = allocateBuffer(context, nbytes, MemoryUsage::DEVICE_ONLY);
|
|
16
|
+
context->recordAllocation(buffer->size());
|
|
15
17
|
return {buffer, buffer, &VulkanAllocator::deleter, device};
|
|
16
18
|
}
|
|
17
19
|
|
|
@@ -19,6 +21,7 @@ void VulkanAllocator::deleter(void* ptr)
|
|
|
19
21
|
{
|
|
20
22
|
if (ptr == nullptr || VulkanContext::IsInBadFork()) return;
|
|
21
23
|
VulkanBuffer* buffer = static_cast<VulkanBuffer*>(ptr);
|
|
24
|
+
buffer->owner()->recordFree(buffer->size());
|
|
22
25
|
buffer->owner()->cache.deleteBuffer(buffer);
|
|
23
26
|
}
|
|
24
27
|
|