torchvulkan 0.2.0a0__tar.gz → 0.2.0a2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/PKG-INFO +1 -1
  2. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/backend.cpp +43 -0
  3. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/matmul.cpp +7 -0
  4. torchvulkan-0.2.0a2/src/api/registration.cpp +36 -0
  5. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/tensor/transfer.cpp +16 -0
  6. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/core/indexing.slang +13 -0
  7. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/arg_reduce.slang +1 -1
  8. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/binary.slang +1 -1
  9. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/cast.slang +1 -1
  10. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/compare.slang +1 -1
  11. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/copy.slang +1 -1
  12. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/ctc.slang +7 -3
  13. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/distribution.slang +2 -1
  14. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/dropout.slang +4 -2
  15. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/embedding_backward.slang +2 -1
  16. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/fft.slang +3 -1
  17. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/fill.slang +1 -1
  18. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/grid_sample.slang +3 -1
  19. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/index_gather.slang +1 -1
  20. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/integer_binary.slang +1 -1
  21. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/ldl.slang +3 -1
  22. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/multinomial.slang +2 -1
  23. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/nllloss.slang +4 -3
  24. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/pool.slang +2 -1
  25. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/random.slang +3 -2
  26. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/reduce.slang +1 -1
  27. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/reduce_subgroup.slang +1 -1
  28. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/resample.slang +3 -1
  29. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/resample_u8.slang +2 -1
  30. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/scan.slang +1 -1
  31. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/scan_arg.slang +1 -1
  32. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/scatter.slang +2 -1
  33. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/searchsorted.slang +2 -1
  34. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/sort.slang +2 -1
  35. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/special.slang +1 -1
  36. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/triangular_solve.slang +3 -1
  37. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/unary.slang +1 -1
  38. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/where.slang +1 -1
  39. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/allocator.cpp +4 -1
  40. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/device_context.cpp +19 -0
  41. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/device_context.h +8 -0
  42. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/shader.cpp +4 -0
  43. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/shader.h +3 -0
  44. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/vulkan_context.cpp +7 -2
  45. torchvulkan-0.2.0a2/torchvulkan/__init__.py +157 -0
  46. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/torchvulkan/_version.py +3 -3
  47. torchvulkan-0.2.0a0/src/api/registration.cpp +0 -15
  48. torchvulkan-0.2.0a0/torchvulkan/__init__.py +0 -89
  49. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.github/actions/setup-toolchain/action.yml +0 -0
  50. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.github/scripts/fetch-moltenvk-macos.sh +0 -0
  51. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.github/scripts/install-toolchain-linux.sh +0 -0
  52. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.github/workflows/release.yaml +0 -0
  53. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.github/workflows/test_torchvulkan.yml +0 -0
  54. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/.gitignore +0 -0
  55. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/CMakeLists.txt +0 -0
  56. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/LICENSE +0 -0
  57. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/README.md +0 -0
  58. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/benchmark/benchmark.py +0 -0
  59. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/examples/llm.py +0 -0
  60. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/examples/mnist.py +0 -0
  61. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/extern/vma/vk_mem_alloc.h +0 -0
  62. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/extern/volk/volk.c +0 -0
  63. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/extern/volk/volk.h +0 -0
  64. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/licenses/MoltenVK-LICENSE +0 -0
  65. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/media/benchmark-cuda.png +0 -0
  66. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/media/benchmark-mps.png +0 -0
  67. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/pyproject.toml +0 -0
  68. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/requirements.txt +0 -0
  69. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/scripts/build_shaders.py +0 -0
  70. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/scripts/config.py +0 -0
  71. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/scripts/pack_shaders.py +0 -0
  72. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/decompositions.cpp +0 -0
  73. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/guard.cpp +0 -0
  74. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/hook.cpp +0 -0
  75. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/binary.cpp +0 -0
  76. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/bitwise.cpp +0 -0
  77. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/compare.cpp +0 -0
  78. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/complex.cpp +0 -0
  79. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/conv.cpp +0 -0
  80. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/ctc.cpp +0 -0
  81. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/distributions.cpp +0 -0
  82. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/dropout.cpp +0 -0
  83. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/fft.cpp +0 -0
  84. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/fill.cpp +0 -0
  85. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/grid_sample.cpp +0 -0
  86. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/helpers.h +0 -0
  87. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/indexing.cpp +0 -0
  88. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/internal.h +0 -0
  89. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/linalg.cpp +0 -0
  90. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/nllloss.cpp +0 -0
  91. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/nn.cpp +0 -0
  92. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/pool.cpp +0 -0
  93. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/random.cpp +0 -0
  94. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/reduce.cpp +0 -0
  95. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/sampling.cpp +0 -0
  96. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/scan.cpp +0 -0
  97. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/scatter.cpp +0 -0
  98. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/search.cpp +0 -0
  99. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/shape.cpp +0 -0
  100. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/sort.cpp +0 -0
  101. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/special.cpp +0 -0
  102. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/unary.cpp +0 -0
  103. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/ops/upsample.cpp +0 -0
  104. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/tensor/allocation.cpp +0 -0
  105. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/tensor/cast.cpp +0 -0
  106. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/api/tensor/views.cpp +0 -0
  107. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/CMakeLists.txt +0 -0
  108. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/core/cast.slang +0 -0
  109. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/core/dtype.slang +0 -0
  110. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/core/philox.slang +0 -0
  111. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/core/special_functions.slang +0 -0
  112. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/cholesky.slang +0 -0
  113. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/conv.slang +0 -0
  114. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/eig.slang +0 -0
  115. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/householder.slang +0 -0
  116. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/jacobi.slang +0 -0
  117. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/lu.slang +0 -0
  118. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/matmul_coop.slang +0 -0
  119. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/matmul_simd.slang +0 -0
  120. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/shaders/ops/unary_ops.slang +0 -0
  121. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/allocator.h +0 -0
  122. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/builders.h +0 -0
  123. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/cache.cpp +0 -0
  124. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/cache.h +0 -0
  125. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/dispatch.cpp +0 -0
  126. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/dispatch.h +0 -0
  127. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/generator.cpp +0 -0
  128. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/generator.h +0 -0
  129. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/memory.cpp +0 -0
  130. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/memory.h +0 -0
  131. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/src/vulkan/vulkan_context.h +0 -0
  132. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/tests/test_pytorch_ops.py +0 -0
  133. {torchvulkan-0.2.0a0 → torchvulkan-0.2.0a2}/tox.ini +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: torchvulkan
3
- Version: 0.2.0a0
3
+ Version: 0.2.0a2
4
4
  Summary: Cross-Platform Vulkan Backend for PyTorch
5
5
  Author: Zain Sharief
6
6
  License-Expression: MIT
@@ -57,10 +57,53 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
57
57
  for (DeviceContext* device : VulkanContext::Instance().devices) device->shader_manager->savePipelineCache();
58
58
  });
59
59
 
60
+ m.def("_amp_supported_dtypes", [](int64_t device_index) {
61
+ DeviceContext* device = VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index));
62
+ std::vector<py::object> dtypes;
63
+ if (device->support_float16) dtypes.push_back(py::reinterpret_borrow<py::object>(reinterpret_cast<PyObject*>(torch::getTHPDtype(at::kHalf))));
64
+ if (device->support_bfloat16) dtypes.push_back(py::reinterpret_borrow<py::object>(reinterpret_cast<PyObject*>(torch::getTHPDtype(at::kBFloat16))));
65
+ return dtypes;
66
+ });
67
+
60
68
  m.def("_memory_usage", [](int64_t device_index) -> uint64_t {
61
69
  return VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index))->memoryUsage();
62
70
  });
63
71
 
72
+ m.def("_memory_allocated", [](int64_t device_index) -> uint64_t {
73
+ return VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index))->allocated_bytes.load();
74
+ });
75
+
76
+ m.def("_max_memory_allocated", [](int64_t device_index) -> uint64_t {
77
+ return VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index))->peak_allocated_bytes.load();
78
+ });
79
+
80
+ m.def("_reset_peak_memory_stats", [](int64_t device_index) {
81
+ DeviceContext* device = VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index));
82
+ device->peak_allocated_bytes.store(device->allocated_bytes.load());
83
+ });
84
+
85
+ m.def("_is_initialized", []() -> bool {
86
+ return VulkanContext::IsInitialized();
87
+ });
88
+
89
+ m.def("_lazy_init", []() {
90
+ VulkanContext::Instance();
91
+ });
92
+
93
+ m.def("_device_properties", [](int64_t device_index) {
94
+ DeviceContext* device = VulkanContext::Instance().GetDeviceContext(static_cast<c10::DeviceIndex>(device_index));
95
+ const VkPhysicalDeviceProperties& props = device->properties;
96
+ py::dict info;
97
+ info["name"] = std::string(props.deviceName);
98
+ info["total_memory"] = device->memoryTotal();
99
+ info["vendor_id"] = props.vendorID;
100
+ info["device_id"] = props.deviceID;
101
+ info["api_version"] = std::to_string(VK_API_VERSION_MAJOR(props.apiVersion)) + "." + std::to_string(VK_API_VERSION_MINOR(props.apiVersion)) + "." + std::to_string(VK_API_VERSION_PATCH(props.apiVersion));
102
+ info["driver_version"] = props.driverVersion;
103
+ info["subgroup_size"] = device->subgroup_size;
104
+ return info;
105
+ });
106
+
64
107
  m.def("_validation_error_count", []() -> uint64_t {
65
108
  return VulkanContext::ValidationErrorCount();
66
109
  });
@@ -236,6 +236,10 @@ at::Tensor dispatch_matmul_simd_shader(
236
236
  std::vector<at::Tensor> readTensors = {ops.self_b, ops.other_b};
237
237
  if (has_bias) readTensors.push_back(ops.bias_b);
238
238
 
239
+ // a 1D grid past the limit would be wrapped into rows by dispatchShader, which the tiles don't index for
240
+ TORCH_CHECK((N + TILE_N - 1) / TILE_N <= device->properties.limits.maxComputeWorkGroupCount[0],
241
+ "torchvulkan [ERROR]: matmul with N = ", N, " needs more column tiles than the device allows.");
242
+
239
243
  // the batch is the grid's z dimension, so it goes in chunks of the device's limit
240
244
  const int64_t max_batch = device->properties.limits.maxComputeWorkGroupCount[2];
241
245
  for (int64_t b0 = 0; b0 < B; b0 += max_batch)
@@ -406,6 +410,9 @@ at::Tensor dispatch_matmul_coop_shader(
406
410
  .push(static_cast<uint32_t>(params.use_shared_memory));
407
411
  SpecializationArgs specialization = spd.build();
408
412
 
413
+ TORCH_CHECK(N_padded / tile_n <= device->properties.limits.maxComputeWorkGroupCount[0],
414
+ "torchvulkan [ERROR]: matmul with N = ", N, " needs more column tiles than the device allows.");
415
+
409
416
  const int64_t max_batch = device->properties.limits.maxComputeWorkGroupCount[2];
410
417
  for (int64_t b0 = 0; b0 < B; b0 += max_batch)
411
418
  {
@@ -0,0 +1,36 @@
1
+ #include <torch/extension.h>
2
+ #include <ATen/autocast_mode.h>
3
+ #include <ATen/native/CPUFallback.h>
4
+ #include "vulkan/vulkan_context.h"
5
+
6
+ void vulkan_cpu_fallback(const c10::OperatorHandle& op, torch::jit::Stack* stack)
7
+ {
8
+ VulkanContext& context = VulkanContext::Instance();
9
+ if (context.isStrict()) TORCH_CHECK(false, "torchvulkan [NOT IMPLEMENTED]: Silent fallback detected for operation: ", op.schema().operator_name(), ". Set TORCHVULKAN_STRICT=0 to disable strict mode.");
10
+ TORCH_WARN_ONCE("torchvulkan [NOT IMPLEMENTED]: Silent fallback detected for operation: ", op.schema().operator_name(), ". Set TORCHVULKAN_STRICT=1 to enable strict mode.");
11
+ at::native::cpu_fallback(op, stack);
12
+ }
13
+
14
+ TORCH_LIBRARY_IMPL(_, PrivateUse1, m) {
15
+ m.fallback(torch::CppFunction::makeFromBoxedFunction<&vulkan_cpu_fallback>());
16
+ }
17
+
18
+ TORCH_LIBRARY_IMPL(_, AutocastPrivateUse1, m) {
19
+ m.fallback(torch::CppFunction::makeFallthrough());
20
+ }
21
+
22
+ namespace at::autocast {
23
+
24
+ TORCH_LIBRARY_IMPL(aten, AutocastPrivateUse1, m) {
25
+ #define _KERNEL_PRIVATEUSEONE_LOW_PRECISION_FP(...) KERNEL_PRIVATEUSEONE(__VA_ARGS__, lower_precision_fp)
26
+ AT_FORALL_LOWER_PRECISION_FP(_KERNEL_PRIVATEUSEONE_LOW_PRECISION_FP)
27
+ #define _KERNEL_PRIVATEUSEONE_FP32(...) KERNEL_PRIVATEUSEONE(__VA_ARGS__, fp32)
28
+ AT_FORALL_FP32(_KERNEL_PRIVATEUSEONE_FP32)
29
+ #define _KERNEL_PRIVATEUSEONE_FP32_SET_OPT_DTYPE(...) KERNEL_PRIVATEUSEONE(__VA_ARGS__, fp32_set_opt_dtype)
30
+ AT_FORALL_FP32_SET_OPT_DTYPE(_KERNEL_PRIVATEUSEONE_FP32_SET_OPT_DTYPE)
31
+ AT_FORALL_DIFFERENT_REDISPATCH_SIGNATURE(KERNEL_DIFFERENT_REDISPATCH_SIGNATURE_PRIVATEUSEONE)
32
+ #define _KERNEL_PRIVATEUSEONE_PROMOTE(...) KERNEL_PRIVATEUSEONE(__VA_ARGS__, promote)
33
+ AT_FORALL_PROMOTE(_KERNEL_PRIVATEUSEONE_PROMOTE)
34
+ }
35
+
36
+ }
@@ -155,6 +155,14 @@ at::Tensor torchvulkan::copy_vulkan(
155
155
  bool /* non_blocking */)
156
156
  {
157
157
  if (dst.numel() == 0) return dst;
158
+
159
+ // as at::native::copy_, copying a tensor onto its own memory is a no-op (and vkCmdCopyBuffer forbids overlapping regions)
160
+ if (self.is_alias_of(dst) && self.storage_offset() == dst.storage_offset() && self.sizes().equals(dst.sizes()) &&
161
+ self.strides().equals(dst.strides()) && self.scalar_type() == dst.scalar_type() &&
162
+ self.is_conj() == dst.is_conj() && self.is_neg() == dst.is_neg()) {
163
+ return dst;
164
+ }
165
+
158
166
  TORCH_CHECK(!dst.is_conj() && !dst.is_neg(), "torchvulkan [NOT IMPLEMENTED]: Copying into a conjugated or negated destination is not yet supported.");
159
167
 
160
168
  // a lazily conjugated or negated source copies its stored values, then the copy applies the pending op
@@ -218,6 +226,13 @@ at::Scalar local_scalar_dense_vulkan(const at::Tensor& self)
218
226
  return self.cpu().item();
219
227
  }
220
228
 
229
+ // without this, Module.to("vulkan") replaces each Parameter instead of setting .data, which unties shared weights
230
+ bool has_compatible_shallow_copy_type_vulkan(const at::Tensor& self, const at::Tensor& from)
231
+ {
232
+ auto dense = [](const at::Tensor& t) { return t.layout() == at::kStrided && (t.is_cpu() || t.is_privateuseone()); };
233
+ return self.unsafeGetTensorImpl()->has_compatible_shallow_copy_type(from.key_set()) || (dense(self) && dense(from));
234
+ }
235
+
221
236
  } // namespace
222
237
 
223
238
  TORCH_LIBRARY_IMPL(aten, PrivateUse1, m) {
@@ -225,4 +240,5 @@ TORCH_LIBRARY_IMPL(aten, PrivateUse1, m) {
225
240
  m.impl("_copy_from_and_resize", &copy_from_and_resize_vulkan);
226
241
  m.impl("copy_", &copy_vulkan_);
227
242
  m.impl("_local_scalar_dense", &local_scalar_dense_vulkan);
243
+ m.impl("_has_compatible_shallow_copy_type", &has_compatible_shallow_copy_type_vulkan);
228
244
  }
@@ -1,5 +1,18 @@
1
1
  module indexing;
2
2
 
3
+ // largest power of two under the maximum vulkan workgroup size of 65535
4
+ public static const uint GROUP_ROW = 32768;
5
+
6
+ public uint flat_group(uint3 group)
7
+ {
8
+ return group.y * GROUP_ROW + group.x;
9
+ }
10
+
11
+ public uint flat_thread(uint3 gi, uint workgroupSize)
12
+ {
13
+ return gi.y * GROUP_ROW * workgroupSize + gi.x;
14
+ }
15
+
3
16
  /*
4
17
  Turns a linear element index into coordinates, one dimension at a time:
5
18
 
@@ -32,7 +32,7 @@ public void argReduceKernel<T : ITorchVulkan>(
32
32
  uint reduce_stride,
33
33
  uint reduce_size)
34
34
  {
35
- uint idx = gi.x;
35
+ uint idx = flat_thread(gi, WG_x);
36
36
  if (idx >= num_lines || reduce_size == 0) return;
37
37
 
38
38
  T* tensor_in = (T*)address_in;
@@ -144,7 +144,7 @@ public void baseBinaryKernel<T : ITorchVulkan, Op : IBinaryOp<T>, let VEC_SIZE :
144
144
  T alpha,
145
145
  T scalar_value)
146
146
  {
147
- uint idx = gi.x;
147
+ uint idx = flat_thread(gi, WG_x);
148
148
 
149
149
  if (IS_CONTIG != 0)
150
150
  {
@@ -36,7 +36,7 @@ public void castKernel<Src : ITorchVulkan, Dst : ITorchVulkan, let VEC_SIZE : in
36
36
  uint64_t address_metadata,
37
37
  uint64_t numel)
38
38
  {
39
- uint idx = gi.x;
39
+ uint idx = flat_thread(gi, WG_x);
40
40
 
41
41
  if (IS_CONTIG != 0)
42
42
  {
@@ -52,7 +52,7 @@ public void baseCompareKernel<T : ITorchVulkan, Op : ICompareOp<T>, let VEC_SIZE
52
52
  uint64_t numel,
53
53
  T scalar_value)
54
54
  {
55
- uint idx = gi.x;
55
+ uint idx = flat_thread(gi, WG_x);
56
56
 
57
57
  if (IS_CONTIG != 0)
58
58
  {
@@ -27,7 +27,7 @@ public void copyKernel<T>(
27
27
  uint64_t address_metadata,
28
28
  uint64_t numel)
29
29
  {
30
- uint idx = gi.x;
30
+ uint idx = flat_thread(gi, WG_x);
31
31
  if (idx >= numel) return;
32
32
 
33
33
  T* tensor_in = (T*)address_in;
@@ -1,5 +1,7 @@
1
1
  module ctc;
2
2
 
3
+ import core.indexing;
4
+
3
5
  [[vk::constant_id(0)]] public const uint MODE = 0;
4
6
  [[vk::constant_id(1)]] public const uint WG_x = 0;
5
7
 
@@ -197,7 +199,9 @@ void ctc_main<T : __BuiltinFloatingPointType>(
197
199
  uint3 gi : SV_DispatchThreadID,
198
200
  uniform CtcPushConstants pcs)
199
201
  {
200
- if (MODE == 0) alphaKernel<T>(group.x, thread.x, pcs);
201
- else if (MODE == 1) betaKernel<T>(group.x, thread.x, pcs);
202
- else gradKernel<T>(uint64_t(gi.x), pcs);
202
+ uint b = flat_group(group);
203
+ if (MODE != 2 && b >= pcs.batch) return;
204
+ if (MODE == 0) alphaKernel<T>(b, thread.x, pcs);
205
+ else if (MODE == 1) betaKernel<T>(b, thread.x, pcs);
206
+ else gradKernel<T>(uint64_t(flat_thread(gi, WG_x)), pcs);
203
207
  }
@@ -1,6 +1,7 @@
1
1
  module distribution;
2
2
 
3
3
  __exported import core.philox;
4
+ import core.indexing;
4
5
 
5
6
  [[vk::constant_id(0)]] public const uint OP_TYPE = 0;
6
7
  [[vk::constant_id(1)]] public const uint WG_x = 0;
@@ -213,7 +214,7 @@ public void baseDistributionKernel<T : IRandomDtype, Op : IDistributionOp<T.Real
213
214
  {
214
215
  typealias R = T.Real;
215
216
 
216
- uint index = gi.x;
217
+ uint index = flat_thread(gi, WG_x);
217
218
  if (index >= pcs.numel) return;
218
219
 
219
220
  var stream = RandomStream<R>(pcs.seed, pcs.offset, index);
@@ -1,6 +1,7 @@
1
1
  module dropout;
2
2
 
3
3
  __exported import core.philox;
4
+ import core.indexing;
4
5
 
5
6
  [[vk::constant_id(0)]] public const uint WG_x = 0;
6
7
 
@@ -24,10 +25,11 @@ void dropout_main<T : IRandomDtype, let VEC_SIZE : int>(
24
25
  {
25
26
  typealias R = T.Real;
26
27
 
27
- uint first = gi.x * VEC_SIZE;
28
+ uint stream_id = flat_thread(gi, WG_x);
29
+ uint first = stream_id * VEC_SIZE;
28
30
  if (first >= pcs.numel) return;
29
31
 
30
- var stream = RandomStream<R>(pcs.seed, pcs.offset, gi.x);
32
+ var stream = RandomStream<R>(pcs.seed, pcs.offset, stream_id);
31
33
  R keepProb = R.fromBits(pcs.keep_prob);
32
34
  R scale = R.fromBits(pcs.scale);
33
35
 
@@ -1,6 +1,7 @@
1
1
  module embedding_backward;
2
2
 
3
3
  __exported import core.dtype;
4
+ import core.indexing;
4
5
 
5
6
  [[vk::constant_id(0)]] public const uint WG_x = 0;
6
7
 
@@ -19,7 +20,7 @@ public struct EmbeddingBackwardPushConstants
19
20
 
20
21
  public void embeddingBackwardKernel<T : ITorchVulkan>(uint3 gi, EmbeddingBackwardPushConstants pcs)
21
22
  {
22
- uint idx = gi.x;
23
+ uint idx = flat_thread(gi, WG_x);
23
24
  if (idx >= pcs.numel_out) return;
24
25
 
25
26
  T* grad = (T*)pcs.grad;
@@ -1,5 +1,7 @@
1
1
  module fft;
2
2
 
3
+ import core.indexing;
4
+
3
5
  [[vk::constant_id(0)]] public const uint MODE = 0;
4
6
  [[vk::constant_id(1)]] public const uint INVERSE = 0;
5
7
  [[vk::constant_id(2)]] public const uint WG_x = 0;
@@ -110,5 +112,5 @@ void fft_main<T : __BuiltinFloatingPointType>(
110
112
  uint3 gi : SV_DispatchThreadID,
111
113
  uniform FftPushConstants pcs)
112
114
  {
113
- fftKernel<T>(uint64_t(gi.x), pcs);
115
+ fftKernel<T>(uint64_t(flat_thread(gi, WG_x)), pcs);
114
116
  }
@@ -28,7 +28,7 @@ public void fillKernel<T, let VEC_SIZE : int>(
28
28
  uint64_t numel,
29
29
  T fill_value)
30
30
  {
31
- uint idx = gi.x;
31
+ uint idx = flat_thread(gi, WG_x);
32
32
  T* tensor = (T*)address;
33
33
 
34
34
  if (IS_CONTIG == 1)
@@ -1,5 +1,7 @@
1
1
  module grid_sample;
2
2
 
3
+ import core.indexing;
4
+
3
5
  [[vk::constant_id(0)]] public const uint MODE = 0;
4
6
  [[vk::constant_id(1)]] public const uint DIMS = 2;
5
7
  [[vk::constant_id(2)]] public const uint INTERPOLATION = 0;
@@ -258,7 +260,7 @@ void inputGradKernel<T : __BuiltinFloatingPointType>(uint64_t idx, Geometry g, G
258
260
 
259
261
  void gridSampleKernel<T : __BuiltinFloatingPointType>(uint3 gi, GridSamplePushConstants pcs)
260
262
  {
261
- uint64_t idx = uint64_t(gi.x);
263
+ uint64_t idx = uint64_t(flat_thread(gi, WG_x));
262
264
  if (idx >= pcs.count) return;
263
265
  Geometry g = loadGeometry(pcs.metadata);
264
266
  if (MODE == 3)
@@ -29,7 +29,7 @@ public struct IndexGatherPushConstants
29
29
 
30
30
  public void indexGatherKernel<T>(uint3 gi, IndexGatherPushConstants pcs)
31
31
  {
32
- uint idx = gi.x;
32
+ uint idx = flat_thread(gi, WG_x);
33
33
  if (idx >= pcs.numel) return;
34
34
 
35
35
  T* tensor_out = (T*)pcs.tensor_out;
@@ -85,7 +85,7 @@ T evalIntegerBinary<T : __BuiltinIntegerType>(T a, T b)
85
85
 
86
86
  public void integerBinaryKernel<T : __BuiltinIntegerType>(uint3 gi, IntegerBinaryPushConstants<T> pcs)
87
87
  {
88
- uint idx = gi.x;
88
+ uint idx = flat_thread(gi, WG_x);
89
89
  if (idx >= pcs.numel) return;
90
90
 
91
91
  T* tensor_a = (T*)pcs.tensor_a;
@@ -1,5 +1,7 @@
1
1
  module ldl;
2
2
 
3
+ import core.indexing;
4
+
3
5
  [[vk::constant_id(0)]] public const uint MODE = 0;
4
6
  [[vk::constant_id(1)]] public const uint WG_x = 0;
5
7
 
@@ -245,7 +247,7 @@ void ldl_main<T : __BuiltinFloatingPointType>(
245
247
  uint3 gi : SV_DispatchThreadID,
246
248
  uniform LdlPushConstants pcs)
247
249
  {
248
- uint64_t idx = uint64_t(gi.x);
250
+ uint64_t idx = uint64_t(flat_thread(gi, WG_x));
249
251
  if (idx >= pcs.count) return;
250
252
  if (MODE == 0) factor<T>(int64_t(idx), pcs);
251
253
  else solve<T>(int64_t(idx) / int64_t(pcs.k), int(int64_t(idx) % int64_t(pcs.k)), pcs);
@@ -1,6 +1,7 @@
1
1
  module multinomial;
2
2
 
3
3
  __exported import core.philox;
4
+ import core.indexing;
4
5
 
5
6
  [[vk::constant_id(0)]] public const uint WG_x = 0;
6
7
 
@@ -21,7 +22,7 @@ void multinomial_main<R : __BuiltinFloatingPointType & IRandomReal>(
21
22
  uint3 gi : SV_DispatchThreadID,
22
23
  uniform MultinomialPushConstants pcs)
23
24
  {
24
- uint index = gi.x;
25
+ uint index = flat_thread(gi, WG_x);
25
26
  if (index >= pcs.numel) return;
26
27
 
27
28
  R* cdf = (R*)pcs.cdf + (index / pcs.samples) * pcs.categories;
@@ -1,6 +1,7 @@
1
1
  module nllloss;
2
2
 
3
3
  __exported import core.dtype;
4
+ import core.indexing;
4
5
 
5
6
  [[vk::constant_id(0)]] public const uint MODE = 0;
6
7
  [[vk::constant_id(1)]] public const uint WG_x = 0;
@@ -71,9 +72,9 @@ public void dispatchNllLoss<T : ITorchVulkan>(uint3 gi, NllLossPushConstants pcs
71
72
 
72
73
  switch (MODE)
73
74
  {
74
- case 0: ForwardGatherOp.apply<T>(gi.x, tensor_in, target, tensor_out, pcs); break;
75
- case 1: BackwardScatterOp.apply<T>(gi.x, tensor_in, target, tensor_out, pcs); break;
76
- case 2: WeightGatherOp.apply<T>(gi.x, tensor_in, target, tensor_out, pcs); break;
75
+ case 0: ForwardGatherOp.apply<T>(flat_thread(gi, WG_x), tensor_in, target, tensor_out, pcs); break;
76
+ case 1: BackwardScatterOp.apply<T>(flat_thread(gi, WG_x), tensor_in, target, tensor_out, pcs); break;
77
+ case 2: WeightGatherOp.apply<T>(flat_thread(gi, WG_x), tensor_in, target, tensor_out, pcs); break;
77
78
  default: break;
78
79
  }
79
80
  }
@@ -1,6 +1,7 @@
1
1
  module pool;
2
2
 
3
3
  __exported import core.dtype;
4
+ import core.indexing;
4
5
 
5
6
  [[vk::constant_id(0)]] public const uint MODE = 0;
6
7
  [[vk::constant_id(1)]] public const uint WG_x = 0;
@@ -273,7 +274,7 @@ void poolBackward<T : ITorchVulkan>(uint64_t idx, PoolPushConstants pcs, Geometr
273
274
 
274
275
  void poolKernel<T : ITorchVulkan>(uint3 gi, PoolPushConstants pcs)
275
276
  {
276
- uint64_t idx = uint64_t(gi.x);
277
+ uint64_t idx = uint64_t(flat_thread(gi, WG_x));
277
278
  if (idx >= pcs.count) return;
278
279
  Geometry g = loadGeometry(pcs.metadata);
279
280
  if (MODE == 5 || MODE == 6) poolBackward<T>(idx, pcs, g);
@@ -134,10 +134,11 @@ public struct RandomIntOp<T : IRandomDtype> : IRandomOp<T>
134
134
 
135
135
  public void baseRandomKernel<T : IRandomDtype, Op : IRandomOp<T>, let VEC_SIZE : int>(uint3 gi, RandomPushConstants pcs)
136
136
  {
137
- uint first = gi.x * VEC_SIZE;
137
+ uint stream_id = flat_thread(gi, WG_x);
138
+ uint first = stream_id * VEC_SIZE;
138
139
  if (first >= pcs.numel) return;
139
140
 
140
- var stream = RandomStream<T.Real>(pcs.seed, pcs.offset, gi.x);
141
+ var stream = RandomStream<T.Real>(pcs.seed, pcs.offset, stream_id);
141
142
  T* tensor = (T*)pcs.tensor;
142
143
 
143
144
  for (int k = 0; k < VEC_SIZE; k++)
@@ -46,7 +46,7 @@ public void baseReduceKernel<T : ITorchVulkan, Op : IReduceOp<T>>(
46
46
  uint split,
47
47
  uint chunk)
48
48
  {
49
- uint idx = gi.x;
49
+ uint idx = flat_thread(gi, WG_x);
50
50
  if (uint64_t(idx) >= numel_out * split) return;
51
51
 
52
52
  uint out_idx = idx / split;
@@ -108,5 +108,5 @@ void reduce_subgroup_main<T : __BuiltinFloatingPointType>(
108
108
  uint3 group_thread_id : SV_GroupThreadID,
109
109
  uniform SubgroupReducePushConstants pcs)
110
110
  {
111
- dispatchSubgroupReduce<T>(group_id.x, group_thread_id.x, pcs);
111
+ dispatchSubgroupReduce<T>(flat_group(group_id), group_thread_id.x, pcs);
112
112
  }
@@ -1,5 +1,7 @@
1
1
  module resample;
2
2
 
3
+ import core.indexing;
4
+
3
5
  [[vk::constant_id(0)]] public const uint MODE = 0;
4
6
  [[vk::constant_id(1)]] public const uint BACKWARD = 0;
5
7
  [[vk::constant_id(2)]] public const uint ALIGN_CORNERS = 0;
@@ -154,7 +156,7 @@ Taps<T> computeTaps<T : __BuiltinFloatingPointType>(int o, int in_length, int ou
154
156
 
155
157
  void resampleKernel<T : __BuiltinFloatingPointType>(uint3 gi, ResamplePushConstants<T> pcs)
156
158
  {
157
- uint64_t idx = uint64_t(gi.x);
159
+ uint64_t idx = uint64_t(flat_thread(gi, WG_x));
158
160
  if (idx >= pcs.count) return;
159
161
 
160
162
  T* input = (T*)pcs.input;
@@ -1,6 +1,7 @@
1
1
  module resample_u8;
2
2
 
3
3
  __exported import core.dtype;
4
+ import core.indexing;
4
5
 
5
6
  [[vk::constant_id(0)]] public const uint WG_x = 0;
6
7
 
@@ -49,5 +50,5 @@ void resample_u8_main<T : ITorchVulkan>(
49
50
  uint3 gi : SV_DispatchThreadID,
50
51
  uniform ResampleU8PushConstants pcs)
51
52
  {
52
- resampleU8Kernel<T>(uint64_t(gi.x), pcs);
53
+ resampleU8Kernel<T>(uint64_t(flat_thread(gi, WG_x)), pcs);
53
54
  }
@@ -135,5 +135,5 @@ void scan_main<T : ITorchVulkan>(
135
135
  uniform ScanPushConstants pcs)
136
136
  {
137
137
  static groupshared T smem[SCAN_SMEM_SIZE];
138
- dispatchScan<T, SCAN_SMEM_SIZE>(group_id.x, group_thread_id.x, smem, pcs);
138
+ dispatchScan<T, SCAN_SMEM_SIZE>(flat_group(group_id), group_thread_id.x, smem, pcs);
139
139
  }
@@ -136,7 +136,7 @@ void scan_arg_main<T : ITorchVulkan>(
136
136
  static groupshared uint smem_idx[SCAN_ARG_SMEM_SIZE];
137
137
 
138
138
  scanArgKernel<T, SCAN_ARG_SMEM_SIZE>(
139
- group_id.x, group_thread_id.x, smem_val, smem_idx,
139
+ flat_group(group_id), group_thread_id.x, smem_val, smem_idx,
140
140
  pcs.tensor_in, pcs.tensor_val, pcs.tensor_idx, pcs.metadata,
141
141
  pcs.num_lines, pcs.scan_size, pcs.scan_stride_in, pcs.scan_stride_out
142
142
  );
@@ -1,6 +1,7 @@
1
1
  module scatter;
2
2
 
3
3
  __exported import core.dtype;
4
+ import core.indexing;
4
5
 
5
6
  [[vk::constant_id(0)]] public const uint MODE = 0;
6
7
  [[vk::constant_id(1)]] public const uint INCLUDE_SELF = 0;
@@ -55,7 +56,7 @@ T identity<T : ITorchVulkan>()
55
56
 
56
57
  public void scatterKernel<T : ITorchVulkan>(uint3 gi, ScatterPushConstants pcs)
57
58
  {
58
- uint64_t i = uint64_t(gi.x);
59
+ uint64_t i = uint64_t(flat_thread(gi, WG_x));
59
60
  if (i >= pcs.count) return;
60
61
 
61
62
  T* out = (T*)pcs.out;
@@ -1,6 +1,7 @@
1
1
  module searchsorted;
2
2
 
3
3
  __exported import core.dtype;
4
+ import core.indexing;
4
5
 
5
6
  [[vk::constant_id(0)]] public const uint RIGHT = 0;
6
7
  [[vk::constant_id(1)]] public const uint HAS_SORTER = 0;
@@ -22,7 +23,7 @@ public struct SearchsortedPushConstants
22
23
  // torch's lower and upper bound, so NaNs land past the end
23
24
  public void searchsortedKernel<T : ITorchVulkan>(uint3 gi, SearchsortedPushConstants pcs)
24
25
  {
25
- uint64_t idx = uint64_t(gi.x);
26
+ uint64_t idx = uint64_t(flat_thread(gi, WG_x));
26
27
  if (idx >= pcs.count) return;
27
28
 
28
29
  T* sorted = (T*)pcs.sorted;
@@ -1,6 +1,7 @@
1
1
  module sort;
2
2
 
3
3
  __exported import core.dtype;
4
+ import core.indexing;
4
5
 
5
6
  [[vk::constant_id(0)]] public const uint MODE = 0;
6
7
  [[vk::constant_id(1)]] public const uint DESCENDING = 0;
@@ -49,7 +50,7 @@ bool sortsBefore<T : ITorchVulkan>(T x, int64_t px, T y, int64_t py, uint n)
49
50
 
50
51
  public void sortKernel<T : ITorchVulkan>(uint3 gi, SortPushConstants pcs)
51
52
  {
52
- uint idx = gi.x;
53
+ uint idx = flat_thread(gi, WG_x);
53
54
  if (uint64_t(idx) >= pcs.threads) return;
54
55
 
55
56
  T* keys_out = (T*)pcs.keys_out;
@@ -67,7 +67,7 @@ float evalSpecial(float a, float b)
67
67
 
68
68
  public void specialKernel<T : ITorchVulkan>(uint3 gi, SpecialPushConstants pcs)
69
69
  {
70
- uint idx = gi.x;
70
+ uint idx = flat_thread(gi, WG_x);
71
71
  if (idx >= pcs.numel) return;
72
72
 
73
73
  T* tensor_a = (T*)pcs.tensor_a;
@@ -1,5 +1,7 @@
1
1
  module triangular_solve;
2
2
 
3
+ import core.indexing;
4
+
3
5
  [[vk::constant_id(0)]] public const uint UPPER = 0;
4
6
  [[vk::constant_id(1)]] public const uint UNIT_DIAGONAL = 0;
5
7
  [[vk::constant_id(2)]] public const uint WG_x = 0;
@@ -50,5 +52,5 @@ void triangular_solve_main<T : __BuiltinFloatingPointType>(
50
52
  uint3 gi : SV_DispatchThreadID,
51
53
  uniform TriangularSolvePushConstants pcs)
52
54
  {
53
- triangularSolveKernel<T>(uint64_t(gi.x), pcs);
55
+ triangularSolveKernel<T>(uint64_t(flat_thread(gi, WG_x)), pcs);
54
56
  }
@@ -30,7 +30,7 @@ public void baseUnaryKernel<T : ITorchVulkan, Op : IUnaryOp<T>, let VEC_SIZE : i
30
30
  uint64_t address_metadata,
31
31
  uint64_t numel)
32
32
  {
33
- uint idx = gi.x;
33
+ uint idx = flat_thread(gi, WG_x);
34
34
 
35
35
  if (IS_CONTIG != 0)
36
36
  {
@@ -34,7 +34,7 @@ public void whereKernel<T>(
34
34
  uint64_t address_metadata,
35
35
  uint64_t numel)
36
36
  {
37
- uint idx = gi.x;
37
+ uint idx = flat_thread(gi, WG_x);
38
38
  if (idx >= numel) return;
39
39
 
40
40
  uint8_t* tensor_cond = (uint8_t*)address_cond;
@@ -11,7 +11,9 @@ c10::DataPtr VulkanAllocator::allocate(size_t nbytes)
11
11
  c10::Device device(c10::DeviceType::PrivateUse1, VulkanContext::CurrentDevice());
12
12
  if (nbytes == 0) return {nullptr, nullptr, &VulkanAllocator::deleter, device};
13
13
 
14
- VulkanBuffer* buffer = allocateBuffer(VulkanContext::Instance().CurrentDeviceContext(), nbytes, MemoryUsage::DEVICE_ONLY);
14
+ DeviceContext* context = VulkanContext::Instance().CurrentDeviceContext();
15
+ VulkanBuffer* buffer = allocateBuffer(context, nbytes, MemoryUsage::DEVICE_ONLY);
16
+ context->recordAllocation(buffer->size());
15
17
  return {buffer, buffer, &VulkanAllocator::deleter, device};
16
18
  }
17
19
 
@@ -19,6 +21,7 @@ void VulkanAllocator::deleter(void* ptr)
19
21
  {
20
22
  if (ptr == nullptr || VulkanContext::IsInBadFork()) return;
21
23
  VulkanBuffer* buffer = static_cast<VulkanBuffer*>(ptr);
24
+ buffer->owner()->recordFree(buffer->size());
22
25
  buffer->owner()->cache.deleteBuffer(buffer);
23
26
  }
24
27