triton-windows 3.3.0.post19__cp39-cp39-win_amd64.whl → 3.3.1.post21__cp39-cp39-win_amd64.whl

triton/backends/amd/include/hip/amd_detail/amd_warp_sync_functions.h

@@ -1,288 +0,0 @@
-/*
-Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-#pragma once
-
-// Warp sync builtins (with explicit mask argument) introduced in ROCm 6.2 as a
-// preview to allow end-users to adapt to the new interface involving 64-bit
-// masks. These are disabled by default, and can be enabled by setting the macro
-// below. The builtins will be enabled unconditionally in ROCm 6.3.
-//
-// This arrangement also applies to the __activemask() builtin defined in
-// amd_warp_functions.h.
-#ifdef HIP_ENABLE_WARP_SYNC_BUILTINS
-
-#if !defined(__HIPCC_RTC__)
-#include "amd_warp_functions.h"
-#include "hip_assert.h"
-#endif
-
-template <typename T>
-__device__ inline
-T __hip_readfirstlane(T val) {
-  // In theory, behaviour is undefined when reading from a union member other
-  // than the member that was last assigned to, but it works in practice because
-  // we rely on the compiler to do the reasonable thing.
-  union {
-    unsigned long long l;
-    T d;
-  } u;
-  u.d = val;
-  // NOTE: The builtin returns int, so we first cast it to unsigned int and only
-  // then extend it to 64 bits.
-  unsigned long long lower = (unsigned)__builtin_amdgcn_readfirstlane(u.l);
-  unsigned long long upper =
-      (unsigned)__builtin_amdgcn_readfirstlane(u.l >> 32);
-  u.l = (upper << 32) | lower;
-  return u.d;
-}
-
-// When compiling for wave32 mode, ignore the upper half of the 64-bit mask.
-#define __hip_adjust_mask_for_wave32(MASK)            \
-  do {                                          \
-    if (warpSize == 32) MASK &= 0xFFFFFFFF;     \
-  } while (0)
-
-// We use a macro to expand each builtin into a waterfall that implements the
-// mask semantics:
-//
-// 1. The mask argument may be divergent.
-// 2. Each active thread must have its own bit set in its own mask value.
-// 3. For a given mask value, all threads that are mentioned in the mask must
-//    execute the same static instance of the builtin with the same mask.
-// 4. The union of all mask values supplied at a static instance must be equal
-//    to the activemask at the program point.
-//
-// Thus, the mask argument partitions the set of currently active threads in the
-// wave into disjoint subsets that cover all active threads.
-//
-// Implementation notes:
-// ---------------------
-//
-// We implement this as a waterfall loop that executes the builtin for each
-// subset separately. The return value is a divergent value across the active
-// threads. The value for inactive threads is defined by each builtin
-// separately.
-//
-// As long as every mask value is non-zero, we don't need to check if a lane
-// specifies itself in the mask; that is done by the later assertion where all
-// chosen lanes must be in the chosen mask.
-
-#define __hip_check_mask(MASK)                                                 \
-  do {                                                                         \
-    __hip_assert(MASK && "mask must be non-zero");                             \
-    bool done = false;                                                         \
-    while (__any(!done)) {                                                     \
-      if (!done) {                                                             \
-        auto chosen_mask = __hip_readfirstlane(MASK);                          \
-        if (MASK == chosen_mask) {                                             \
-          __hip_assert(MASK == __ballot(true) &&                               \
-                       "all threads specified in the mask"                     \
-                       " must execute the same operation with the same mask"); \
-          done = true;                                                         \
-        }                                                                      \
-      }                                                                        \
-    }                                                                          \
-  } while(0)
-
-#define __hip_do_sync(RETVAL, FUNC, MASK, ...)                                 \
-  do {                                                                         \
-    __hip_assert(MASK && "mask must be non-zero");                             \
-    bool done = false;                                                         \
-    while (__any(!done)) {                                                     \
-      if (!done) {                                                             \
-        auto chosen_mask = __hip_readfirstlane(MASK);                          \
-        if (MASK == chosen_mask) {                                             \
-          __hip_assert(MASK == __ballot(true) &&                               \
-                       "all threads specified in the mask"                     \
-                       " must execute the same operation with the same mask"); \
-          RETVAL = FUNC(__VA_ARGS__);                                          \
-          done = true;                                                         \
-        }                                                                      \
-      }                                                                        \
-    }                                                                          \
-  } while(0)
-
-// __all_sync, __any_sync, __ballot_sync
-
-template <typename MaskT>
-__device__ inline
-unsigned long long __ballot_sync(MaskT mask, int predicate) {
-  static_assert(
-      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
-      "The mask must be a 64-bit integer. "
-      "Implicitly promoting a smaller integer is almost always an error.");
-  __hip_adjust_mask_for_wave32(mask);
-  __hip_check_mask(mask);
-  return __ballot(predicate) & mask;
-}
-
-template <typename MaskT>
-__device__ inline
-int __all_sync(MaskT mask, int predicate) {
-  static_assert(
-      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
-      "The mask must be a 64-bit integer. "
-      "Implicitly promoting a smaller integer is almost always an error.");
-  __hip_adjust_mask_for_wave32(mask);
-  return __ballot_sync(mask, predicate) == mask;
-}
-
-template <typename MaskT>
-__device__ inline
-int __any_sync(MaskT mask, int predicate) {
-  static_assert(
-      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
-      "The mask must be a 64-bit integer. "
-      "Implicitly promoting a smaller integer is almost always an error.");
-  __hip_adjust_mask_for_wave32(mask);
-  return __ballot_sync(mask, predicate) != 0;
-}
-
-// __match_any, __match_all and sync variants
-
-template <typename T>
-__device__ inline
-unsigned long long __match_any(T value) {
-  static_assert(
-      (__hip_internal::is_integral<T>::value || __hip_internal::is_floating_point<T>::value) &&
-          (sizeof(T) == 4 || sizeof(T) == 8),
-      "T can be int, unsigned int, long, unsigned long, long long, unsigned "
-      "long long, float or double.");
-  bool done = false;
-  unsigned long long retval = 0;
-
-  while (__any(!done)) {
-    if (!done) {
-      T chosen = __hip_readfirstlane(value);
-      if (chosen == value) {
-        retval = __activemask();
-        done = true;
-      }
-    }
-  }
-
-  return retval;
-}
-
-template <typename MaskT, typename T>
-__device__ inline
-unsigned long long __match_any_sync(MaskT mask, T value) {
-  static_assert(
-      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
-      "The mask must be a 64-bit integer. "
-      "Implicitly promoting a smaller integer is almost always an error.");
-  __hip_adjust_mask_for_wave32(mask);
-  __hip_check_mask(mask);
-  return __match_any(value) & mask;
-}
-
-template <typename T>
-__device__ inline
-unsigned long long __match_all(T value, int* pred) {
-  static_assert(
-      (__hip_internal::is_integral<T>::value || __hip_internal::is_floating_point<T>::value) &&
-          (sizeof(T) == 4 || sizeof(T) == 8),
-      "T can be int, unsigned int, long, unsigned long, long long, unsigned "
-      "long long, float or double.");
-  T first = __hip_readfirstlane(value);
-  if (__all(first == value)) {
-    *pred = true;
-    return __activemask();
-  } else {
-    *pred = false;
-    return 0;
-  }
-}
-
-template <typename MaskT, typename T>
-__device__ inline
-unsigned long long __match_all_sync(MaskT mask, T value, int* pred) {
-  static_assert(
-      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
-      "The mask must be a 64-bit integer. "
-      "Implicitly promoting a smaller integer is almost always an error.");
-  MaskT retval = 0;
-  __hip_adjust_mask_for_wave32(mask);
-  __hip_do_sync(retval, __match_all, mask, value, pred);
-  return retval;
-}
-
-// various variants of shfl
-
-template <typename MaskT, typename T>
-__device__ inline
-T __shfl_sync(MaskT mask, T var, int srcLane,
-              int width = __AMDGCN_WAVEFRONT_SIZE) {
-  static_assert(
-      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
-      "The mask must be a 64-bit integer. "
-      "Implicitly promoting a smaller integer is almost always an error.");
-  __hip_adjust_mask_for_wave32(mask);
-  __hip_check_mask(mask);
-  return __shfl(var, srcLane, width);
-}
-
-template <typename MaskT, typename T>
-__device__ inline
-T __shfl_up_sync(MaskT mask, T var, unsigned int delta,
-                                   int width = __AMDGCN_WAVEFRONT_SIZE) {
-  static_assert(
-      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
-      "The mask must be a 64-bit integer. "
-      "Implicitly promoting a smaller integer is almost always an error.");
-  __hip_adjust_mask_for_wave32(mask);
-  __hip_check_mask(mask);
-  return __shfl_up(var, delta, width);
-}
-
-template <typename MaskT, typename T>
-__device__ inline
-T __shfl_down_sync(MaskT mask, T var, unsigned int delta,
-                                     int width = __AMDGCN_WAVEFRONT_SIZE) {
-  static_assert(
-      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
-      "The mask must be a 64-bit integer. "
-      "Implicitly promoting a smaller integer is almost always an error.");
-  __hip_adjust_mask_for_wave32(mask);
-  __hip_check_mask(mask);
-  return __shfl_down(var, delta, width);
-}
-
-template <typename MaskT, typename T>
-__device__ inline
-T __shfl_xor_sync(MaskT mask, T var, int laneMask,
-                                    int width = __AMDGCN_WAVEFRONT_SIZE) {
-  static_assert(
-      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
-      "The mask must be a 64-bit integer. "
-      "Implicitly promoting a smaller integer is almost always an error.");
-  __hip_adjust_mask_for_wave32(mask);
-  __hip_check_mask(mask);
-  return __shfl_xor(var, laneMask, width);
-}
-
-#undef __hip_do_sync
-#undef __hip_check_mask
-#undef __hip_adjust_mask_for_wave32
-
-#endif // HIP_ENABLE_WARP_SYNC_BUILTINS

triton/backends/amd/include/hip/amd_detail/concepts.hpp

@@ -1,30 +0,0 @@
-/*
-Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-#pragma once
-
-namespace hip_impl  // Documentation only.
-{
-#define requires(...)
-
-#define FunctionalProcedure typename
-}  // namespace hip_impl

triton/backends/amd/include/hip/amd_detail/device_library_decls.h

@@ -1,133 +0,0 @@
-/*
-Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-/**
- *  @file  amd_detail/device_library_decls.h
- *  @brief Contains declarations for types and functions in device library.
- *         Uses int64_t and uint64_t instead of long, long long, unsigned
- *         long and unsigned long long types for device library API
- *         declarations.
- */
-
-#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_LIBRARY_DECLS_H
-#define HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_LIBRARY_DECLS_H
-
-#if !defined(__HIPCC_RTC__)
-#include "hip/amd_detail/host_defines.h"
-#endif
-
-typedef unsigned char uchar;
-typedef unsigned short ushort;
-typedef unsigned int uint;
-typedef unsigned long ulong;
-typedef unsigned long long ullong;
-
-extern "C" __device__ __attribute__((const)) bool __ockl_wfany_i32(int);
-extern "C" __device__ __attribute__((const)) bool __ockl_wfall_i32(int);
-extern "C" __device__ uint __ockl_activelane_u32(void);
-
-extern "C" __device__ __attribute__((const)) uint __ockl_mul24_u32(uint, uint);
-extern "C" __device__ __attribute__((const)) int __ockl_mul24_i32(int, int);
-extern "C" __device__ __attribute__((const)) uint __ockl_mul_hi_u32(uint, uint);
-extern "C" __device__ __attribute__((const)) int __ockl_mul_hi_i32(int, int);
-extern "C" __device__ __attribute__((const)) uint __ockl_sadd_u32(uint, uint, uint);
-
-extern "C" __device__ __attribute__((const)) uchar __ockl_clz_u8(uchar);
-extern "C" __device__ __attribute__((const)) ushort __ockl_clz_u16(ushort);
-extern "C" __device__ __attribute__((const)) uint __ockl_clz_u32(uint);
-extern "C" __device__ __attribute__((const)) uint64_t __ockl_clz_u64(uint64_t);
-
-extern "C" __device__ __attribute__((const)) float __ocml_floor_f32(float);
-extern "C" __device__ __attribute__((const)) float __ocml_rint_f32(float);
-extern "C" __device__ __attribute__((const)) float __ocml_ceil_f32(float);
-extern "C" __device__ __attribute__((const)) float __ocml_trunc_f32(float);
-
-extern "C" __device__ __attribute__((const)) float __ocml_fmin_f32(float, float);
-extern "C" __device__ __attribute__((const)) float __ocml_fmax_f32(float, float);
-
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_f64(double);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_f64(double);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_f64(double);
-
-extern "C" __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
-extern "C" __device__ __attribute__((const)) _Float16 __ocml_cvtrtp_f16_f32(float);
-extern "C" __device__ __attribute__((const)) _Float16 __ocml_cvtrtz_f16_f32(float);
-
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_s32(int);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_s32(int);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_s32(int);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_u32(uint32_t);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_u32(uint32_t);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_u32(uint32_t);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_s64(int64_t);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_s64(int64_t);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_s64(int64_t);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_u64(uint64_t);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_u64(uint64_t);
-extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_u64(uint64_t);
-extern "C" __device__ __attribute__((const)) double __ocml_cvtrtn_f64_s64(int64_t);
-extern "C" __device__ __attribute__((const)) double __ocml_cvtrtp_f64_s64(int64_t);
-extern "C" __device__ __attribute__((const)) double __ocml_cvtrtz_f64_s64(int64_t);
-extern "C" __device__ __attribute__((const)) double __ocml_cvtrtn_f64_u64(uint64_t);
-extern "C" __device__ __attribute__((const)) double __ocml_cvtrtp_f64_u64(uint64_t);
-extern "C" __device__ __attribute__((const)) double __ocml_cvtrtz_f64_u64(uint64_t);
-
-extern "C" __device__ __attribute__((convergent)) void __ockl_gws_init(uint nwm1, uint rid);
-extern "C" __device__ __attribute__((convergent)) void __ockl_gws_barrier(uint nwm1, uint rid);
-
-extern "C" __device__ __attribute__((const)) uint32_t __ockl_lane_u32();
-extern "C" __device__ __attribute__((const)) int __ockl_grid_is_valid(void);
-extern "C" __device__ __attribute__((convergent)) void __ockl_grid_sync(void);
-extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_num_grids(void);
-extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_grid_rank(void);
-extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_size(void);
-extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_thread_rank(void);
-extern "C" __device__ __attribute__((const)) int __ockl_multi_grid_is_valid(void);
-extern "C" __device__ __attribute__((convergent)) void __ockl_multi_grid_sync(void);
-
-extern "C" __device__ void __ockl_atomic_add_noret_f32(float*, float);
-
-extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_add_i32(int a);
-extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_and_i32(int a);
-extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_or_i32(int a);
-
-extern "C" __device__ uint64_t __ockl_fprintf_stderr_begin();
-extern "C" __device__ uint64_t __ockl_fprintf_append_args(uint64_t msg_desc, uint32_t num_args,
-                                                          uint64_t value0, uint64_t value1,
-                                                          uint64_t value2, uint64_t value3,
-                                                          uint64_t value4, uint64_t value5,
-                                                          uint64_t value6, uint32_t is_last);
-extern "C" __device__ uint64_t __ockl_fprintf_append_string_n(uint64_t msg_desc, const char* data,
-                                                              uint64_t length, uint32_t is_last);
-
-// Introduce local address space
-#define __local __attribute__((address_space(3)))
-
-#ifdef __HIP_DEVICE_COMPILE__
-__device__ inline static __local void* __to_local(unsigned x) { return (__local void*)x; }
-#endif //__HIP_DEVICE_COMPILE__
-
-// Using hip.amdgcn.bc - sync threads
-#define __CLK_LOCAL_MEM_FENCE    0x01
-typedef unsigned __cl_mem_fence_flags;
-
-#endif

triton/backends/amd/include/hip/amd_detail/functional_grid_launch.hpp

@@ -1,218 +0,0 @@
-/*
-Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-#pragma once
-
-#include "concepts.hpp"
-#include "helpers.hpp"
-#include "program_state.hpp"
-#include "hip_runtime_api.h"
-
-#include <cstdint>
-#include <cstring>
-#include <stdexcept>
-#include <tuple>
-#include <type_traits>
-#include <utility>
-
-hipError_t ihipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList, int numDevices,
-                                               unsigned int flags, hip_impl::program_state& ps);
-
-hipError_t hipLaunchCooperativeKernel(const void* f, dim3 gridDim,
-                                    dim3 blockDim, void** args,
-                                    size_t sharedMem, hipStream_t stream,
-                                    hip_impl::program_state& ps);
-
-hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
-                                                 int  numDevices,
-                                                 unsigned int flags,
-                                                 hip_impl::program_state& ps);
-
-#pragma GCC visibility push(hidden)
-
-namespace hip_impl {
-template <typename T, typename std::enable_if<std::is_integral<T>{}>::type* = nullptr>
-inline T round_up_to_next_multiple_nonnegative(T x, T y) {
-    T tmp = x + y - 1;
-    return tmp - tmp % y;
-}
-
-template <
-    std::size_t n,
-    typename... Ts,
-    typename std::enable_if<n == sizeof...(Ts)>::type* = nullptr>
-inline hip_impl::kernarg make_kernarg(
-    const std::tuple<Ts...>&,
-    const kernargs_size_align&,
-    hip_impl::kernarg kernarg) {
-    return kernarg;
-}
-
-template <
-    std::size_t n,
-    typename... Ts,
-    typename std::enable_if<n != sizeof...(Ts)>::type* = nullptr>
-inline hip_impl::kernarg make_kernarg(
-    const std::tuple<Ts...>& formals,
-    const kernargs_size_align& size_align,
-    hip_impl::kernarg kernarg) {
-    using T = typename std::tuple_element<n, std::tuple<Ts...>>::type;
-
-    static_assert(
-        !std::is_reference<T>{},
-        "A __global__ function cannot have a reference as one of its "
-            "arguments.");
-    #if defined(HIP_STRICT)
-        static_assert(
-            std::is_trivially_copyable<T>{},
-            "Only TriviallyCopyable types can be arguments to a __global__ "
-                "function");
-    #endif
-
-    kernarg.resize(round_up_to_next_multiple_nonnegative(
-        kernarg.size(), size_align.alignment(n)) + size_align.size(n));
-
-    std::memcpy(
-        kernarg.data() + kernarg.size() - size_align.size(n),
-        &std::get<n>(formals),
-        size_align.size(n));
-    return make_kernarg<n + 1>(formals, size_align, std::move(kernarg));
-}
-
-template <typename... Formals, typename... Actuals>
-inline hip_impl::kernarg make_kernarg(
-    void (*kernel)(Formals...), std::tuple<Actuals...> actuals) {
-    static_assert(sizeof...(Formals) == sizeof...(Actuals),
-        "The count of formal arguments must match the count of actuals.");
-
-    if (sizeof...(Formals) == 0) return {};
-
-    std::tuple<Formals...> to_formals{std::move(actuals)};
-    hip_impl::kernarg kernarg;
-    kernarg.reserve(sizeof(to_formals));
-
-    auto& ps = hip_impl::get_program_state();
-    return make_kernarg<0>(to_formals, 
-                           ps.get_kernargs_size_align(
-                               reinterpret_cast<std::uintptr_t>(kernel)),
-                           std::move(kernarg));
-}
-
-
-HIP_INTERNAL_EXPORTED_API hsa_agent_t target_agent(hipStream_t stream);
-
-inline
-__attribute__((visibility("hidden")))
-void hipLaunchKernelGGLImpl(
-    std::uintptr_t function_address,
-    const dim3& numBlocks,
-    const dim3& dimBlocks,
-    std::uint32_t sharedMemBytes,
-    hipStream_t stream,
-    void** kernarg) {
-
-    const auto& kd = hip_impl::get_program_state().kernel_descriptor(function_address, 
-                                                               target_agent(stream));
-
-    hipModuleLaunchKernel(kd, numBlocks.x, numBlocks.y, numBlocks.z,
-                          dimBlocks.x, dimBlocks.y, dimBlocks.z, sharedMemBytes,
-                          stream, nullptr, kernarg);
-}
-} // Namespace hip_impl.
-
-
-template <class T>
-inline
-hipError_t hipOccupancyMaxPotentialBlockSize(int* gridSize, int* blockSize,
-    T kernel, size_t dynSharedMemPerBlk = 0, int blockSizeLimit = 0) {
-
-    using namespace hip_impl;
-
-    hip_impl::hip_init();
-    auto f = get_program_state().kernel_descriptor(reinterpret_cast<std::uintptr_t>(kernel),
-                                                   target_agent(0));
-
-    return hipModuleOccupancyMaxPotentialBlockSize(gridSize, blockSize, f,
-                                      dynSharedMemPerBlk, blockSizeLimit);
-}
-
-template <class T>
-inline
-hipError_t hipOccupancyMaxPotentialBlockSizeWithFlags(int* gridSize, int* blockSize,
-    T kernel, size_t dynSharedMemPerBlk = 0, int blockSizeLimit = 0, unsigned int  flags = 0 ) {
-
-    using namespace hip_impl;
-
-    hip_impl::hip_init();
-    if(flags != hipOccupancyDefault) return hipErrorNotSupported;
-    auto f = get_program_state().kernel_descriptor(reinterpret_cast<std::uintptr_t>(kernel),
-                                                   target_agent(0));
-
-    return hipModuleOccupancyMaxPotentialBlockSize(gridSize, blockSize, f,
-                                      dynSharedMemPerBlk, blockSizeLimit);
-}
-
-template <typename... Args, typename F = void (*)(Args...)>
-inline
-void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks,
-                        std::uint32_t sharedMemBytes, hipStream_t stream,
-                        Args... args) {
-    hip_impl::hip_init();
-    auto kernarg = hip_impl::make_kernarg(kernel, std::tuple<Args...>{std::move(args)...});
-    std::size_t kernarg_size = kernarg.size();
-
-    void* config[]{
-        HIP_LAUNCH_PARAM_BUFFER_POINTER,
-        kernarg.data(),
-        HIP_LAUNCH_PARAM_BUFFER_SIZE,
-        &kernarg_size,
-        HIP_LAUNCH_PARAM_END};
-
-    hip_impl::hipLaunchKernelGGLImpl(reinterpret_cast<std::uintptr_t>(kernel),
-                                     numBlocks, dimBlocks, sharedMemBytes,
-                                     stream, &config[0]);
-}
-
-template <typename F>
-inline
-__attribute__((visibility("hidden")))
-hipError_t hipLaunchCooperativeKernel(F f, dim3 gridDim, dim3 blockDim,
-                                      void** args, size_t sharedMem,
-                                      hipStream_t stream) {
-    hip_impl::hip_init();
-    auto& ps = hip_impl::get_program_state();
-    return hipLaunchCooperativeKernel(reinterpret_cast<void*>(f), gridDim,
-                                      blockDim, args, sharedMem, stream, ps);
-}
-
-inline
-__attribute__((visibility("hidden")))
-hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
-                                                 int  numDevices,
-                                                 unsigned int  flags) {
-
-    hip_impl::hip_init();
-    auto& ps = hip_impl::get_program_state();
-    return hipLaunchCooperativeKernelMultiDevice(launchParamsList, numDevices, flags, ps);
-}
-
-#pragma GCC visibility pop