numba-cuda 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

numba_cuda/numba/cuda/cudadrv/nvvm.py

@@ -1,12 +1,12 @@
 """
 This is a direct translation of nvvm.h
 """
+
 import logging
 import re
 import sys
 import warnings
-from ctypes import (c_void_p, c_int, POINTER, c_char_p, c_size_t, byref,
-                    c_char)
+from ctypes import c_void_p, c_int, POINTER, c_char_p, c_size_t, byref, c_char
 
 import threading
 
@@ -31,7 +31,7 @@ nvvm_program = c_void_p
 # Result code
 nvvm_result = c_int
 
-RESULT_CODE_NAMES = '''
+RESULT_CODE_NAMES = """
 NVVM_SUCCESS
 NVVM_ERROR_OUT_OF_MEMORY
 NVVM_ERROR_PROGRAM_CREATION_FAILURE
@@ -42,19 +42,23 @@ NVVM_ERROR_INVALID_IR
 NVVM_ERROR_INVALID_OPTION
 NVVM_ERROR_NO_MODULE_IN_PROGRAM
 NVVM_ERROR_COMPILATION
-'''.split()
+""".split()
 
 for i, k in enumerate(RESULT_CODE_NAMES):
     setattr(sys.modules[__name__], k, i)
 
 # Data layouts. NVVM IR 1.8 (CUDA 11.6) introduced 128-bit integer support.
 
-_datalayout_original = ('e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-'
-                        'i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-'
-                        'v64:64:64-v128:128:128-n16:32:64')
-_datalayout_i128 = ('e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-'
-                    'i128:128:128-f32:32:32-f64:64:64-v16:16:16-v32:32:32-'
-                    'v64:64:64-v128:128:128-n16:32:64')
+_datalayout_original = (
+    "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
+    "i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-"
+    "v64:64:64-v128:128:128-n16:32:64"
+)
+_datalayout_i128 = (
+    "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-"
+    "i128:128:128-f32:32:32-f64:64:64-v16:16:16-v32:32:32-"
+    "v64:64:64-v128:128:128-n16:32:64"
+)
 
 
 def is_available():
@@ -73,59 +77,74 @@ _nvvm_lock = threading.Lock()
 
 
 class NVVM(object):
-    '''Process-wide singleton.
-    '''
-    _PROTOTYPES = {
+    """Process-wide singleton."""
 
+    _PROTOTYPES = {
         # nvvmResult nvvmVersion(int *major, int *minor)
-        'nvvmVersion': (nvvm_result, POINTER(c_int), POINTER(c_int)),
-
+        "nvvmVersion": (nvvm_result, POINTER(c_int), POINTER(c_int)),
         # nvvmResult nvvmCreateProgram(nvvmProgram *cu)
-        'nvvmCreateProgram': (nvvm_result, POINTER(nvvm_program)),
-
+        "nvvmCreateProgram": (nvvm_result, POINTER(nvvm_program)),
         # nvvmResult nvvmDestroyProgram(nvvmProgram *cu)
-        'nvvmDestroyProgram': (nvvm_result, POINTER(nvvm_program)),
-
+        "nvvmDestroyProgram": (nvvm_result, POINTER(nvvm_program)),
         # nvvmResult nvvmAddModuleToProgram(nvvmProgram cu, const char *buffer,
         #                                   size_t size, const char *name)
-        'nvvmAddModuleToProgram': (
-            nvvm_result, nvvm_program, c_char_p, c_size_t, c_char_p),
-
+        "nvvmAddModuleToProgram": (
+            nvvm_result,
+            nvvm_program,
+            c_char_p,
+            c_size_t,
+            c_char_p,
+        ),
         # nvvmResult nvvmLazyAddModuleToProgram(nvvmProgram cu,
         #                                       const char* buffer,
         #                                       size_t size,
         #                                       const char *name)
-        'nvvmLazyAddModuleToProgram': (
-            nvvm_result, nvvm_program, c_char_p, c_size_t, c_char_p),
-
+        "nvvmLazyAddModuleToProgram": (
+            nvvm_result,
+            nvvm_program,
+            c_char_p,
+            c_size_t,
+            c_char_p,
+        ),
         # nvvmResult nvvmCompileProgram(nvvmProgram cu, int numOptions,
         #                          const char **options)
-        'nvvmCompileProgram': (
-            nvvm_result, nvvm_program, c_int, POINTER(c_char_p)),
-
+        "nvvmCompileProgram": (
+            nvvm_result,
+            nvvm_program,
+            c_int,
+            POINTER(c_char_p),
+        ),
         # nvvmResult nvvmGetCompiledResultSize(nvvmProgram cu,
         #                                      size_t *bufferSizeRet)
-        'nvvmGetCompiledResultSize': (
-            nvvm_result, nvvm_program, POINTER(c_size_t)),
-
+        "nvvmGetCompiledResultSize": (
+            nvvm_result,
+            nvvm_program,
+            POINTER(c_size_t),
+        ),
         # nvvmResult nvvmGetCompiledResult(nvvmProgram cu, char *buffer)
-        'nvvmGetCompiledResult': (nvvm_result, nvvm_program, c_char_p),
-
+        "nvvmGetCompiledResult": (nvvm_result, nvvm_program, c_char_p),
         # nvvmResult nvvmGetProgramLogSize(nvvmProgram cu,
         #                                      size_t *bufferSizeRet)
-        'nvvmGetProgramLogSize': (nvvm_result, nvvm_program, POINTER(c_size_t)),
-
+        "nvvmGetProgramLogSize": (nvvm_result, nvvm_program, POINTER(c_size_t)),
         # nvvmResult nvvmGetProgramLog(nvvmProgram cu, char *buffer)
-        'nvvmGetProgramLog': (nvvm_result, nvvm_program, c_char_p),
-
+        "nvvmGetProgramLog": (nvvm_result, nvvm_program, c_char_p),
         # nvvmResult nvvmIRVersion (int* majorIR, int* minorIR, int* majorDbg,
         #                           int* minorDbg )
-        'nvvmIRVersion': (nvvm_result, POINTER(c_int), POINTER(c_int),
-                          POINTER(c_int), POINTER(c_int)),
+        "nvvmIRVersion": (
+            nvvm_result,
+            POINTER(c_int),
+            POINTER(c_int),
+            POINTER(c_int),
+            POINTER(c_int),
+        ),
         # nvvmResult nvvmVerifyProgram (nvvmProgram prog, int numOptions,
         #                               const char** options)
-        'nvvmVerifyProgram': (nvvm_result, nvvm_program, c_int,
-                              POINTER(c_char_p))
+        "nvvmVerifyProgram": (
+            nvvm_result,
+            nvvm_program,
+            c_int,
+            POINTER(c_char_p),
+        ),
     }
 
     # Singleton reference
@@ -136,11 +155,13 @@ class NVVM(object):
             if cls.__INSTANCE is None:
                 cls.__INSTANCE = inst = object.__new__(cls)
                 try:
-                    inst.driver = open_cudalib('nvvm')
+                    inst.driver = open_cudalib("nvvm")
                 except OSError as e:
                     cls.__INSTANCE = None
-                    errmsg = ("libNVVM cannot be found. Do `conda install "
-                              "cudatoolkit`:\n%s")
+                    errmsg = (
+                        "libNVVM cannot be found. Do `conda install "
+                        "cudatoolkit`:\n%s"
+                    )
                     raise NvvmSupportError(errmsg % e)
 
                 # Find & populate functions
@@ -175,7 +196,7 @@ class NVVM(object):
         major = c_int()
         minor = c_int()
         err = self.nvvmVersion(byref(major), byref(minor))
-        self.check_error(err, 'Failed to get version.')
+        self.check_error(err, "Failed to get version.")
         return major.value, minor.value
 
     def get_ir_version(self):
@@ -183,9 +204,10 @@ class NVVM(object):
         minorIR = c_int()
         majorDbg = c_int()
         minorDbg = c_int()
-        err = self.nvvmIRVersion(byref(majorIR), byref(minorIR),
-                                 byref(majorDbg), byref(minorDbg))
-        self.check_error(err, 'Failed to get IR version.')
+        err = self.nvvmIRVersion(
+            byref(majorIR), byref(minorIR), byref(majorDbg), byref(minorDbg)
+        )
+        self.check_error(err, "Failed to get IR version.")
         return majorIR.value, minorIR.value, majorDbg.value, minorDbg.value
 
     def check_error(self, error, msg, exit=False):
@@ -223,18 +245,18 @@ class CompilationUnit(object):
         self.driver = NVVM()
         self._handle = nvvm_program()
         err = self.driver.nvvmCreateProgram(byref(self._handle))
-        self.driver.check_error(err, 'Failed to create CU')
+        self.driver.check_error(err, "Failed to create CU")
 
         def stringify_option(k, v):
-            k = k.replace('_', '-')
+            k = k.replace("_", "-")
 
             if v is None:
-                return f'-{k}'.encode('utf-8')
+                return f"-{k}".encode("utf-8")
 
             if isinstance(v, bool):
                 v = int(v)
 
-            return f'-{k}={v}'.encode('utf-8')
+            return f"-{k}={v}".encode("utf-8")
 
         options = [stringify_option(k, v) for k, v in options.items()]
         option_ptrs = (c_char_p * len(options))(*[c_char_p(x) for x in options])
@@ -248,17 +270,18 @@ class CompilationUnit(object):
     def __del__(self):
         driver = NVVM()
         err = driver.nvvmDestroyProgram(byref(self._handle))
-        driver.check_error(err, 'Failed to destroy CU', exit=True)
+        driver.check_error(err, "Failed to destroy CU", exit=True)
 
     def add_module(self, buffer):
         """
-         Add a module level NVVM IR to a compilation unit.
-         - The buffer should contain an NVVM module IR either in the bitcode
-           representation (LLVM3.0) or in the text representation.
+        Add a module level NVVM IR to a compilation unit.
+        - The buffer should contain an NVVM module IR either in the bitcode
+          representation (LLVM3.0) or in the text representation.
         """
-        err = self.driver.nvvmAddModuleToProgram(self._handle, buffer,
-                                                 len(buffer), None)
-        self.driver.check_error(err, 'Failed to add module')
+        err = self.driver.nvvmAddModuleToProgram(
+            self._handle, buffer, len(buffer), None
+        )
+        self.driver.check_error(err, "Failed to add module")
 
     def lazy_add_module(self, buffer):
         """
@@ -266,37 +289,41 @@ class CompilationUnit(object):
         The buffer should contain NVVM module IR either in the bitcode
         representation or in the text representation.
         """
-        err = self.driver.nvvmLazyAddModuleToProgram(self._handle, buffer,
-                                                     len(buffer), None)
-        self.driver.check_error(err, 'Failed to add module')
+        err = self.driver.nvvmLazyAddModuleToProgram(
+            self._handle, buffer, len(buffer), None
+        )
+        self.driver.check_error(err, "Failed to add module")
 
     def verify(self):
         """
         Run the NVVM verifier on all code added to the compilation unit.
         """
-        err = self.driver.nvvmVerifyProgram(self._handle, self.n_options,
-                                            self.option_ptrs)
-        self._try_error(err, 'Failed to verify\n')
+        err = self.driver.nvvmVerifyProgram(
+            self._handle, self.n_options, self.option_ptrs
+        )
+        self._try_error(err, "Failed to verify\n")
 
     def compile(self):
         """
         Compile all modules added to the compilation unit and return the
         resulting PTX or LTO-IR (depending on the options).
         """
-        err = self.driver.nvvmCompileProgram(self._handle, self.n_options,
-                                             self.option_ptrs)
-        self._try_error(err, 'Failed to compile\n')
+        err = self.driver.nvvmCompileProgram(
+            self._handle, self.n_options, self.option_ptrs
+        )
+        self._try_error(err, "Failed to compile\n")
 
         # Get result
         result_size = c_size_t()
-        err = self.driver.nvvmGetCompiledResultSize(self._handle,
-                                                    byref(result_size))
+        err = self.driver.nvvmGetCompiledResultSize(
+            self._handle, byref(result_size)
+        )
 
-        self._try_error(err, 'Failed to get size of compiled result.')
+        self._try_error(err, "Failed to get size of compiled result.")
 
         output_buffer = (c_char * result_size.value)()
         err = self.driver.nvvmGetCompiledResult(self._handle, output_buffer)
-        self._try_error(err, 'Failed to get compiled result.')
+        self._try_error(err, "Failed to get compiled result.")
 
         # Get log
         self.log = self.get_log()
@@ -311,31 +338,44 @@ class CompilationUnit(object):
     def get_log(self):
         reslen = c_size_t()
         err = self.driver.nvvmGetProgramLogSize(self._handle, byref(reslen))
-        self.driver.check_error(err, 'Failed to get compilation log size.')
+        self.driver.check_error(err, "Failed to get compilation log size.")
 
         if reslen.value > 1:
             logbuf = (c_char * reslen.value)()
             err = self.driver.nvvmGetProgramLog(self._handle, logbuf)
-            self.driver.check_error(err, 'Failed to get compilation log.')
+            self.driver.check_error(err, "Failed to get compilation log.")
 
-            return logbuf.value.decode('utf8')  # populate log attribute
+            return logbuf.value.decode("utf8")  # populate log attribute
 
-        return ''
+        return ""
 
 
 COMPUTE_CAPABILITIES = (
-    (3, 5), (3, 7),
-    (5, 0), (5, 2), (5, 3),
-    (6, 0), (6, 1), (6, 2),
-    (7, 0), (7, 2), (7, 5),
-    (8, 0), (8, 6), (8, 7), (8, 9),
+    (3, 5),
+    (3, 7),
+    (5, 0),
+    (5, 2),
+    (5, 3),
+    (6, 0),
+    (6, 1),
+    (6, 2),
+    (7, 0),
+    (7, 2),
+    (7, 5),
+    (8, 0),
+    (8, 6),
+    (8, 7),
+    (8, 9),
     (9, 0),
-    (10, 0), (10, 1),
+    (10, 0),
+    (10, 1),
     (12, 0),
 )
 
+
 # Maps CTK version -> (min supported cc, max supported cc) inclusive
-CTK_SUPPORTED = {
+_CUDA_CC_MIN_MAX_SUPPORT = {
+    (11, 1): ((3, 5), (8, 0)),
     (11, 2): ((3, 5), (8, 6)),
     (11, 3): ((3, 5), (8, 6)),
     (11, 4): ((3, 5), (8, 7)),
@@ -357,34 +397,43 @@ CTK_SUPPORTED = {
 def ccs_supported_by_ctk(ctk_version):
     try:
         # For supported versions, we look up the range of supported CCs
-        min_cc, max_cc = CTK_SUPPORTED[ctk_version]
-        return tuple([cc for cc in COMPUTE_CAPABILITIES
-                      if min_cc <= cc <= max_cc])
+        min_cc, max_cc = _CUDA_CC_MIN_MAX_SUPPORT[ctk_version]
+        return tuple(
+            [cc for cc in COMPUTE_CAPABILITIES if min_cc <= cc <= max_cc]
+        )
     except KeyError:
         # For unsupported CUDA toolkit versions, all we can do is assume all
         # non-deprecated versions we are aware of are supported.
-        return tuple([cc for cc in COMPUTE_CAPABILITIES
-                      if cc >= config.CUDA_DEFAULT_PTX_CC])
+        return tuple(
+            [
+                cc
+                for cc in COMPUTE_CAPABILITIES
+                if cc >= config.CUDA_DEFAULT_PTX_CC
+            ]
+        )
 
 
 def get_supported_ccs():
     try:
         from numba.cuda.cudadrv.runtime import runtime
+
         cudart_version = runtime.get_version()
-    except: # noqa: E722
+    except:  # noqa: E722
         # We can't support anything if there's an error getting the runtime
         # version (e.g. if it's not present or there's another issue)
         _supported_cc = ()
         return _supported_cc
 
     # Ensure the minimum CTK version requirement is met
-    min_cudart = min(CTK_SUPPORTED)
+    min_cudart = min(_CUDA_CC_MIN_MAX_SUPPORT)
     if cudart_version < min_cudart:
         _supported_cc = ()
         ctk_ver = f"{cudart_version[0]}.{cudart_version[1]}"
-        unsupported_ver = (f"CUDA Toolkit {ctk_ver} is unsupported by Numba - "
-                           f"{min_cudart[0]}.{min_cudart[1]} is the minimum "
-                           "required version.")
+        unsupported_ver = (
+            f"CUDA Toolkit {ctk_ver} is unsupported by Numba - "
+            f"{min_cudart[0]}.{min_cudart[1]} is the minimum "
+            "required version."
+        )
         warnings.warn(unsupported_ver)
         return _supported_cc
 
@@ -403,8 +452,10 @@ def find_closest_arch(mycc):
     supported_ccs = NVVM().supported_ccs
 
     if not supported_ccs:
-        msg = "No supported GPU compute capabilities found. " \
-              "Please check your cudatoolkit version matches your CUDA version."
+        msg = (
+            "No supported GPU compute capabilities found. "
+            "Please check your cudatoolkit version matches your CUDA version."
+        )
         raise NvvmSupportError(msg)
 
     for i, cc in enumerate(supported_ccs):
@@ -415,8 +466,10 @@ def find_closest_arch(mycc):
             # Exceeded
             if i == 0:
                 # CC lower than supported
-                msg = "GPU compute capability %d.%d is not supported" \
-                      "(requires >=%d.%d)" % (mycc + cc)
+                msg = (
+                    "GPU compute capability %d.%d is not supported"
+                    "(requires >=%d.%d)" % (mycc + cc)
+                )
                 raise NvvmSupportError(msg)
             else:
                 # return the previous CC
@@ -427,16 +480,15 @@ def find_closest_arch(mycc):
 
 
 def get_arch_option(major, minor):
-    """Matches with the closest architecture option
-    """
+    """Matches with the closest architecture option"""
     if config.FORCE_CUDA_CC:
         arch = config.FORCE_CUDA_CC
     else:
         arch = find_closest_arch((major, minor))
-    return 'compute_%d%d' % arch
+    return "compute_%d%d" % arch
 
 
-MISSING_LIBDEVICE_FILE_MSG = '''Missing libdevice file.
+MISSING_LIBDEVICE_FILE_MSG = """Missing libdevice file.
 Please ensure you have a CUDA Toolkit 11.2 or higher.
 For CUDA 12, ``cuda-nvcc`` and ``cuda-nvrtc`` are required:
 
@@ -445,7 +497,7 @@ For CUDA 12, ``cuda-nvcc`` and ``cuda-nvrtc`` are required:
 For CUDA 11, ``cudatoolkit`` is required:
 
     $ conda install -c conda-forge cudatoolkit "cuda-version>=11.2,<12.0"
-'''
+"""
 
 
 class LibDevice(object):
@@ -466,7 +518,7 @@ class LibDevice(object):
 cas_nvvm = """
     %cas_success = cmpxchg volatile {Ti}* %iptr, {Ti} %old, {Ti} %new monotonic monotonic
     %cas = extractvalue {{ {Ti}, i1 }} %cas_success, 0
-""" # noqa: E501
+"""  # noqa: E501
 
 
 # Translation of code from CUDA Programming Guide v6.5, section B.12
@@ -490,7 +542,7 @@ done:
     %result = bitcast {Ti} %old to {T}
     ret {T} %result
 }}
-""" # noqa: E501
+"""  # noqa: E501
 
 ir_numba_atomic_inc_template = """
 define internal {T} @___numba_atomic_{Tu}_inc({T}* %iptr, {T} %val) alwaysinline {{
@@ -510,7 +562,7 @@ attempt:
 done:
     ret {T} %old
 }}
-""" # noqa: E501
+"""  # noqa: E501
 
 ir_numba_atomic_dec_template = """
 define internal {T} @___numba_atomic_{Tu}_dec({T}* %iptr, {T} %val) alwaysinline {{
@@ -530,7 +582,7 @@ attempt:
 done:
     ret {T} %old
 }}
-""" # noqa: E501
+"""  # noqa: E501
 
 ir_numba_atomic_minmax_template = """
 define internal {T} @___numba_atomic_{T}_{NAN}{FUNC}({T}* %ptr, {T} %val) alwaysinline {{
@@ -561,7 +613,7 @@ attempt:
 done:
     ret {T} %ptrval
 }}
-""" # noqa: E501
+"""  # noqa: E501
 
 
 def ir_cas(Ti):
@@ -574,8 +626,15 @@ def ir_numba_atomic_binary(T, Ti, OP, FUNC):
 
 
 def ir_numba_atomic_minmax(T, Ti, NAN, OP, PTR_OR_VAL, FUNC):
-    params = dict(T=T, Ti=Ti, NAN=NAN, OP=OP, PTR_OR_VAL=PTR_OR_VAL,
-                  FUNC=FUNC, CAS=ir_cas(Ti))
+    params = dict(
+        T=T,
+        Ti=Ti,
+        NAN=NAN,
+        OP=OP,
+        PTR_OR_VAL=PTR_OR_VAL,
+        FUNC=FUNC,
+        CAS=ir_cas(Ti),
+    )
 
     return ir_numba_atomic_minmax_template.format(**params)
 
@@ -590,41 +649,115 @@ def ir_numba_atomic_dec(T, Tu):
 
 def llvm_replace(llvmir):
     replacements = [
-        ('declare double @"___numba_atomic_double_add"(double* %".1", double %".2")',     # noqa: E501
-         ir_numba_atomic_binary(T='double', Ti='i64', OP='fadd', FUNC='add')),
-        ('declare float @"___numba_atomic_float_sub"(float* %".1", float %".2")',         # noqa: E501
-         ir_numba_atomic_binary(T='float', Ti='i32', OP='fsub', FUNC='sub')),
-        ('declare double @"___numba_atomic_double_sub"(double* %".1", double %".2")',     # noqa: E501
-         ir_numba_atomic_binary(T='double', Ti='i64', OP='fsub', FUNC='sub')),
-        ('declare i64 @"___numba_atomic_u64_inc"(i64* %".1", i64 %".2")',
-         ir_numba_atomic_inc(T='i64', Tu='u64')),
-        ('declare i64 @"___numba_atomic_u64_dec"(i64* %".1", i64 %".2")',
-         ir_numba_atomic_dec(T='i64', Tu='u64')),
-        ('declare float @"___numba_atomic_float_max"(float* %".1", float %".2")',         # noqa: E501
-         ir_numba_atomic_minmax(T='float', Ti='i32', NAN='', OP='nnan olt',
-                                PTR_OR_VAL='ptr', FUNC='max')),
-        ('declare double @"___numba_atomic_double_max"(double* %".1", double %".2")',     # noqa: E501
-         ir_numba_atomic_minmax(T='double', Ti='i64', NAN='', OP='nnan olt',
-                                PTR_OR_VAL='ptr', FUNC='max')),
-        ('declare float @"___numba_atomic_float_min"(float* %".1", float %".2")',         # noqa: E501
-         ir_numba_atomic_minmax(T='float', Ti='i32', NAN='', OP='nnan ogt',
-                                PTR_OR_VAL='ptr', FUNC='min')),
-        ('declare double @"___numba_atomic_double_min"(double* %".1", double %".2")',     # noqa: E501
-         ir_numba_atomic_minmax(T='double', Ti='i64', NAN='', OP='nnan ogt',
-                                PTR_OR_VAL='ptr', FUNC='min')),
-        ('declare float @"___numba_atomic_float_nanmax"(float* %".1", float %".2")',      # noqa: E501
-         ir_numba_atomic_minmax(T='float', Ti='i32', NAN='nan', OP='ult',
-                                PTR_OR_VAL='', FUNC='max')),
-        ('declare double @"___numba_atomic_double_nanmax"(double* %".1", double %".2")',  # noqa: E501
-         ir_numba_atomic_minmax(T='double', Ti='i64', NAN='nan', OP='ult',
-                                PTR_OR_VAL='', FUNC='max')),
-        ('declare float @"___numba_atomic_float_nanmin"(float* %".1", float %".2")',      # noqa: E501
-         ir_numba_atomic_minmax(T='float', Ti='i32', NAN='nan', OP='ugt',
-                                PTR_OR_VAL='', FUNC='min')),
-        ('declare double @"___numba_atomic_double_nanmin"(double* %".1", double %".2")',  # noqa: E501
-         ir_numba_atomic_minmax(T='double', Ti='i64', NAN='nan', OP='ugt',
-                                PTR_OR_VAL='', FUNC='min')),
-        ('immarg', '')
+        (
+            'declare double @"___numba_atomic_double_add"(double* %".1", double %".2")',  # noqa: E501
+            ir_numba_atomic_binary(T="double", Ti="i64", OP="fadd", FUNC="add"),
+        ),
+        (
+            'declare float @"___numba_atomic_float_sub"(float* %".1", float %".2")',  # noqa: E501
+            ir_numba_atomic_binary(T="float", Ti="i32", OP="fsub", FUNC="sub"),
+        ),
+        (
+            'declare double @"___numba_atomic_double_sub"(double* %".1", double %".2")',  # noqa: E501
+            ir_numba_atomic_binary(T="double", Ti="i64", OP="fsub", FUNC="sub"),
+        ),
+        (
+            'declare i64 @"___numba_atomic_u64_inc"(i64* %".1", i64 %".2")',
+            ir_numba_atomic_inc(T="i64", Tu="u64"),
+        ),
+        (
+            'declare i64 @"___numba_atomic_u64_dec"(i64* %".1", i64 %".2")',
+            ir_numba_atomic_dec(T="i64", Tu="u64"),
+        ),
+        (
+            'declare float @"___numba_atomic_float_max"(float* %".1", float %".2")',  # noqa: E501
+            ir_numba_atomic_minmax(
+                T="float",
+                Ti="i32",
+                NAN="",
+                OP="nnan olt",
+                PTR_OR_VAL="ptr",
+                FUNC="max",
+            ),
+        ),
+        (
+            'declare double @"___numba_atomic_double_max"(double* %".1", double %".2")',  # noqa: E501
+            ir_numba_atomic_minmax(
+                T="double",
+                Ti="i64",
+                NAN="",
+                OP="nnan olt",
+                PTR_OR_VAL="ptr",
+                FUNC="max",
+            ),
+        ),
+        (
+            'declare float @"___numba_atomic_float_min"(float* %".1", float %".2")',  # noqa: E501
+            ir_numba_atomic_minmax(
+                T="float",
+                Ti="i32",
+                NAN="",
+                OP="nnan ogt",
+                PTR_OR_VAL="ptr",
+                FUNC="min",
+            ),
+        ),
+        (
+            'declare double @"___numba_atomic_double_min"(double* %".1", double %".2")',  # noqa: E501
+            ir_numba_atomic_minmax(
+                T="double",
+                Ti="i64",
+                NAN="",
+                OP="nnan ogt",
+                PTR_OR_VAL="ptr",
+                FUNC="min",
+            ),
+        ),
+        (
+            'declare float @"___numba_atomic_float_nanmax"(float* %".1", float %".2")',  # noqa: E501
+            ir_numba_atomic_minmax(
+                T="float",
+                Ti="i32",
+                NAN="nan",
+                OP="ult",
+                PTR_OR_VAL="",
+                FUNC="max",
+            ),
+        ),
+        (
+            'declare double @"___numba_atomic_double_nanmax"(double* %".1", double %".2")',  # noqa: E501
+            ir_numba_atomic_minmax(
+                T="double",
+                Ti="i64",
+                NAN="nan",
+                OP="ult",
+                PTR_OR_VAL="",
+                FUNC="max",
+            ),
+        ),
+        (
+            'declare float @"___numba_atomic_float_nanmin"(float* %".1", float %".2")',  # noqa: E501
+            ir_numba_atomic_minmax(
+                T="float",
+                Ti="i32",
+                NAN="nan",
+                OP="ugt",
+                PTR_OR_VAL="",
+                FUNC="min",
+            ),
+        ),
+        (
+            'declare double @"___numba_atomic_double_nanmin"(double* %".1", double %".2")',  # noqa: E501
+            ir_numba_atomic_minmax(
+                T="double",
+                Ti="i64",
+                NAN="nan",
+                OP="ugt",
+                PTR_OR_VAL="",
+                FUNC="min",
+            ),
+        ),
+        ("immarg", ""),
     ]
 
     for decl, fn in replacements:
@@ -639,19 +772,21 @@ def compile_ir(llvmir, **options):
     if isinstance(llvmir, str):
         llvmir = [llvmir]
 
-    if options.pop('fastmath', False):
-        options.update({
-            'ftz': True,
-            'fma': True,
-            'prec_div': False,
-            'prec_sqrt': False,
-        })
+    if options.pop("fastmath", False):
+        options.update(
+            {
+                "ftz": True,
+                "fma": True,
+                "prec_div": False,
+                "prec_sqrt": False,
+            }
+        )
 
     cu = CompilationUnit(options)
 
     for mod in llvmir:
         mod = llvm_replace(mod)
-        cu.add_module(mod.encode('utf8'))
+        cu.add_module(mod.encode("utf8"))
     cu.verify()
 
     # We add libdevice following verification so that it is not subject to the
@@ -671,16 +806,16 @@ def llvm150_to_70_ir(ir):
     """
     buf = []
     for line in ir.splitlines():
-        if line.startswith('attributes #'):
+        if line.startswith("attributes #"):
             # Remove function attributes unsupported by LLVM 7.0
             m = re_attributes_def.match(line)
             attrs = m.group(1).split()
-            attrs = ' '.join(a for a in attrs if a != 'willreturn')
+            attrs = " ".join(a for a in attrs if a != "willreturn")
             line = line.replace(m.group(1), attrs)
 
         buf.append(line)
 
-    return '\n'.join(buf)
+    return "\n".join(buf)
 
 
 def set_cuda_kernel(function):
@@ -704,7 +839,7 @@ def set_cuda_kernel(function):
     mdvalue = ir.Constant(ir.IntType(32), 1)
     md = module.add_metadata((function, mdstr, mdvalue))
 
-    nmd = cgutils.get_or_insert_named_metadata(module, 'nvvm.annotations')
+    nmd = cgutils.get_or_insert_named_metadata(module, "nvvm.annotations")
     nmd.add(md)
 
     # Create the used list
@@ -713,13 +848,13 @@ def set_cuda_kernel(function):
 
     fnptr = function.bitcast(ptrty)
 
-    llvm_used = ir.GlobalVariable(module, usedty, 'llvm.used')
-    llvm_used.linkage = 'appending'
-    llvm_used.section = 'llvm.metadata'
+    llvm_used = ir.GlobalVariable(module, usedty, "llvm.used")
+    llvm_used.linkage = "appending"
+    llvm_used.section = "llvm.metadata"
     llvm_used.initializer = ir.Constant(usedty, [fnptr])
 
     # Remove 'noinline' if it is present.
-    function.attributes.discard('noinline')
+    function.attributes.discard("noinline")
 
 
 def add_ir_version(mod):
@@ -728,4 +863,4 @@ def add_ir_version(mod):
     i32 = ir.IntType(32)
     ir_versions = [i32(v) for v in NVVM().get_ir_version()]
     md_ver = mod.add_metadata(ir_versions)
-    mod.add_named_metadata('nvvmir.version', md_ver)
+    mod.add_named_metadata("nvvmir.version", md_ver)