kernel-tuner 0.1.0b0__tar.gz

kernel_tuner-0.1.0b0/LICENSE

@@ -0,0 +1,14 @@
+* Copyright 2016 Netherlands eScience Center
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+

kernel_tuner-0.1.0b0/MANIFEST.in

@@ -0,0 +1,4 @@
+include setup.py
+include LICENSE
+include README.md
+include requirements.txt

kernel_tuner-0.1.0b0/PKG-INFO

@@ -0,0 +1,136 @@
+Metadata-Version: 1.1
+Name: kernel_tuner
+Version: 0.1.0b0
+Summary: A simple CUDA/OpenCL kernel tuner in Python
+Home-page: http://benvanwerkhoven.github.io/kernel_tuner/
+Author: Ben van Werkhoven
+Author-email: b.vanwerkhoven@esciencecenter.nl
+License: Apache 2.0
+Description:  
+        A simple CUDA/OpenCL kernel tuner in Python
+        ===========================================
+        
+        The goal of this project is to provide a - as simple as possible - tool
+        for tuning CUDA and OpenCL kernels. This implies that any CUDA or OpenCL
+        kernel can be tuned without requiring extensive changes to the original
+        kernel code.
+        
+        A very common problem in GPU programming is that some combination of
+        thread block dimensions and other kernel parameters, like tiling or
+        unrolling factors, results in dramatically better performance than other
+        kernel configurations. The goal of auto-tuning is to automate the
+        process of finding the best performing configuration for a given device.
+        
+        This kernel tuner aims that you can directly use the tuned kernel
+        without introducing any new dependencies. The tuned kernels can
+        afterwards be used independently of the programming environment, whether
+        that is using C/C++/Java/Fortran or Python doesn't matter.
+        
+        The kernel_tuner module currently only contains main one function which
+        is called tune_kernel to which you pass at least the kernel name, a string
+        containing the kernel code, the problem size, a list of kernel function
+        arguments, and a dictionary of tunable parameters. There are also a lot
+        of optional parameters, for a complete list see the full documentation of
+        tune_kernel.
+        
+        Installation
+        ------------
+        | clone the repository
+        |  ``git clone git@github.com:benvanwerkhoven/kernel_tuner.git``
+        | change into the top-level directory
+        |  ``cd kernel_tuner``
+        | install using
+        |  ``pip install -r requirements.txt``
+        |  ``pip install .``
+        
+        Dependencies
+        ------------
+         * Python 2.7 or Python 3.5
+         * PyCuda and/or PyOpenCL (https://mathema.tician.de/software/)
+        
+        Example usage
+        -------------
+        The following shows a simple example for tuning a CUDA kernel:
+        
+        ::
+        
+            kernel_string = """
+            __global__ void vector_add(float *c, float *a, float *b, int n) {
+                int i = blockIdx.x * block_size_x + threadIdx.x;
+                if (i<n) {
+                    c[i] = a[i] + b[i];
+                }
+            }
+            """
+        
+            size = 10000000
+            problem_size = (size, 1)
+        
+            a = numpy.random.randn(size).astype(numpy.float32)
+            b = numpy.random.randn(size).astype(numpy.float32)
+            c = numpy.zeros_like(b)
+            n = numpy.int32(size)
+            args = [c, a, b, n]
+        
+            tune_params = dict()
+            tune_params["block_size_x"] = [128+64*i for i in range(15)]
+        
+            tune_kernel("vector_add", kernel_string, problem_size, args, tune_params)
+        
+        The exact same Python code can be used to tune an OpenCL kernel:
+        
+        ::
+        
+            kernel_string = """
+            __kernel void vector_add(__global float *c, __global float *a, __global float *b, int n) {
+                int i = get_global_id(0);
+                if (i<n) {
+                    c[i] = a[i] + b[i];
+                }
+            }
+            """
+        
+        
+        Or even just a C function, see the example `here <https://github.com/benvanwerkhoven/kernel_tuner/blob/master/examples/c/vector_add.py>`_.
+        
+        You can find these and many - more extensive - example codes, in the
+        `examples <https://github.com/benvanwerkhoven/kernel_tuner/blob/master/examples/>`_
+        directory. See the `full documentation <http://benvanwerkhoven.github.io/kernel_tuner/sphinxdoc/html/index.html>`_
+        for several highly detailed tutorial-style explanations of example
+        kernels and the scripts to tune them.
+        
+        Author
+        ------
+        Ben van Werkhoven <b.vanwerkhoven@esciencenter.nl>
+        
+        Copyright and License
+        ---------------------
+        * Copyright 2016 Netherlands eScience Center
+        
+        Licensed under the Apache License, Version 2.0 (the "License");
+        you may not use this file except in compliance with the License.
+        You may obtain a copy of the License at
+        
+            http://www.apache.org/licenses/LICENSE-2.0
+        
+        Unless required by applicable law or agreed to in writing, software
+        distributed under the License is distributed on an "AS IS" BASIS,
+        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+        See the License for the specific language governing permissions and
+        limitations under the License.
+        
+Keywords: auto-tuning gpu computing pycuda cuda pyopencl opencl
+Platform: UNKNOWN
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Education
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Natural Language :: English
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Development Status :: 4 - Beta

kernel_tuner-0.1.0b0/README.md

@@ -0,0 +1,134 @@
+
+A simple CUDA/OpenCL kernel tuner in Python
+====================================
+[![Build Status](https://api.travis-ci.org/benvanwerkhoven/kernel_tuner.svg?branch=master)](https://travis-ci.org/benvanwerkhoven/kernel_tuner)
+[![Codacy Badge](https://api.codacy.com/project/badge/grade/016dc85044ab4d57b777449d93275608)](https://www.codacy.com/app/b-vanwerkhoven/kernel_tuner)
+[![Codacy Badge](https://api.codacy.com/project/badge/coverage/016dc85044ab4d57b777449d93275608)](https://www.codacy.com/app/b-vanwerkhoven/kernel_tuner)
+
+The goal of this project is to provide a - as simple as possible - tool 
+for tuning CUDA and OpenCL kernels. This implies that any CUDA or OpenCL 
+kernel can be tuned without requiring extensive changes to the original 
+kernel code.
+
+A very common problem in GPU programming is that some combination of 
+thread block dimensions and other kernel parameters, like tiling or 
+unrolling factors, results in dramatically better performance than other 
+kernel configurations. The goal of auto-tuning is to automate the 
+process of finding the best performing configuration for a given device.
+
+This kernel tuner aims that you can directly use the tuned kernel
+without introducing any new dependencies. The tuned kernels can
+afterwards be used independently of the programming environment, whether
+that is using C/C++/Java/Fortran or Python doesn't matter.
+
+The kernel_tuner module currently only contains main one function which
+is called tune_kernel to which you pass at least the kernel name, a string
+containing the kernel code, the problem size, a list of kernel function
+arguments, and a dictionary of tunable parameters. There are also a lot
+of optional parameters, for a complete list see the full documentation of
+[tune_kernel](http://benvanwerkhoven.github.io/kernel_tuner/sphinxdoc/html/details.html).
+
+Documentation
+-------------
+The full documentation is available [here](http://benvanwerkhoven.github.io/kernel_tuner/sphinxdoc/html/index.html).
+
+Installation
+------------
+clone the repository  
+    `git clone git@github.com:benvanwerkhoven/kernel_tuner.git`  
+change into the top-level directory  
+    `cd kernel_tuner`  
+install using  
+    `pip install -r requirements.txt`  
+    `pip install .`
+
+Dependencies
+------------
+ * Python 2.7 or Python 3.5
+ * PyCuda and/or PyOpenCL (https://mathema.tician.de/software/)
+
+Example usage
+-------------
+The following shows a simple example for tuning a CUDA kernel:
+
+```python
+kernel_string = """
+__global__ void vector_add(float *c, float *a, float *b, int n) {
+    int i = blockIdx.x * block_size_x + threadIdx.x;
+    if (i<n) {
+        c[i] = a[i] + b[i];
+    }
+}
+"""
+
+size = 10000000
+problem_size = (size, 1)
+
+a = numpy.random.randn(size).astype(numpy.float32)
+b = numpy.random.randn(size).astype(numpy.float32)
+c = numpy.zeros_like(b)
+n = numpy.int32(size)
+args = [c, a, b, n]
+
+tune_params = dict()
+tune_params["block_size_x"] = [128+64*i for i in range(15)]
+
+tune_kernel("vector_add", kernel_string, problem_size, args, tune_params)
+```
+The exact same Python code can be used to tune an OpenCL kernel:
+```python
+kernel_string = """
+__kernel void vector_add(__global float *c, __global float *a, __global float *b, int n) {
+    int i = get_global_id(0);
+    if (i<n) {
+        c[i] = a[i] + b[i];
+    }
+}
+"""
+```
+Or even just a C function, see the example [here](https://github.com/benvanwerkhoven/kernel_tuner/blob/master/examples/c/vector_add.py).
+
+You can find these and many - more extensive - example codes, in the [examples](https://github.com/benvanwerkhoven/kernel_tuner/blob/master/examples/) directory.
+See the [full documentation](http://benvanwerkhoven.github.io/kernel_tuner/sphinxdoc/html/index.html) for several highly detailed tutorial-style explanations of example kernels and the scripts to tune them.
+
+Tuning host and kernel code
+---------------------------
+It is also possible to tune for combinations of tunable parameters in both host and kernel code. This allows for a number of powerfull things, such as tuning the number of streams for a kernel that uses CUDA Streams or OpenCL Command Queues to overlap transfers between host and device with kernel execution. This can be done in combination with tuning the parameters inside the kernel code. See the [convolution_streams example code](https://github.com/benvanwerkhoven/kernel_tuner/blob/master/examples/) and the [documentation](http://benvanwerkhoven.github.io/kernel_tuner/sphinxdoc/html/hostcode.html) for a detailed explanation of the kernel tuner Python script.
+
+Correctness verification
+------------------------
+Optionally, you can let the kernel tuner verify the output of every kernel it compiles and benchmarks,
+by passing an `answer` list. This list matches the list of arguments to the kernel, but contains the expected output of the kernel. Input arguments are replaced with None.
+```python
+answer = [a+b, None, None]  # the order matches the arguments (in args) to the kernel
+tune_kernel("vector_add", kernel_string, problem_size, args, tune_params, answer=answer)
+```
+
+Contribution guide
+------------------
+The kernel tuner follows the Google Python style guide, with Sphinxdoc docstrings for module public functions. If you want to
+contribute to the project please fork it, create a branch including your addition, and create a pull request.
+
+The tests use relative imports and can be run directly after making
+changes to the code. To run all tests use `nosetests` in the main directory.
+To run the examples after code changes, you need to run `pip install --upgrade .` in the main directory.
+Documentation is generated by typing `make html` in the doc directory, the contents
+of `doc/build/html/` should then be copied to `sphinxdoc` directory of the `gh-pages` branch.
+
+Before creating a pull request please ensure the following:
+* You have written unit tests to test your additions
+* All unit tests pass
+* The examples still work and produce the same (or better) results
+* The code is compatible with both Python 2.7 and Python 3.5
+* An entry about the change or addition is created in CHANGELOG.md
+
+Contributing authors so far:
+* Ben van Werkhoven
+* Berend Weel
+
+Related work
+------------
+You may also like [CLTune](https://github.com/CNugteren/CLTune) by Cedric
+Nugteren. CLTune is a C++ library for kernel tuning and supports various
+advanced features like machine learning to optimize the time spent on tuning
+kernels.

kernel_tuner-0.1.0b0/kernel_tuner/__init__.py

@@ -0,0 +1,3 @@
+
+
+from kernel_tuner.interface import tune_kernel, run_kernel

kernel_tuner-0.1.0b0/kernel_tuner/c.py

@@ -0,0 +1,217 @@
+""" This module contains the functionality for running and compiling C functions """
+
+import numpy
+import ctypes as C
+import subprocess
+import os
+import errno
+
+import numpy.ctypeslib
+
+from kernel_tuner.util import get_temp_filename, delete_temp_file
+
+class CFunctions(object):
+    """Class that groups the code for running and compiling C functions"""
+
+    def __init__(self, iterations=7):
+        """instantiate CFunctions object used for interacting with C code
+
+        :param iterations: Number of iterations used while benchmarking a kernel, 7 by default.
+        :type iterations: int
+        """
+        self.ITERATIONS = iterations
+        self.max_threads = 1024
+
+        #test if nvcc is available, otherwise use gcc
+        self.compiler = "nvcc"
+        try:
+            subprocess.check_call([self.compiler, "--version"], stdout=open(os.devnull, 'w'))
+        except OSError as e:
+            self.compiler = "gcc"
+            if e.errno != errno.ENOENT:
+                raise e
+
+    def ready_argument_list(self, arguments):
+        """ready argument list to be passed to the C function
+
+        :param arguments: List of arguments to be passed to the C function.
+            The order should match the argument list on the C function.
+            Allowed values are numpy.ndarray, and/or numpy.int32, numpy.float32, and so on.
+        :type arguments: list(numpy objects)
+
+        :returns: A list of arguments that can be passed to the C function.
+        :rtype: list()
+        """
+        self.arg_mapping = dict()
+        ctype_args = []
+        for arg in arguments:
+            if isinstance(arg, numpy.ndarray):
+                if arg.dtype == 'float32':
+                    ctype_args.append(arg.ctypes.data_as(C.POINTER(C.c_float)))
+                elif arg.dtype == 'float64':
+                    ctype_args.append(arg.ctypes.data_as(C.POINTER(C.c_double)))
+                elif arg.dtype == 'int32':
+                    ctype_args.append(arg.ctypes.data_as(C.POINTER(C.c_int)))
+                else:
+                    raise TypeError("unknown dtype for ndarray")
+                self.arg_mapping[str(ctype_args[-1])] = arg.shape
+            elif numpy.isscalar(arg):
+                if hasattr(arg, 'dtype'):
+                    if str(arg.dtype).startswith('int'):
+                        ctype_args.append(int(arg))
+                    elif str(arg.dtype).startswith('float'):
+                        ctype_args.append(float(arg))
+                    else:
+                        raise TypeError("Argument is scalar with a dtype, but does not start with int or float")
+                else:
+                    ctype_args.append(arg)
+            else:
+                raise TypeError("Argument is not a numpy.ndarray and is not a scalar %s" % type(arg))
+
+        return ctype_args
+
+
+    def compile(self, kernel_name, kernel_string):
+        """call the C compiler to compile the kernel, return the function
+
+        :param kernel_name: The name of the kernel to be compiled, used to lookup the
+            function after compilation.
+        :type kernel_name: string
+
+        :param kernel_string: The C code that contains the function `kernel_name`
+        :type kernel_string: string
+
+        :returns: An ctypes function that can be called directly.
+        :rtype: ctypes._FuncPtr
+        """
+        filename = get_temp_filename()
+        source_file = filename+".cc"
+        kernel_string = "extern \"C\" {\n" + kernel_string + "\n}"
+
+        compiler_options = ["-fPIC"]
+        if "#include <omp.h>" in kernel_string:
+            compiler_options.append("-fopenmp")
+
+        if self.compiler == "nvcc":
+            source_file = source_file[:-1] + "u"
+            compiler_options = ["-Xcompiler=" + c for c in compiler_options]
+            #might be better to have an optional argument to pass the desired compute capability
+            compiler_options += ["-arch=compute_52"]
+
+        lib_args = []
+        if "CL/cl.h" in kernel_string:
+            lib_args = ["-lOpenCL"]
+
+        try:
+            with open(source_file, 'w') as f:
+                f.write(kernel_string)
+
+            subprocess.check_call([self.compiler, "-c", source_file] + compiler_options + ["-o", filename+".o"])
+            subprocess.check_call([self.compiler, filename+".o"] + compiler_options + [ "-shared", "-o", filename+".so"] + lib_args)
+
+            self.lib = numpy.ctypeslib.load_library(filename, '.')
+
+        finally:
+            delete_temp_file(source_file)
+            delete_temp_file(filename+".o")
+            delete_temp_file(filename+".so")
+
+        func = getattr(self.lib, kernel_name)
+        func.restype = C.c_float
+
+        return func
+
+    def benchmark(self, func, c_args, threads, grid):
+        """runs the kernel repeatedly, returns averaged returned value
+
+        The C function tuning is a little bit more flexible than direct CUDA
+        or OpenCL kernel tuning. The C function needs to measure time, or some
+        other quality metric you wish to tune on, on its own and should
+        therefore return a single floating-point value.
+
+        Benchmark runs the C function repeatedly and returns the average of the
+        values returned by the C function. The number of iterations is set
+        during the creation of the CFunctions object. For all measurements the
+        lowest and highest values are discarded and the rest is included in the
+        average. The reason for this is to be robust against initialization
+        artifacts and other exceptional cases.
+
+        :param func: A C function compiled for this specific configuration
+        :type func: ctypes._FuncPtr
+
+        :param c_args: A list of arguments to the function, order should match the
+            order in the code. The list should be prepared using
+            ready_argument_list().
+        :type c_args: list()
+
+        :param threads: Ignored, but left as argument for now to have the same
+            interface as CudaFunctions and OpenCLFunctions.
+        :type threads: any
+
+        :param grid: Ignored, but left as argument for now to have the same
+            interface as CudaFunctions and OpenCLFunctions.
+        :type grid: any
+
+        :returns: A robust average of values returned by the C function.
+        :rtype: float
+        """
+        results = []
+        for _ in range(self.ITERATIONS):
+            value = self.run_kernel(func, c_args, threads, grid)
+            results.append(value)
+        results = sorted(results)
+        return numpy.mean(results[1:-1])
+
+
+    def run_kernel(self, func, c_args, threads, grid):
+        """runs the kernel once, returns whatever the kernel returns
+
+        :param func: A C function compiled for this specific configuration
+        :type func: ctypes._FuncPtr
+
+        :param c_args: A list of arguments to the function, order should match the
+            order in the code. The list should be prepared using
+            ready_argument_list().
+        :type c_args: list()
+
+        :param threads: Ignored, but left as argument for now to have the same
+            interface as CudaFunctions and OpenCLFunctions.
+        :type threads: any
+
+        :param grid: Ignored, but left as argument for now to have the same
+            interface as CudaFunctions and OpenCLFunctions.
+        :type grid: any
+
+        :returns: A robust average of values returned by the C function.
+        :rtype: float
+        """
+        return func(*c_args)
+
+
+    def memset(self, allocation, value, size):
+        """set the memory in allocation to the value in value
+
+        :param allocation: A memory allocation unit
+        :type allocation: pycuda.driver.DeviceAllocation
+
+        :param value: The value to set the memory to
+        :type value: a single 32-bit float or int
+
+        :param size: The size of to the allocation unit in bytes
+        :type size: int
+        """
+        C.memset(allocation, value, size)
+
+
+    def memcpy_dtoh(self, dest, src):
+        """a simple memcpy expects a ctypes pointer, returns a numpy array
+
+        :param dest: A numpy array to store the data
+        :type dest: numpy.ndarray
+
+        :param src: A ctypes pointer to some memory allocation
+        :type src: ctypes.pointer
+        """
+        dest[:] = numpy.ctypeslib.as_array(src, shape=self.arg_mapping[str(src)])
+
+

kernel_tuner-0.1.0b0/kernel_tuner/core.py

@@ -0,0 +1,131 @@
+""" Module for grouping the core functionality needed by most runners """
+from __future__ import print_function
+
+from kernel_tuner.cuda import CudaFunctions
+from kernel_tuner.opencl import OpenCLFunctions
+from kernel_tuner.c import CFunctions
+from kernel_tuner.util import *
+
+def get_device_interface(lang, device, platform):
+    if lang == "CUDA":
+        dev = CudaFunctions(device)
+    elif lang == "OpenCL":
+        dev = OpenCLFunctions(device, platform)
+    elif lang == "C":
+        dev = CFunctions()
+    else:
+        raise UnImplementedException("Sorry, support for languages other than CUDA, OpenCL, or C is not implemented yet")
+    return dev
+
+def check_kernel_correctness(dev, func, gpu_args, threads, grid, answer, params, verbose, atol=1e-6):
+    """runs the kernel once and checks the result against answer"""
+    for result, expected in zip(gpu_args, answer):
+        if expected is not None:
+            dev.memset(result, 0, expected.nbytes)
+    try:
+        dev.run_kernel(func, gpu_args, threads, grid)
+    except Exception as e:
+        if "too many resources requested for launch" in str(e) or "OUT_OF_RESOURCES" in str(e):
+            #ignore this error for now, it will show up when benchmarking the kernel
+            return True
+        else:
+            raise e
+    correct = True
+    for result,expected in zip(gpu_args,answer):
+        if expected is not None:
+            result_host = numpy.zeros_like(expected)
+            dev.memcpy_dtoh(result_host, result)
+            output_test = numpy.allclose(result_host.ravel(), expected.ravel(), atol=atol)
+            if not output_test and verbose:
+                print("Error: " + get_config_string(params) + " detected during correctness check")
+                print("Printing kernel output and expected result, set verbose=False to suppress this debug print")
+                numpy.set_printoptions(edgeitems=500)
+                print("Kernel output:")
+                print(result_host)
+                print("Expected:")
+                print(expected)
+            correct = correct and output_test
+    if not correct:
+        raise Exception("Error: " + get_config_string(params) + " failed correctness check")
+    return correct
+
+def compile_kernel(dev, kernel_name, kernel_string, params, grid, instance_string, verbose):
+    """compile the kernel for this specific instance"""
+
+    #prepare kernel_string for compilation
+    name, kernel_string = setup_kernel_strings(kernel_name, kernel_string, params, grid, instance_string)
+
+    #compile kernel_string into device func
+    func = None
+    try:
+        func = dev.compile(name, kernel_string)
+    except Exception as e:
+        #compiles may fail because certain kernel configurations use too
+        #much shared memory for example, the desired behavior is to simply
+        #skip over this configuration and try the next one
+        if "uses too much shared data" in str(e):
+            if verbose:
+                print("skipping config", instance_string, "reason: too much shared memory used")
+        else:
+            raise e
+    return func
+
+def benchmark(dev, func, gpu_args, threads, grid, instance_string, verbose):
+    """benchmark the kernel instance"""
+    time = None
+    try:
+        time = dev.benchmark(func, gpu_args, threads, grid)
+    except Exception as e:
+        #some launches may fail because too many registers are required
+        #to run the kernel given the current thread block size
+        #the desired behavior is to simply skip over this configuration
+        #and proceed to try the next one
+        if "too many resources requested for launch" in str(e) or "OUT_OF_RESOURCES" in str(e):
+            if verbose:
+                print("skipping config", instance_string, "reason: too many resources requested for launch")
+        else:
+            print("Error while benchmarking:", instance_string)
+            raise e
+    return time
+
+def compile_and_benchmark(dev, gpu_args, kernel_name, original_kernel, params,
+        problem_size, grid_div_y, grid_div_x, cmem_args, answer, atol, instance_string, verbose):
+
+    #setup thread block and grid dimensions
+    threads, grid = setup_block_and_grid(dev, problem_size, grid_div_y, grid_div_x, params, instance_string, verbose)
+    if threads is None:
+        return None
+
+    temp_files = dict()
+
+    try:
+        #obtain the kernel_string and prepare additional files, if any
+        if isinstance(original_kernel, list):
+            kernel_string, temp_files = prepare_list_of_files(original_kernel, params, grid)
+        else:
+            kernel_string = get_kernel_string(original_kernel)
+
+        #compile the kernel
+        func = compile_kernel(dev, kernel_name, kernel_string, params, grid, instance_string, verbose)
+        if func is None:
+            return None
+
+        #add constant memory arguments to compiled module
+        if cmem_args is not None:
+            dev.copy_constant_memory_args(cmem_args)
+
+        #test kernel for correctness and benchmark
+        if answer is not None:
+            check_kernel_correctness(dev, func, gpu_args, threads, grid, answer, params, verbose, atol)
+
+        #benchmark
+        time = benchmark(dev, func, gpu_args, threads, grid, instance_string, verbose)
+
+    #clean up any temporary files
+    finally:
+        for v in temp_files.values():
+            delete_temp_file(v)
+
+    return time
+
+