gpuNUFFT 0.7.1__tar.gz → 0.7.3__tar.gz

{gpuNUFFT-0.7.1 → gpuNUFFT-0.7.3}/CUDA/inc/cuda_utils.hpp

@@ -110,13 +110,28 @@ inline void allocateAndSetMem(TypeName **device_ptr, IndType num_elements,
  */
 template <typename TypeName>
 inline void copyDeviceToDevice(TypeName *device_ptr_src,
-                               TypeName *device_ptr_dest, IndType num_elements)
+                               TypeName *device_ptr_dest, IndType num_elements
+                               )
 {
   HANDLE_ERROR(cudaMemcpy(device_ptr_dest, device_ptr_src,
                           num_elements * sizeof(TypeName),
                           cudaMemcpyDeviceToDevice));
 }
 
+/** \brief CUDA memcpy call to copy data from device ptr to device ptr
+ *
+ * @param device_ptr_src   source device pointer
+ * @param device_ptr_dest  destination device pointer
+ * @param num_elements     amount of elements of size TypeName
+ */
+template <typename TypeName>
+inline void copyDeviceToDeviceAsync(TypeName *device_ptr_src,
+                               TypeName *device_ptr_dest, IndType num_elements, cudaStream_t stream=0)
+{
+  HANDLE_ERROR(cudaMemcpyAsync(device_ptr_dest, device_ptr_src,
+                          num_elements * sizeof(TypeName),
+                          cudaMemcpyDeviceToDevice, stream));
+}
 /** \brief Copy CUDA memory from device to host
  *
  * @param device_ptr    device pointer

{gpuNUFFT-0.7.1 → gpuNUFFT-0.7.3}/CUDA/src/gpu/python/gpuNUFFT_operator_python_factory.cpp

@@ -107,6 +107,7 @@ class GpuNUFFTPythonOperator
     gpuNUFFT::GpuNUFFTOperatorFactory factory;
     gpuNUFFT::GpuNUFFTOperator *gpuNUFFTOp;
     int trajectory_length, n_coils, dimension;
+    float osr;
     bool has_sense_data;
     gpuNUFFT::Dimensions imgDims;
     // sensitivity maps
@@ -129,7 +130,7 @@ class GpuNUFFTPythonOperator
     public:
     GpuNUFFTPythonOperator(py::array_t<DType> kspace_loc, py::array_t<int> image_size, int num_coils,
     py::array_t<std::complex<DType>> sense_maps,  py::array_t<DType> density_comp, int kernel_width=3,
-    int sector_width=8, float osr=2, bool balance_workload=1) 
+    int sector_width=8, float osr=2, bool balance_workload=1) : osr(osr)
     {
         // k-space coordinates
         py::buffer_info sample_loc = kspace_loc.request();
@@ -140,12 +141,11 @@ class GpuNUFFTPythonOperator
 
         // density compensation weights
         gpuNUFFT::Array<DType> density_compArray;
-        //if(density_comp.has_value())
-        //{
+        if(density_comp == Py_None)
+        {
             density_compArray = readNumpyArray(density_comp);
             density_compArray.dim.length = trajectory_length;
-            // No need else as the init is by default with 0 length and density comp is not applied
-        //}
+        }
 
         // image size
         py::buffer_info img_dim = image_size.request();
@@ -305,7 +305,6 @@ class GpuNUFFTPythonOperator
         allocate_pinned_memory(&densArray, n_samples * sizeof(CufftType));
         densArray.dim.length = n_samples;
 
-        // TODO: Allocate directly on device and set with kernel.
         for (long int cnt = 0; cnt < n_samples; cnt++)
         {
           densArray.data[cnt].x = 1.0;
@@ -324,7 +323,7 @@ class GpuNUFFTPythonOperator
 
         gpuNUFFT::GpuArray<CufftType> image_gpu;
         image_gpu.dim = imgDims;
-        allocateDeviceMem(&image_gpu.data, imgDims.count());
+        allocateDeviceMem(&image_gpu.data, image_gpu.dim.count());
 
         if (DEBUG && (cudaDeviceSynchronize() != cudaSuccess))
           printf("error at adj thread synchronization a: %s\n",

{gpuNUFFT-0.7.1 → gpuNUFFT-0.7.3}/CUDA/src/gpu/std_gpuNUFFT_kernels.cu

@@ -41,6 +41,7 @@ __global__ void updateDensityCompKernel(DType2* density_data, DType2* estimation
     DType2 data_p = density_data[t];
     DType2 esti_p = estimation_data[t];
     data_p.x *= rsqrtf(esti_p.x * esti_p.x + esti_p.y * esti_p.y);
+    data_p.y = 0;
     density_data[t] = data_p;
     t = t + blockDim.x*gridDim.x;
   } 

{gpuNUFFT-0.7.1 → gpuNUFFT-0.7.3}/CUDA/src/gpuNUFFT_operator.cpp

@@ -457,7 +457,7 @@ void gpuNUFFT::GpuNUFFTOperator::performGpuNUFFTAdj(
     if (DEBUG && (cudaDeviceSynchronize() != cudaSuccess))
       fprintf(stderr, "error at adj  thread synchronization 2: %s\n",
               cudaGetErrorString(cudaGetLastError()));
-    if (gpuNUFFTOut == CONVOLUTION)
+    if (gpuNUFFTOut == CONVOLUTION || gpuNUFFTOut == DENSITY_ESTIMATION)
     {
       if (DEBUG)
         printf("stopping output after CONVOLUTION step\n");
@@ -939,10 +939,11 @@ void gpuNUFFT::GpuNUFFTOperator::performForwardGpuNUFFT(
     printf("Memory allocation: %.2f ms\n", stopTiming());
 
   int err;
-
+  cudaStream_t new_stream, old_stream;
   // iterate over coils and compute result
   for (int coil_it = 0; coil_it < n_coils; coil_it += n_coils_cc)
   {
+    cudaStreamCreate(&new_stream);
     unsigned long int data_coil_offset = (long int)coil_it * data_count;
     unsigned long int im_coil_offset = coil_it * (long int)imdata_count;
 
@@ -954,32 +955,52 @@ void gpuNUFFT::GpuNUFFTOperator::performForwardGpuNUFFT(
       // perform automatically "repeating" of input image in case
       // of existing sensitivity data
       for (int cnt = 0; cnt < n_coils_cc; cnt++)
-        copyDeviceToDevice<DType2>(imgData_gpu.data,
-                                   imdata_d + cnt * imdata_count, imdata_count);
+        copyDeviceToDeviceAsync<DType2>(imgData_gpu.data,
+                                   imdata_d + cnt * imdata_count, imdata_count, new_stream);
     else
-      copyDeviceToDevice<DType2>(imgData_gpu.data + im_coil_offset, imdata_d,
-                                 imdata_count * n_coils_cc);
+      copyDeviceToDeviceAsync<DType2>(imgData_gpu.data + im_coil_offset, imdata_d,
+                                 imdata_count * n_coils_cc, new_stream);
 
     // reset temp arrays
-    cudaMemset(gdata_d, 0,
-               sizeof(CufftType) * gi_host->grid_width_dim * n_coils_cc);
-    cudaMemset(data_d, 0, sizeof(CufftType) * data_count * n_coils_cc);
+    cudaMemsetAsync(gdata_d, 0,
+               sizeof(CufftType) * gi_host->grid_width_dim * n_coils_cc, new_stream);
+    cudaMemsetAsync(data_d, 0, sizeof(CufftType) * data_count * n_coils_cc, new_stream);
 
-    if (DEBUG && (cudaDeviceSynchronize() != cudaSuccess))
+    if (DEBUG && (cudaStreamSynchronize(new_stream)!= cudaSuccess))
       printf("error at thread synchronization 1: %s\n",
              cudaGetErrorString(cudaGetLastError()));
 
     if (this->applySensData())
     {
-      copyToDevice(this->sens.data + im_coil_offset, sens_d,
-                   imdata_count * n_coils_cc);
+      copyToDeviceAsync(this->sens.data + im_coil_offset, sens_d,
+                   imdata_count * n_coils_cc, new_stream);
       performSensMul(imdata_d, sens_d, gi_host, false);
     }
 
+	 if(gpuNUFFTOut == DENSITY_ESTIMATION)
+      {
+        // convolution and resampling to non-standard trajectory
+        forwardConvolution(data_d, crds_d, imdata_d, NULL, sectors_d,
+                       sector_centers_d, gi_host);
+        if (DEBUG && (cudaStreamSynchronize(new_stream) != cudaSuccess))
+              printf("error at thread synchronization 7: %s\n",
+                 cudaGetErrorString(cudaGetLastError()));
+
+        if (debugTiming)
+          printf("Forward Convolution: %.2f ms\n", stopTiming());
+        // write result in correct order back into output array
+        writeOrderedGPU(data_sorted_d, data_indices_d, data_d,
+                    (int)this->kSpaceTraj.count(), n_coils_cc);
+        copyDeviceToDeviceAsync(data_sorted_d, data_d, data_count * n_coils_cc, new_stream);
+        if ((coil_it + n_coils_cc) < (n_coils))
+            continue;
+        freeTotalDeviceMemory(imdata_d, NULL);
+        this->freeDeviceMemory();
+        return;
+    } 
     // apodization Correction
     performForwardDeapodization(imdata_d, deapo_d, gi_host);
-	  
-    if (DEBUG && (cudaDeviceSynchronize() != cudaSuccess))
+    if (DEBUG && (cudaStreamSynchronize(new_stream) != cudaSuccess))
       printf("error at thread synchronization 2: %s\n",
              cudaGetErrorString(cudaGetLastError()));
     // resize by oversampling factor and zero pad
@@ -988,13 +1009,13 @@ void gpuNUFFT::GpuNUFFTOperator::performForwardGpuNUFFT(
     if (debugTiming)
       startTiming();
 
-    if (DEBUG && (cudaDeviceSynchronize() != cudaSuccess))
+    if (DEBUG && (cudaStreamSynchronize(new_stream) != cudaSuccess))
       printf("error at thread synchronization 3: %s\n",
              cudaGetErrorString(cudaGetLastError()));
     // shift image to get correct zero frequency position
     performFFTShift(gdata_d, INVERSE, getGridDims(), gi_host);
 
-    if (DEBUG && (cudaDeviceSynchronize() != cudaSuccess))
+    if (DEBUG && (cudaStreamSynchronize(new_stream) != cudaSuccess))
       printf("error at thread synchronization 4: %s\n",
              cudaGetErrorString(cudaGetLastError()));
     // eventually free imdata_d
@@ -1012,12 +1033,12 @@ void gpuNUFFT::GpuNUFFTOperator::performForwardGpuNUFFT(
       c++;
     }
 
-    if (DEBUG && (cudaDeviceSynchronize() != cudaSuccess))
+    if (DEBUG && (cudaStreamSynchronize(new_stream) != cudaSuccess))
       printf("error at thread synchronization 5: %s\n",
              cudaGetErrorString(cudaGetLastError()));
     performFFTShift(gdata_d, FORWARD, getGridDims(), gi_host);
 
-    if (DEBUG && (cudaDeviceSynchronize() != cudaSuccess))
+    if (DEBUG && (cudaStreamSynchronize(new_stream) != cudaSuccess))
       printf("error at thread synchronization 6: %s\n",
              cudaGetErrorString(cudaGetLastError()));
 
@@ -1030,7 +1051,7 @@ void gpuNUFFT::GpuNUFFTOperator::performForwardGpuNUFFT(
     // convolution and resampling to non-standard trajectory
     forwardConvolution(data_d, crds_d, gdata_d, NULL, sectors_d,
                        sector_centers_d, gi_host);
-    if (DEBUG && (cudaDeviceSynchronize() != cudaSuccess))
+    if (DEBUG && (cudaStreamSynchronize(new_stream) != cudaSuccess))
       printf("error at thread synchronization 7: %s\n",
              cudaGetErrorString(cudaGetLastError()));
 
@@ -1038,15 +1059,20 @@ void gpuNUFFT::GpuNUFFTOperator::performForwardGpuNUFFT(
       printf("Forward Convolution: %.2f ms\n", stopTiming());
 
     performFFTScaling(data_d, gi_host->data_count, gi_host);
-    if (DEBUG && (cudaDeviceSynchronize() != cudaSuccess))
+    if (DEBUG && (cudaStreamSynchronize(new_stream) != cudaSuccess))
       printf("error: at thread synchronization 8: %s\n",
              cudaGetErrorString(cudaGetLastError()));
     
     // write result in correct order back into output array
     writeOrderedGPU(data_sorted_d, data_indices_d, data_d,
                     (int)this->kSpaceTraj.count(), n_coils_cc);
-
-    copyDeviceToDevice(data_sorted_d, data_d, data_count * n_coils_cc);
+    if(coil_it > 1)
+    {
+      cudaStreamSynchronize(old_stream);
+      cudaStreamDestroy(old_stream);
+    }
+    copyDeviceToDeviceAsync(data_sorted_d, data_d, data_count * n_coils_cc, new_stream);
+    old_stream = new_stream;
   }  // iterate over coils
 
   freeTotalDeviceMemory(imdata_d, NULL);
@@ -1172,8 +1198,6 @@ void gpuNUFFT::GpuNUFFTOperator::performForwardGpuNUFFT(
       performSensMul(imdata_d, sens_d, gi_host, false);
     }
 
-    // apodization Correction
-    performForwardDeapodization(imdata_d, deapo_d, gi_host);
 	  if(gpuNUFFTOut == DENSITY_ESTIMATION)
 	  {
 	      forwardConvolution(data_d, crds_d, imdata_d, NULL, sectors_d,
@@ -1188,6 +1212,8 @@ void gpuNUFFT::GpuNUFFTOperator::performForwardGpuNUFFT(
         this->freeDeviceMemory();
         return;
     }
+    // apodization Correction
+    performForwardDeapodization(imdata_d, deapo_d, gi_host);
     if (DEBUG && (cudaThreadSynchronize() != cudaSuccess))
       printf("error at thread synchronization 2: %s\n",
              cudaGetErrorString(cudaGetLastError()));

{gpuNUFFT-0.7.1 → gpuNUFFT-0.7.3}/PKG-INFO

@@ -1,5 +1,5 @@
 Metadata-Version: 2.1
 Name: gpuNUFFT
-Version: 0.7.1
+Version: 0.7.3
 Summary: gpuNUFFT - An open source GPU Library for 3D Gridding and NUFFT
 License-File: LICENSE

{gpuNUFFT-0.7.1 → gpuNUFFT-0.7.3}/gpuNUFFT.egg-info/PKG-INFO

@@ -1,5 +1,5 @@
 Metadata-Version: 2.1
 Name: gpuNUFFT
-Version: 0.7.1
+Version: 0.7.3
 Summary: gpuNUFFT - An open source GPU Library for 3D Gridding and NUFFT
 License-File: LICENSE

{gpuNUFFT-0.7.1 → gpuNUFFT-0.7.3}/setup.py

@@ -103,7 +103,7 @@ class CMakeBuild(build_ext):
 
 setup(
     name="gpuNUFFT",
-    version="0.7.1",
+    version="0.7.3",
     description="gpuNUFFT - An open source GPU Library for 3D Gridding and NUFFT",
     ext_modules=[
         CMakeExtension("gpuNUFFT", sourcedir=os.path.join("CUDA")),