ngsolve 6.2.2506.post216.dev0__cp314-cp314-manylinux_2_28_x86_64.whl

ngsolve-6.2.2506.post216.dev0.data/data/include/netgen/cuda_ngbla.hpp

@@ -0,0 +1,317 @@
+#ifndef CUDA_NGBLA
+#define CUDA_NGBLA
+
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+
+#include <vector.hpp>
+#include <matrix.hpp>
+
+#include "cuda_ngstd.hpp"
+#include "linalg_kernels.hpp"
+
+
+namespace ngla
+{
+  cublasHandle_t Get_CuBlas_Handle ();
+}
+
+
+namespace ngbla
+{
+  using namespace ngs_cuda;
+    
+  // template<> struct trivtrans<Dev<double>> { static constexpr bool value = true; };
+  template<> struct is_scalar_type<Dev<double>> { static constexpr bool value = true; };
+  
+
+  template <typename T>  
+  class Vector<Dev<T>> : public FlatVector<Dev<T>>
+  { 
+    using FlatVector<Dev<T>>::Size;
+    using FlatVector<Dev<T>>::Data;
+
+  public:
+    Vector (Vector&) = delete;
+    Vector (Vector&&v2)
+      : FlatVector<Dev<T>>(v2.Size(), v2.Data())
+    {
+      v2.layout = { nullptr, 0 };      
+    }
+         
+    Vector (size_t asize)
+      : FlatVector<Dev<T>>(asize, Dev<T>::Malloc(asize)) { ; }
+         
+    Vector (FlatVector<T> vec)
+      : FlatVector<Dev<T>>(vec.Size(), Dev<T>::Malloc(vec.Size()))
+    {
+      H2D(vec);
+    }
+    
+    ~Vector()
+    {
+      Dev<T>::Free(Data());
+    }
+
+
+    template<typename TB>
+    Vector & operator= (const Expr<TB> & v)
+    {
+      MatExpr<FlatVector<Dev<T>> >::operator= (v);
+      return *this;
+    }
+
+    
+    void D2H (FlatVector<T> vec) const
+    {
+      cudaMemcpy (vec.Data(), Data(), sizeof(T)*Size(), cudaMemcpyDeviceToHost);
+    }
+
+    void H2D (FlatVector<T> vec)
+    {
+      cudaMemcpy (Data(), vec.Data(), sizeof(T)*Size(), cudaMemcpyHostToDevice);
+    }
+
+    Vector<T> D2H() const
+    {
+      Vector<T> vh(Size());
+      D2H (vh);
+      return vh;
+    }
+  };
+  
+  inline Vector<double> D2H (FlatVector<Dev<double>> dvec)
+  {
+    Vector<double> hvec(dvec.Size());
+    cudaMemcpy (hvec.Data(), dvec.Data(), sizeof(double)*hvec.Size(), cudaMemcpyDeviceToHost);
+    return hvec;
+  }
+
+#ifdef OLDOLD
+#ifdef __CUDACC__  
+  template <typename TS, typename TD>
+  __global__ void kernel_Assign (size_t n,  TD pod_dst, TS pod_src)
+  {
+    auto dst = *pod_dst;
+    auto src = *pod_src;
+    
+    int tid = blockIdx.x*blockDim.x+threadIdx.x;
+    for (int i = tid; i < n; i += blockDim.x*gridDim.x)
+      if (i < 5)
+        dst(i) = src.S();
+      else
+        dst(i) = src.A()(i);
+  }
+#endif
+
+  template <typename T>
+  class AsPOD
+  {
+    std::array<char, sizeof(T)> data;
+  public:
+    AsPOD(const AsPOD&) = default;
+    INLINE AsPOD (const T & adata)
+    {
+      char * pdata = (char*)(void*)&adata;
+      for (int i = 0; i < sizeof(T); i++)
+        data[i] = pdata[i];
+    }
+
+    INLINE const T & operator* () const
+    {
+      T * val = (T*)(void*)&data[0];
+      return *val;
+    }
+    INLINE int operator[] (int i) const { return data[i]; }
+  };
+#endif
+  
+  
+
+  template <typename TOP, typename T, typename TS, typename TDIST, typename TB>
+  class assign_trait<TOP, VectorView<T,TS,TDIST>, TB,
+                     enable_if_t < std::is_same<std::invoke_result_t<VectorView<T,TS,TDIST>,size_t>, Dev<double>&>::value, int>>
+  {
+  public:
+    static INLINE VectorView<T,TS,TDIST> & Assign (MatExpr<VectorView<T,TS,TDIST>> & self, const Expr<TB> & v)
+    {
+
+#ifdef __CUDACC__ 
+      
+      ngs_cuda::DeviceParallelFor
+        (self.Height(),
+         [devself=self.Spec(), devv=v.Spec()] DEVICE_LAMBDA (auto tid) -> void
+         {
+           // devself(tid) = devv(tid);
+           TOP()(devself(tid),devv(tid));
+         });
+
+#endif
+
+      return self.Spec();
+    }
+  };    
+
+
+
+  
+    
+  template <typename T>  
+  class Matrix<Dev<T>> : public FlatMatrix<Dev<T>>
+  { 
+    using FlatMatrix<Dev<T>>::h;
+    using FlatMatrix<Dev<T>>::w;
+    using FlatMatrix<Dev<T>>::data;
+
+  public:
+    Matrix (Matrix&) = delete;
+    Matrix (Matrix&&m2)
+      : FlatMatrix<Dev<T>>(m2.Height(), m2.Width(), m2.Data())
+    {
+      m2.data = nullptr;
+    }
+         
+    Matrix (size_t h_, size_t w_)
+      : FlatMatrix<Dev<T>>(h_, w_, Dev<T>::Malloc(h_*w_)) { ; }
+         
+    Matrix (FlatMatrix<T> mat)
+      : FlatMatrix<Dev<T>>(mat.Height(), mat.Width(),
+                           Dev<T>::Malloc(mat.Height()*mat.Width()))
+    {
+      H2D(mat);
+    }
+    
+    ~Matrix()
+    {
+      Dev<T>::Free(data);
+    }
+         
+    void D2H (FlatMatrix<T> mat) const
+    {
+      cudaMemcpy (mat.Data(), data, sizeof(T)*h*w, cudaMemcpyDeviceToHost);
+    }
+
+    void H2D (FlatMatrix<T> mat)
+    {
+      cudaMemcpy (data, mat.Data(), sizeof(T)*h*w, cudaMemcpyHostToDevice);
+    }
+
+    Matrix<T> D2H() const
+    {
+      Matrix<T> mh(h, w);
+      D2H (mh);
+      return mh;
+    }
+  };
+  
+  inline Matrix<double> D2H (SliceMatrix<Dev<double>> dmat)
+  {
+    Matrix<double> hmat(dmat.Height(), dmat.Width());
+    for (size_t i = 0; i < hmat.Height(); i++)
+      cudaMemcpy (&hmat(i,0), &dmat(i,0), sizeof(double)*hmat.Width(), cudaMemcpyDeviceToHost);
+    return hmat;
+  }
+  
+  inline Matrix<double,ColMajor> D2H (SliceMatrix<Dev<double>,ColMajor> dmat)
+  {
+    return Trans(D2H(Trans(dmat)));
+  }
+   
+  
+  
+    
+  template <ORDERING ORDA, ORDERING ORDB>  
+  void CudaMultMatMat2 (SliceMatrix<Dev<double>, ORDA> a, SliceMatrix<Dev<double>,ORDB> b, 
+                        SliceMatrix<Dev<double>, ORDERING::ColMajor> c,
+                       double alpha, double beta)
+  {
+    static Timer t("cublasDgemm");
+    CudaRegionTimer rt(t);
+    cublasSetStream(ngla::Get_CuBlas_Handle(), ngs_cuda_stream);
+    cublasStatus_t stat =
+      cublasDgemm(ngla::Get_CuBlas_Handle(), 
+                  ORDA==ORDERING::RowMajor ? CUBLAS_OP_T : CUBLAS_OP_N, 
+                  ORDB==ORDERING::RowMajor ? CUBLAS_OP_T : CUBLAS_OP_N, 
+                  c.Height(), c.Width(), a.Width(),
+                  &alpha, (double*)a.Data(), a.Dist(), (double*)b.Data(), b.Dist(),
+                  &beta, (double*)c.Data(), c.Dist());
+  }
+  
+  template <ORDERING ORDA, ORDERING ORDB>  
+  void CudaMultMatMat2 (SliceMatrix<Dev<double>, ORDA> a, SliceMatrix<Dev<double>,ORDB> b, 
+                        SliceMatrix<Dev<double>, ORDERING::RowMajor> c,
+                       double alpha, double beta)
+  {
+    CudaMultMatMat2 (Trans(b), Trans(a), Trans(c), alpha, beta);
+  }
+   
+    
+  template <typename TA, typename TB, typename TC,
+            enable_if_t<IsConvertibleToSliceMatrix<TA,Dev<double>>(),int> = 0,
+            enable_if_t<IsConvertibleToSliceMatrix<TB,Dev<double>>(),int> = 0,
+            enable_if_t<IsConvertibleToSliceMatrix<TC,Dev<double>>(),int> = 0>
+  void MultMatMat (const TA & a, const TB & b, const TC & c, double alpha=1, double beta=0)
+  {
+    CudaMultMatMat2(make_SliceMatrix(a), make_SliceMatrix(b), make_SliceMatrix(c), alpha, beta);
+  }
+ 
+
+  template <typename TOP, typename T, typename TB1, typename TB2>
+  class assign_trait<TOP, T, MultExpr<TB1,TB2>,
+                     enable_if_t<IsConvertibleToSliceMatrix<T,Dev<double>>(),int>>
+  {
+  public:
+    static INLINE T & Assign (MatExpr<T> & self, const Expr<MultExpr<TB1,TB2>> & v)
+    {
+      auto res = self.View();
+
+      double alpha = std::is_same_v<TOP,typename MatExpr<T>::AsSub> ? -1 : 1;
+      double beta = std::is_same_v<TOP,typename MatExpr<T>::As> ? 0 : 1;
+
+      MultMatMat (v.Spec().A(), v.Spec().B(), self.Spec(), alpha, beta);
+      return self.Spec();
+    }
+  };    
+    
+  template <typename TOP, typename T, typename TB1, typename TB2>
+  class assign_trait<TOP, T, ScaleExpr<MultExpr<TB1,TB2>,double>,
+                     enable_if_t<IsConvertibleToSliceMatrix<T,Dev<double>>(),int>>
+  {
+  public:
+    static inline T & Assign (MatExpr<T> & self, const Expr<ScaleExpr<MultExpr<TB1,TB2>,double>> & v)
+    {
+      auto res = self.View();
+
+      double alpha = is_same_v<TOP,typename MatExpr<T>::AsSub> ? -1 : 1;
+      double beta = is_same_v<TOP,typename MatExpr<T>::As> ? 0 : 1;
+      
+      alpha *= v.View().S();
+
+      MultMatMat (v.View().A().A(), v.View().A().B(), self.ViewRW(), alpha, beta);
+      return self.Spec();
+    }
+  };    
+    
+  template <typename TOP, typename T, typename TB1, typename TB2>
+  class assign_trait<TOP, T, MultExpr<ScaleExpr<TB1,double>,TB2>,
+                     enable_if_t<IsConvertibleToSliceMatrix<T,Dev<double>>(),int>>
+  {
+  public:
+    static inline T & Assign (MatExpr<T> & self, const Expr<MultExpr<ScaleExpr<TB1,double>,TB2>> & v)
+    {
+      auto res = self.View();
+
+      double alpha = is_same_v<TOP,typename MatExpr<T>::AsSub> ? -1 : 1;
+      double beta = is_same_v<TOP,typename MatExpr<T>::As> ? 0 : 1;
+      
+      alpha *= v.View().A().S();
+
+      MultMatMat (v.View().A().A(), v.View().B(), self.ViewRW(), alpha, beta);
+      return self.Spec();
+    }
+  };    
+    
+      
+}
+
+#endif

ngsolve-6.2.2506.post216.dev0.data/data/include/netgen/cuda_ngstd.hpp

@@ -0,0 +1,414 @@
+#ifndef CUDA_NGSTD_HPP
+#define CUDA_NGSTD_HPP
+
+#include <cuda_runtime.h>
+#include <ngstd.hpp>
+
+#include "cuda_core.hpp"
+#include "cuda_profiler.hpp"
+
+namespace ngs_cuda
+{
+  using namespace ngstd;
+
+
+  
+  extern int gpu_clock;
+  void InitCUDA (int verbose = 2);
+
+}
+
+namespace ngcore {
+  template <typename T>
+  struct IsSafe<ngs_cuda::Dev<T>> {
+    constexpr operator bool() const { return true; }
+  };
+}
+
+
+namespace ngs_cuda
+{
+
+    // TODO: Resize + error checking
+  class DevStackMemory
+  {
+    char * data;
+    char * stackptr;
+  public:
+    DevStackMemory (size_t s = 512*1024*1025)
+      {
+        cudaMalloc (&data, s);
+        stackptr = data;
+      }
+    
+    ~DevStackMemory ()
+      {
+        cudaFree (data);        
+      }
+    
+    template <typename T>
+      T * Alloc (size_t s)
+    {
+      char * tmp = stackptr;
+      s *= sizeof(T);
+      s = (s+255) & size_t(-256);
+      stackptr += s;
+      return reinterpret_cast<T*>(tmp);
+    }
+    
+    void Free (void * ptr)
+    {
+      stackptr = reinterpret_cast<char*> (ptr);
+    }
+  };
+
+  extern DevStackMemory stackmemory;
+
+  template <typename T>
+  class DevStackArray : public FlatArray<Dev<T>>
+  {
+  public:
+    DevStackArray (size_t s)
+      : FlatArray<Dev<T>> (s, (Dev<T>*)stackmemory.Alloc<T>(s))
+      { ; } 
+    ~DevStackArray ()
+      {
+        stackmemory.Free(this->data);
+      }
+    T * DevData () const { return (T*)this->data; }
+  };
+}
+
+namespace std {
+  template <typename T>  
+  struct is_integral<ngs_cuda::Dev<T>> {
+    static constexpr bool value = is_integral<T>::value;
+  };
+}
+
+
+
+namespace ngcore 
+{
+  using ngs_cuda::Dev;
+  template <typename T>  
+  class Array<Dev<T>> : public FlatArray<Dev<T>>
+  {
+  public:
+    Array() = default;
+    Array (size_t s)
+      : FlatArray<Dev<T>>(s, Dev<T>::Malloc(s)) { } ;     
+    Array (FlatArray<T> a2)
+      : Array(a2.Size())
+    {
+      this->data->H2D(a2);
+    }
+    
+    Array& operator= (Array<Dev<T>> && a2)
+    {
+      Swap (this->data, a2.data);
+      Swap (this->size, a2.size);
+      return *this;
+    }
+    
+    Array& operator= (FlatArray<T> a2)
+    {
+      SetSize(a2.Size());
+      this->data->H2D(a2);
+      return *this;
+    }
+    
+    void SetSize(size_t s)
+    {
+      if (this->Size() != s)
+        {
+          Dev<T>::Free(this->data);
+          this->data = Dev<T>::Malloc(s);
+          this->size = s;
+        }
+    }
+    
+    ~Array()
+    {
+      Dev<T>::Free(this->data);
+    }
+  };
+}
+
+
+namespace ngs_cuda
+{
+  // use Array<Dev<T>> instead 
+  template <typename T>
+  class [[deprecated]] DevArray 
+  {
+    int size;
+    T * dev_data;
+  
+  public: 
+    DevArray (int asize)
+    {
+      size = asize;
+      cudaMalloc((T**)&dev_data, size*sizeof(T));
+    }
+
+    DevArray (FlatArray<T> a2)
+    {
+      size = a2.Size();
+      cudaMalloc((T**)&dev_data, size*sizeof(T));
+      cudaMemcpy (dev_data, &a2[0], sizeof(T)*size, cudaMemcpyHostToDevice);
+    }
+
+    ~DevArray ()
+    {
+      cudaFree (dev_data);
+    }
+
+    T * DevPtr() { return dev_data; }
+
+    DevArray & operator= (FlatArray<T> a2)
+    {
+      cudaMemcpy (dev_data, &a2[0], sizeof(T)*size, cudaMemcpyHostToDevice);
+      return *this;
+    }
+
+    void D2H (FlatArray<T> a2) const
+    {
+      cudaMemcpy (&a2[0], dev_data, sizeof(T)*size, cudaMemcpyDeviceToHost);    
+    }
+
+    INLINE int Size() const { return size; }
+
+    /*
+    INLINE operator FlatArray<T> ()
+    {
+      return FlatArray<T> (size, dev_data);
+    }
+    */
+    INLINE FlatArray<T> Dev() const
+    {
+      return FlatArray<T> (size, dev_data);
+    }
+
+    explicit INLINE operator Array<T> () const
+    {
+      Array<T> temp(size);
+#ifdef __CUDA_ARCH__
+      temp = FlatArray<T> (*this);
+#else
+      D2H (temp);
+#endif
+      return temp;
+    }
+
+    INLINE Array<T> Host() const
+    {
+      return Array<T> (*this);
+    }
+    
+    T * DevData() const { return dev_data; }
+  }; 
+
+
+
+  template <typename T>
+  inline Array<T> D2H (FlatArray<Dev<T>> deva)
+  {
+    Array<T> hosta(deva.Size());
+    cudaMemcpy (hosta.Data(), deva.Data(), sizeof(T)*hosta.Size(), cudaMemcpyDeviceToHost);    
+    return hosta;
+  }
+
+  template <typename T>
+  inline void H2D (FlatArray<Dev<T>> deva, FlatArray<T> hosta)
+  {
+    cudaMemcpy (deva.Data(), hosta.Data(), sizeof(T)*hosta.Size(), cudaMemcpyHostToDevice);    
+  }
+
+  /*
+    template <class T>
+    class TableWrapper : public Table<T>
+    {
+    using Table<T>::size;
+    using Table<T>::data;
+    using Table<T>::index;
+    public:
+    INLINE TableWrapper (int asize, int * aindex, T * adata)
+    // : Table<T> (0,0)
+    { 
+    size = asize;
+    index = aindex;
+    data = adata;
+    }
+
+    INLINE TableWrapper (const Table<T> & tab)
+    // : Table<T> (0,0) 
+    {
+    const TableWrapper<T> & htab = static_cast<const TableWrapper<T>&> (tab);
+    size = htab.size;
+    data = htab.data;
+    index = htab.index;
+    }
+    INLINE ~TableWrapper ()
+    {
+    data = NULL;
+    index = NULL;
+    }
+
+    INLINE int SizeData() { return index[size]; }
+    INLINE int* & Index() { return index; }
+    INLINE T* & Data() { return data; }
+
+    // HD const int * & Index() const { return index; }
+    // HD const T * & Data() const { return data; }
+    };
+  */
+
+
+
+  // only data at device, but index at host
+  template <typename T>
+  class DevDataTable
+  {
+    int size;
+    size_t * index = nullptr;
+    Dev<T> * dev_data = nullptr;
+  
+  public: 
+
+    DevDataTable (FlatTable<T> t2)
+    {
+      size = t2.Size();
+      if (size == 0) return;
+
+      index = new size_t[size+1];
+      for (int i = 0; i <= size; i++)
+        index[i] = t2.IndexArray()[i];
+
+      int sizedata = t2.AsArray().Size();
+      dev_data = Dev<T>::Malloc(sizedata);
+      cudaMemcpy (dev_data, t2.Data(), sizeof(T)*sizedata, cudaMemcpyHostToDevice);
+    }
+
+    ~DevDataTable ()
+    {
+      Dev<T>::Free (dev_data);
+      delete [] index;
+    }
+
+    void D2H (FlatTable<T> & t2) const
+    {
+      int sizedata = t2.AsArray().Size();
+      cudaMemcpy (&t2[0][0], dev_data, sizeof(T)*sizedata, cudaMemcpyDeviceToHost);    
+    }
+
+    operator FlatTable<Dev<T>> () const
+    {
+      return FlatTable<Dev<T>> (size, index, dev_data);
+    }
+
+    auto Index() const { return index; }
+    auto DevData() const { return dev_data; }
+
+    FlatArray<Dev<T>> Row(int i) const { return { index[i+1]-index[i], dev_data+index[i] }; }
+    
+    class Iterator
+    {
+      const DevDataTable & tab;
+      size_t row;
+    public:
+      Iterator (const DevDataTable & _tab, size_t _row) : tab(_tab), row(_row) { ; }
+      Iterator & operator++ () { ++row; return *this; }
+      auto operator* () const { return tab.Row(row); }
+      bool operator!= (const Iterator & it2) { return row != it2.row; }
+    };
+
+    Iterator begin() const { return Iterator(*this, 0); }
+    Iterator end() const { return Iterator(*this, size); }
+  }; 
+
+
+  template <typename T>
+  class DevTable
+  {
+    int size;
+    Dev<size_t> * dev_index = nullptr;
+    Dev<T> * dev_data = nullptr;
+  
+  public: 
+
+    DevTable (FlatTable<T> t2)
+    {
+      size = t2.Size();
+      if (size == 0) return;
+      
+      cudaMalloc((size_t**)&dev_index, (size+1)*sizeof(size_t));
+      cudaMemcpy (dev_index, &t2.IndexArray()[0], sizeof(size_t)*(size+1), cudaMemcpyHostToDevice); 
+      // cout << "res = " << cudaMemcpy (dev_index, t2.Index(), sizeof(int)*(size+1), cudaMemcpyHostToDevice) << endl;
+    
+      int sizedata = t2.AsArray().Size();
+      cudaMalloc((int**)&dev_data, sizedata*sizeof(T));
+      cudaMemcpy (dev_data, t2.Data(), sizeof(T)*sizedata, cudaMemcpyHostToDevice);
+    }
+
+    ~DevTable ()
+    {
+      cudaFree (dev_data);
+      cudaFree (dev_index);
+    }
+
+    void D2H (FlatTable<T> & t2) const
+    {
+      int sizedata = t2.AsArray().Size();
+      cudaMemcpy (&t2[0][0], dev_data, sizeof(T)*sizedata, cudaMemcpyDeviceToHost);    
+    }
+
+    operator FlatTable<T> () const
+    {
+      return FlatTable<T> (size, (size_t*)dev_index, (T*)dev_data);
+    }
+
+    size_t * DevIndex() const { return (size_t*)dev_index; }
+    T * DevData() const { return (T*)dev_data; }
+
+    FlatArray<Dev<T>> AsArray() const
+    {
+      return FlatArray<Dev<T>> ( dev_index[size].D2H(), dev_data );
+    }    
+  }; 
+
+
+
+
+
+
+  class DevBitArray
+  {
+  protected:
+    size_t size = 0;
+    unsigned char * dev_data = nullptr;
+
+  public:
+    DevBitArray (size_t asize);
+    DevBitArray (const ngcore::BitArray & ba);
+
+    ~DevBitArray ();
+
+    DevBitArray & operator= (const ngcore::BitArray &ba);
+
+    size_t Size () const { return size; }
+    auto Data () const { return dev_data; }
+
+    void SetSize (size_t asize);
+
+  private:
+    size_t Addr (size_t i) const
+    {
+      return (i / CHAR_BIT);
+    }
+  };
+
+}
+
+
+#endif