ngsolve 6.2.2506.post75.dev0__cp314-cp314-manylinux_2_28_x86_64.whl

ngsolve-6.2.2506.post75.dev0.data/data/include/netgen/paralleldofs.hpp

@@ -0,0 +1,340 @@
+#ifndef FILE_PARALLELDOFS
+#define FILE_PARALLELDOFS
+
+/**************************************************************************/
+/* File:   paralleldofs.hpp                                               */
+/* Author: Joachim Schoeberl                                              */
+/* Date:   July 2011                                                      */
+/**************************************************************************/
+
+
+#include <core/mpi_wrapper.hpp>
+#include <ngstd.hpp>
+
+namespace ngla
+{
+  using namespace ngstd;
+
+  
+#ifdef PARALLEL
+
+  /**
+     Handles the distribution of degrees of freedom for vectors and matrices
+   */
+  class ParallelDofs
+  {
+  protected:
+    /// the communicator 
+    NgMPI_Comm comm;
+    
+    /// local ndof
+    size_t ndof;
+
+    /// global ndof
+    size_t global_ndof;
+
+    /// proc 2 dofs
+    Table<int> exchangedofs;
+    
+    /// dof 2 procs
+    Table<int> dist_procs;
+
+    /// all procs with connected dofs
+    Array<int> all_dist_procs;
+    
+    /// mpi-datatype to send exchange dofs
+    Array<NG_MPI_Datatype> mpi_t;
+    
+    /// am I the master process ?
+    BitArray ismasterdof;
+
+    /// entry-size
+    int es;
+    bool complex;
+    
+  public:
+    /**
+       setup parallel dofs.
+       Table adist_procs must provide the distant processes for each dof.
+       table 
+     */
+    ParallelDofs (NgMPI_Comm acomm, Table<int> && adist_procs, 
+		  int dim = 1, bool iscomplex = false);
+
+    shared_ptr<ParallelDofs> SubSet (shared_ptr<BitArray> take_dofs) const;
+
+    shared_ptr<ParallelDofs> Range (IntRange range) const;
+      
+    virtual ~ParallelDofs();
+
+    int GetNTasks() const { return exchangedofs.Size(); }
+
+    FlatArray<int> GetExchangeDofs (int proc) const
+    { return exchangedofs[proc]; }
+
+    FlatArray<int> GetDistantProcs (int dof) const
+    { return dist_procs[dof]; }
+
+    FlatArray<int> GetDistantProcs () const
+    { return all_dist_procs; }
+
+    bool IsMasterDof (size_t localdof) const
+    { return ismasterdof.Test(localdof); }
+    
+    const auto & MasterDofs () const
+    { return ismasterdof; }
+    
+    size_t GetNDofLocal () const { return ndof; }
+
+    size_t GetNDofGlobal () const { return global_ndof; }
+
+    bool IsExchangeProc (int proc) const
+    { return exchangedofs[proc].Size() != 0; }
+
+    NG_MPI_Datatype GetMPI_Type (int dest) const
+    { return mpi_t[dest]; }
+
+    const NgMPI_Comm & GetCommunicator () const { return comm; }
+
+    int GetEntrySize () const { return es; }
+    bool IsComplex () const { return complex; }
+    
+    int GetMasterProc (int dof) const
+    {
+      auto dps = GetDistantProcs(dof);
+      return (dps.Size() > 0) ? min2(comm.Rank(), dps[0]) : comm.Rank();
+    }
+
+    void EnumerateGlobally (shared_ptr<BitArray> freedofs, Array<int> & globnum, int & num_glob_dofs) const;
+
+
+    template <typename T>
+    void ReduceDofData (FlatArray<T> data, NG_MPI_Op op) const;
+
+    template <typename T>
+    void ScatterDofData (FlatArray<T> data) const;
+    
+    template <typename T>
+    void AllReduceDofData (FlatArray<T> data, NG_MPI_Op op) const
+    {
+      // if (this == NULL)  // illformed C++, shall get rid of this
+        // throw Exception("AllReduceDofData for null-object");
+      ReduceDofData (data, op);
+      ScatterDofData (data);
+    }
+
+  };
+
+#else
+
+  class ParallelDofs 
+  {
+  protected:
+    int ndof;
+    int es;
+    bool complex;
+    BitArray masterdofs;
+  public:
+    ParallelDofs (NgMPI_Comm acomm, Table<int> && adist_procs, 
+		  int dim = 1, bool iscomplex = false)
+      : ndof(adist_procs.Size()), es(dim), complex(iscomplex)
+    { ; }
+    
+    int GetNDofLocal () const { return ndof; }
+    int GetNDofGlobal () const { return ndof; }
+    
+    int GetNTasks() const { return 1; }
+    NgMPI_Comm GetCommunicator () const { return NgMPI_Comm(NG_MPI_COMM_WORLD); }
+
+    shared_ptr<ParallelDofs> Range (IntRange range) const
+    { return nullptr; }
+    
+    FlatArray<int> GetExchangeDofs (int proc) const
+    { return FlatArray<int> (0, nullptr); }
+
+    FlatArray<int> GetDistantProcs (int dof) const
+    { return FlatArray<int> (0, nullptr); }
+
+    FlatArray<int> GetDistantProcs () const
+    { return FlatArray<int> (0, nullptr); }      
+
+    bool IsMasterDof (size_t localdof) const
+    { return true; }
+
+    const BitArray & MasterDofs () const;
+    void EnumerateGlobally (shared_ptr<BitArray> freedofs, Array<int> & globnum, int & num_glob_dofs) const;
+    
+    template <typename T>
+    void ReduceDofData (FlatArray<T> data, NG_MPI_Op op) const { ; }
+
+    template <typename T>
+    void ScatterDofData (FlatArray<T> data) const { ; } 
+    
+    template <typename T>
+    void AllReduceDofData (FlatArray<T> data, NG_MPI_Op op) const { ; }
+
+    int GetEntrySize () const { return es; }
+    bool IsComplex () const { return complex; }
+  };
+  
+#endif
+  
+
+  
+  template <typename T>
+  [[deprecated("use pardofs.ReduceDofData")]]
+  void ReduceDofData (FlatArray<T> data, NG_MPI_Op op, const shared_ptr<ParallelDofs> & pardofs)
+  {
+    if (pardofs)
+      pardofs->ReduceDofData(data, op);
+  }
+  
+  template <typename T>
+  [[deprecated("use pardofs.ScatterDofData")]]   
+  void ScatterDofData (FlatArray<T> data, const shared_ptr<ParallelDofs> & pardofs)
+  {
+    if (pardofs)
+      pardofs->ScatterDofData (data);
+  }
+  
+  template <typename T>
+  [[deprecated("use pardofs.AllReduceDofData")]]
+  void AllReduceDofData (FlatArray<T> data, NG_MPI_Op op, 
+			 const shared_ptr<ParallelDofs> & pardofs)
+  {
+    if (pardofs)
+      pardofs->AllReduceDofData (data, op);
+  }
+  
+
+
+#ifdef PARALLEL
+
+  template <typename T>
+  void ParallelDofs :: ReduceDofData (FlatArray<T> data, NG_MPI_Op op) const
+  {
+    static Timer t0("ParallelDofs :: ReduceDofData");
+    RegionTimer rt(t0);
+
+    auto comm = GetCommunicator();
+    int ntasks = comm.Size();
+    int rank = comm.Rank();
+    if (ntasks <= 1) return;
+
+
+    Array<int> nsend(ntasks), nrecv(ntasks);
+    nsend = 0;
+    nrecv = 0;
+
+    /** Count send/recv size **/
+    for (int i = 0; i < GetNDofLocal(); i++)
+      if (auto dps = GetDistantProcs(i); dps.Size())
+        {
+          if (rank < dps[0])
+            for (auto p : dps)
+              nrecv[p]++;
+          else
+            nsend[dps[0]]++;
+        }
+    
+    Table<T> send_data(nsend);
+    Table<T> recv_data(nrecv);
+
+    /** Fill send_data **/
+    nsend = 0;
+    for (int i = 0; i < GetNDofLocal(); i++)
+      if (auto dps = GetDistantProcs(i); dps.Size())
+        if (rank > dps[0])
+          send_data[dps[0]][nsend[dps[0]]++] = data[i];
+
+    NgMPI_Requests send_requests; 
+    NgMPI_Requests recv_requests; 
+    for (int i = 0; i < ntasks; i++)
+      {
+	if (nsend[i])
+	  send_requests += comm.ISend(send_data[i], i, NG_MPI_TAG_SOLVE);
+	if (nrecv[i])
+	  recv_requests += comm.IRecv(recv_data[i], i, NG_MPI_TAG_SOLVE);
+      }
+
+    Array<int> cnt(ntasks);
+    cnt = 0;
+    
+    NG_MPI_Datatype type = GetMPIType<T>();
+
+    recv_requests.WaitAll();
+    for (int i = 0; i < GetNDofLocal(); i++)
+      if (IsMasterDof(i))
+        for (auto p : GetDistantProcs (i))
+          NG_MPI_Reduce_local (&recv_data[p][cnt[p]++], &data[i], 1, type, op);
+
+    send_requests.WaitAll();    
+  }    
+
+
+
+
+
+  template <typename T>
+  void ParallelDofs :: ScatterDofData (FlatArray<T> data) const
+  {
+    static Timer t0("ParallelDofs :: ScatterDofData");
+    RegionTimer rt(t0);
+
+    NgMPI_Comm comm = GetCommunicator();
+    int ntasks = comm.Size();
+    int rank = comm.Rank();
+    if (ntasks <= 1) return;
+
+    Array<int> nsend(ntasks), nrecv(ntasks);
+    nsend = 0;
+    nrecv = 0;
+
+    /** Count send/recv size **/
+    for (int i = 0; i < GetNDofLocal(); i++) 
+      if (auto dps = GetDistantProcs(i); dps.Size() > 0)
+        {
+          if (rank < dps[0])
+            for (auto p : dps)
+              nsend[p]++;
+          else
+            nrecv[dps[0]]++;
+        }
+    
+    Table<T> send_data(nsend);
+    Table<T> recv_data(nrecv);
+
+    /** Fill send_data **/
+    nsend = 0;
+    for (int i = 0; i < GetNDofLocal(); i++) 
+      if (auto dps = GetDistantProcs(i); dps.Size() > 0)
+        if (rank < dps[0])
+          for (auto p : dps)
+            send_data[p][nsend[p]++] = data[i];
+    
+    NgMPI_Requests requests;
+    for (int i = 0; i < ntasks; i++)
+      {
+	if (nsend[i])
+	  requests += comm.ISend (send_data[i], i, NG_MPI_TAG_SOLVE);
+	if (nrecv[i])
+	  requests += comm.IRecv (recv_data[i], i, NG_MPI_TAG_SOLVE);
+      }
+    requests.WaitAll();
+
+    Array<int> cnt(ntasks);
+    cnt = 0;
+
+    for (int i = 0; i < GetNDofLocal(); i++)
+      if (!IsMasterDof(i))
+	{
+	  int master = GetDistantProcs (i)[0];
+	  data[i] = recv_data[master][cnt[master]++];
+	}
+  }    
+
+#endif //PARALLEL
+
+}
+
+#endif

ngsolve-6.2.2506.post75.dev0.data/data/include/netgen/parallelngs.hpp

@@ -0,0 +1,23 @@
+#ifndef FILE_PARALLELNGS
+#define FILE_PARALLELNGS
+
+/*
+#include <ngstd.hpp>
+#include <la.hpp>
+
+namespace ngparallel
+{
+  using namespace ngstd;
+  using namespace ngla;
+}
+*/
+
+#include "parallelvector.hpp"
+#include "parallel_matrices.hpp"
+
+
+#endif
+
+
+
+

ngsolve-6.2.2506.post75.dev0.data/data/include/netgen/parallelvector.hpp

@@ -0,0 +1,269 @@
+#ifndef FILE_NGS_PARALLELVECTOR
+#define FILE_NGS_PARALLELVECTOR
+
+/* ************************************************************************/
+/* File:   parallelvector.hpp                                             */
+/* Author: Astrid Sinwel, Joachim Schoeberl                               */
+/* Date:   2007,2011                                                      */
+/* ************************************************************************/
+
+#include <vvector.hpp>
+#include <multivector.hpp>
+#include <paralleldofs.hpp>
+
+namespace ngla
+{
+  
+  class NGS_DLL_HEADER ParallelBaseVector : virtual public BaseVector
+  {
+  protected:
+    mutable PARALLEL_STATUS status;
+    shared_ptr<ParallelDofs> paralleldofs;    
+    shared_ptr<BaseVector> local_vec;
+    
+    mutable NgMPI_Requests sreqs;
+    mutable NgMPI_Requests rreqs;
+
+  public:
+    ParallelBaseVector ()
+    { ; }
+
+    template <typename T> 
+    BaseVector & operator= (const VVecExpr<T> & v)
+    {
+      v.AssignTo (1.0, *this);
+      return *this;
+    }
+
+    virtual PARALLEL_STATUS Status () const { return status; }
+
+    virtual void SetStatus ( PARALLEL_STATUS astatus ) const  
+    {
+      status = astatus;
+    }
+
+    virtual PARALLEL_STATUS GetParallelStatus () const override { return Status(); }
+    virtual void SetParallelStatus (PARALLEL_STATUS stat) const override { SetStatus (stat); }
+    virtual optional<NgMPI_Comm> GetCommunicator() const override
+    {
+      if (paralleldofs)
+        return paralleldofs->GetCommunicator();
+      else
+        return nullopt;
+    }
+
+
+    virtual shared_ptr<ParallelDofs> GetParallelDofs () const
+    {
+      return paralleldofs; 
+    }
+    
+    virtual bool IsParallelVector () const
+    {
+      return (this->Status() != NOT_PARALLEL);
+    }
+    
+    virtual BaseVector & SetScalar (double scal) override;
+    virtual BaseVector & SetScalar (Complex scal) override;
+    virtual void SetZero () override;
+    
+    virtual BaseVector & Set (double scal, const BaseVector & v) override;
+    virtual BaseVector & Set (Complex scal, const BaseVector & v) override;
+
+    virtual BaseVector & Add (double scal, const BaseVector & v) override;
+    virtual BaseVector & Add (Complex scal, const BaseVector & v) override;
+
+    void PrintStatus ( ostream & ost ) const;
+
+    virtual shared_ptr<BaseVector> GetLocalVector () const override
+    { return local_vec; }
+    
+    virtual void Cumulate () const override; 
+    
+    virtual void Distribute() const override = 0;
+    // { cerr << "ERROR -- Distribute called for BaseVector, is not parallel" << endl; }
+    
+    virtual NgMPI_Request ISend ( int dest ) const;
+    // virtual void Send ( int dest ) const;
+    
+    virtual NgMPI_Request IRecvVec ( int dest ) = 0;
+    // { cerr << "ERROR -- IRecvVec called for BaseVector, is not parallel" << endl; }
+
+    // virtual void RecvVec ( int dest )
+    // { cerr << "ERROR -- IRecvVec called for BaseVector, is not parallel" << endl; }
+    
+    virtual void AddRecvValues( int sender ) = 0;
+    // { cerr << "ERROR -- AddRecvValues called for BaseVector, is not parallel" << endl; }
+
+    virtual void SetParallelDofs (shared_ptr<ParallelDofs> aparalleldofs) = 0;
+    // const Array<int> * procs = 0) = 0;
+  };
+
+
+
+
+  inline ParallelBaseVector * dynamic_cast_ParallelBaseVector (BaseVector * x)
+  {
+    // cout << "my dynamic * cast" << endl;
+    AutoVector * ax = dynamic_cast<AutoVector*> (x);
+    if (ax)
+      return dynamic_cast<ParallelBaseVector*> (&**ax);
+    return dynamic_cast<ParallelBaseVector*> (x);
+  }
+
+  inline const ParallelBaseVector * dynamic_cast_ParallelBaseVector (const BaseVector * x)
+  {
+    // cout << "my dynamic const * cast" << endl;
+    const AutoVector * ax = dynamic_cast<const AutoVector*> (x);
+    if (ax)
+      { 
+        // cout << "is an autovector" << endl; 
+        return dynamic_cast<const ParallelBaseVector*> (&**ax);
+      }
+    return dynamic_cast<const ParallelBaseVector*> (x);
+  }
+
+  inline ParallelBaseVector & dynamic_cast_ParallelBaseVector (BaseVector & x)
+  {
+    // cout << "my dynamic cast" << endl;
+    AutoVector * ax = dynamic_cast<AutoVector*> (&x);
+    if (ax)
+      return dynamic_cast<ParallelBaseVector&> (**ax);
+    return dynamic_cast<ParallelBaseVector&> (x);
+  }
+
+  inline const ParallelBaseVector & dynamic_cast_ParallelBaseVector (const BaseVector & x)
+  {
+    // cout << "my dynamic cast" << endl;
+    const AutoVector * ax = dynamic_cast<const AutoVector*> (&x);
+    if (ax)
+      return dynamic_cast<const ParallelBaseVector&> (**ax);
+    return dynamic_cast<const ParallelBaseVector&> (x);
+  }
+
+
+
+
+
+
+  
+  template <class SCAL>
+  class NGS_DLL_HEADER S_ParallelBaseVector 
+    : virtual public S_BaseVector<SCAL>, 
+      virtual public ParallelBaseVector
+  {
+  protected:
+    virtual SCAL InnerProduct (const BaseVector & v2, bool conjugate = false) const;
+    virtual BaseVector & SetScalar (double scal)
+    { return ParallelBaseVector::SetScalar(scal); }
+  };
+
+
+  template <class SCAL>
+  class NGS_DLL_HEADER S_ParallelBaseVectorPtr
+    : virtual public S_BaseVectorPtr<SCAL>, 
+      virtual public S_ParallelBaseVector<SCAL>
+  {
+  protected:
+    typedef SCAL TSCAL;
+    using ParallelBaseVector :: status;
+    using ParallelBaseVector :: paralleldofs;
+
+    using ParallelBaseVector :: sreqs;
+    using ParallelBaseVector :: rreqs;
+
+    Table<SCAL> recvvalues;
+
+    using S_BaseVectorPtr<TSCAL> :: pdata;
+    using ParallelBaseVector :: local_vec;
+
+  public:
+    S_ParallelBaseVectorPtr (int as, int aes, void * adata, shared_ptr<ParallelDofs> apd, PARALLEL_STATUS stat) throw();
+    S_ParallelBaseVectorPtr (int as, int aes, shared_ptr<ParallelDofs> apd, PARALLEL_STATUS stat) throw();
+
+    virtual ~S_ParallelBaseVectorPtr ();
+    virtual void SetParallelDofs (shared_ptr<ParallelDofs> aparalleldofs) override; 
+
+    virtual void Distribute() const override;
+    virtual ostream & Print (ostream & ost) const override;
+    virtual AutoVector Range (T_Range<size_t> range) const override;
+    virtual AutoVector Range (DofRange range) const override;
+    
+    virtual NgMPI_Request IRecvVec ( int dest ) override;
+    // virtual void  RecvVec ( int dest );
+    virtual void AddRecvValues( int sender ) override;
+    virtual AutoVector CreateVector () const override;
+    virtual unique_ptr<MultiVector> CreateMultiVector (size_t cnt) const override;
+    
+    virtual double L2Norm () const override;
+  };
+ 
+
+
+
+  template <typename T = double>
+  class ParallelVVector : // public VVector<T>, 
+			  public S_ParallelBaseVectorPtr<typename mat_traits<T>::TSCAL>
+  {
+    typedef typename mat_traits<T>::TSCAL TSCAL;
+    enum { ES = sizeof(T) / sizeof(TSCAL) };
+
+    using S_BaseVectorPtr<TSCAL> :: pdata;
+    using ParallelBaseVector :: local_vec;
+
+  public:
+    [[deprecated("too much info, use ParallelVVector(pardofs) instead")]]
+    explicit ParallelVVector (int as, shared_ptr<ParallelDofs> aparalleldofs,
+			      PARALLEL_STATUS astatus = CUMULATED)
+    : S_BaseVectorPtr<TSCAL> (as, ES), // VVector<T> (as), 
+	S_ParallelBaseVectorPtr<TSCAL> (as, ES, aparalleldofs, astatus)
+    { local_vec = make_shared<VFlatVector<T>>(as, (T*)pdata); }
+
+    explicit ParallelVVector (shared_ptr<ParallelDofs> aparalleldofs,
+			      PARALLEL_STATUS astatus = CUMULATED)
+      : S_BaseVectorPtr<TSCAL> (aparalleldofs->GetNDofLocal(), ES), 
+      // VVector<T> (aparalleldofs->GetNDofLocal()), 
+	S_ParallelBaseVectorPtr<TSCAL> (aparalleldofs->GetNDofLocal(), ES, aparalleldofs, astatus)
+    { local_vec = make_shared<VFlatVector<T>>(aparalleldofs->GetNDofLocal(), (T*)pdata); }
+
+
+    virtual ~ParallelVVector() throw()
+    { ; }
+  };
+
+
+  template <typename T = double>
+  class ParallelVFlatVector : public VFlatVector<T>,
+			      public S_ParallelBaseVectorPtr<typename mat_traits<T>::TSCAL>
+  {
+    typedef typename mat_traits<T>::TSCAL TSCAL;
+    enum { ES = sizeof(T) / sizeof(TSCAL) };
+
+    using S_BaseVectorPtr<TSCAL> :: pdata;
+    using ParallelBaseVector :: local_vec;
+
+  public:
+    explicit ParallelVFlatVector (int as, T * adata, 
+				  shared_ptr<ParallelDofs> aparalleldofs, 
+				  PARALLEL_STATUS astatus)
+    : S_BaseVectorPtr<TSCAL> (as, ES, adata),
+      VFlatVector<T> (as, adata),
+      S_ParallelBaseVectorPtr<TSCAL> (as, ES, aparalleldofs, astatus)
+    { local_vec = make_shared<VFlatVector<T>>(aparalleldofs->GetNDofLocal(), (T*)pdata); }
+
+    explicit ParallelVFlatVector ()
+    : S_BaseVectorPtr<TSCAL> (0, ES, NULL),
+      S_ParallelBaseVectorPtr<TSCAL> (0, ES, NULL)
+    { local_vec = make_shared<VFlatVector<T>>(0, NULL); }
+      
+    virtual ~ParallelVFlatVector() throw()
+    { ; }
+  };
+
+
+
+  extern AutoVector CreateParallelVector (shared_ptr<ParallelDofs> pardofs, PARALLEL_STATUS status);
+  
+}
+
+#endif