ngsolve 6.2.2506.post33.dev0__cp310-cp310-macosx_10_15_universal2.whl → 6.2.2506.post45.dev0__cp310-cp310-macosx_10_15_universal2.whl

netgen/include/mptools.hpp

@@ -25,6 +25,7 @@ namespace ngsbem
 
   constexpr int FMM_SW = 4;
 
+  
 
   // ************************ SIMD - creation (should end up in simd.hpp) ************* 
 
@@ -42,7 +43,19 @@ namespace ngsbem
   }
   
 
-
+  class NGS_DLL_HEADER PrecomputedSqrts
+  {
+  public:
+    Array<double> sqrt_int;
+    // Array<double> inv_sqrt_int;
+    Array<double> sqrt_n_np1;    // sqrt(n*(n+1))
+    Array<double> inv_sqrt_2np1_2np3;  // 1/sqrt( (2n+1)*(2n+3) )
+    
+    PrecomputedSqrts();
+  };
+  
+  extern NGS_DLL_HEADER PrecomputedSqrts presqrt;
+  
 
 
   
@@ -189,18 +202,26 @@ namespace ngsbem
     void RotateY (double alpha, bool parallel = false);
 
     
+    
     static double CalcAmn (int m, int n)
     {
       if (m < 0) m=-m;
       if (n < m) return 0;
-      return sqrt( (n+1.0+m)*(n+1.0-m) / ( (2*n+1)*(2*n+3) ));
+
+      if (2*n+1 < presqrt.sqrt_int.Size())
+        return presqrt.sqrt_int[n+1+m]*presqrt.sqrt_int[n+1-m] * presqrt.inv_sqrt_2np1_2np3[n];
+      else
+        return sqrt( (n+1.0+m)*(n+1.0-m) / ( (2*n+1)*(2*n+3) ));
     }
   
     static double CalcBmn (int m, int n)
     {
       double sgn = (m >= 0) ? 1 : -1;
-      if ( (m > n) || (-m > n) ) return 0;
-      return sgn * sqrt( (n-m-1.0)*(n-m) / ( (2*n-1.0)*(2*n+1))); 
+      if ( (m >= n) || (-m > n) ) return 0;
+      if (n <= presqrt.inv_sqrt_2np1_2np3.Size())
+        return sgn * presqrt.sqrt_n_np1[n-m-1] * presqrt.inv_sqrt_2np1_2np3[n-1];
+      else
+        return sgn * sqrt( (n-m-1.0)*(n-m) / ( (2*n-1.0)*(2*n+1)));
     }
   
     static double CalcDmn (int m, int n)
@@ -543,7 +564,7 @@ namespace ngsbem
     };
 
 
-    static void ProcessBatch(FlatArray<RecordingSS*> batch, double len, double theta) {
+    static void ProcessBatchSS(FlatArray<RecordingSS*> batch, double len, double theta) {
       constexpr int vec_length = VecLength<entry_type>;
       int batch_size = batch.Size();
       int N = batch_size * vec_length;
@@ -555,42 +576,45 @@ namespace ngsbem
         }
       }
       else if (N <= 3) {
-        ProcessVectorizedBatch<3, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchSS<3, vec_length>(batch, len, theta);
       }
       else if (N <= 4) {
-        ProcessVectorizedBatch<4, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchSS<4, vec_length>(batch, len, theta);
       }
       else if (N <= 6) {
-        ProcessVectorizedBatch<6, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchSS<6, vec_length>(batch, len, theta);
       }
       else if (N <= 12) {
-        ProcessVectorizedBatch<12, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchSS<12, vec_length>(batch, len, theta);
       }
       else if (N <= 24) {
-        ProcessVectorizedBatch<24, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchSS<24, vec_length>(batch, len, theta);
       }
       else if (N <= 48) {
-        ProcessVectorizedBatch<48, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchSS<48, vec_length>(batch, len, theta);
       }
       else if (N <= 96) {
-        ProcessVectorizedBatch<96, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchSS<96, vec_length>(batch, len, theta);
       }
       else if (N <= 192) {
-        ProcessVectorizedBatch<192, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchSS<192, vec_length>(batch, len, theta);
       }
       else {
         // Split large batches
-        ProcessBatch(batch.Range(0, 192 / vec_length), len, theta);
-        ProcessBatch(batch.Range(192 / vec_length, batch_size), len, theta);
+        ProcessBatchSS(batch.Range(0, 192 / vec_length), len, theta);
+        ProcessBatchSS(batch.Range(192 / vec_length, batch_size), len, theta);
       }
     }
 
     template<int N, int vec_length>
-    static void ProcessVectorizedBatch(FlatArray<RecordingSS*> batch, double len, double theta) {
+    static void ProcessVectorizedBatchSS(FlatArray<RecordingSS*> batch, double len, double theta) {
 
       // *testout << "Processing vectorized S->S batch of size " << batch.Size() << ", with N = " << N << ", vec_length = " << vec_length << ", len = " << len << ", theta = " << theta << endl;
-      SphericalExpansion<Singular, Vec<N,Complex>> vec_source(batch[0]->mp_source->Order(), batch[0]->mp_source->Kappa(), batch[0]->mp_source->RTyp());
-      SphericalExpansion<Singular, Vec<N,Complex>> vec_target(batch[0]->mp_target->Order(), batch[0]->mp_target->Kappa(), batch[0]->mp_target->RTyp());
+      double kappa = batch[0]->mp_source->Kappa();
+      int so = batch[0]->mp_source->Order();
+      int to = batch[0]->mp_target->Order();
+      SphericalExpansion<Singular, Vec<N,Complex>> vec_source(so, kappa, batch[0]->mp_source->RTyp());
+      SphericalExpansion<Singular, Vec<N,Complex>> vec_target(to, kappa, batch[0]->mp_target->RTyp());
 
       // Copy multipoles into vectorized multipole
       for (int i = 0; i < batch.Size(); i++)
@@ -688,10 +712,10 @@ namespace ngsbem
         for (auto [sp,ep,j,num] : currents)
           AddCurrent (sp,ep,j,num);
         
-        charges.SetSize0();
-        dipoles.SetSize0();
-        chargedipoles.SetSize0();        
-        currents.SetSize0();
+        charges.DeleteAll();
+        dipoles.DeleteAll();
+        chargedipoles.DeleteAll();        
+        currents.DeleteAll();
       }
 
       
@@ -718,7 +742,7 @@ namespace ngsbem
 
         // if (r*mp.Kappa() < 1e-8) return;
         if (level > 20) return;
-        if (charges.Size() < maxdirect && r*mp.Kappa() < 1)
+        if (charges.Size() < maxdirect && r*mp.Kappa() < 5)
           return;
         
         SendSourcesToChilds();
@@ -746,8 +770,9 @@ namespace ngsbem
           }
         
         dipoles.Append (tuple{x,d,c});
-
-        if (dipoles.Size() < maxdirect || r < 1e-8)
+        
+        if (level > 20) return;
+        if (dipoles.Size() < maxdirect)
           return;
 
         SendSourcesToChilds();
@@ -797,7 +822,7 @@ namespace ngsbem
             Array<double> split;
             split.Append(0);
             for (int i = 0; i < 3; i++)
-              if (sp(i) < center(i) != ep(i) < center(i))
+              if ((sp(i) < center(i)) != (ep(i) < center(i)))
                 split += (center(i)-sp(i)) / (ep(i)-sp(i));  // segment cuts i-th coordinate plane
             split.Append(1);
             BubbleSort(split);
@@ -859,26 +884,30 @@ namespace ngsbem
             return sum;
           }
 
-        {
-          // static Timer t("fmm direct eval"); RegionTimer reg(t);
-          // t.AddFlops (charges.Size());
         if (simd_charges.Size())
           {
             // static Timer t("mptool singmp, evaluate, simd charges"); RegionTimer r(t);
+            // t.AddFlops (charges.Size());
             
             simd_entry_type vsum{0.0};
             if (mp.Kappa() < 1e-12)
               {
                 for (auto [x,c] : simd_charges)
                   {
+                    auto rho = L2Norm(p-x);
+                    auto kernel = 1/(4*M_PI)/rho;
+                    kernel = If(rho > 0.0, kernel, SIMD<double,FMM_SW>(0.0));
+                    vsum += kernel * c;
+
+                    /*
                     auto rho2 = L2Norm2(p-x);
                     auto kernel = (1/(4*M_PI)) * rsqrt(rho2);
                     kernel = If(rho2 > 0.0, kernel, SIMD<double,FMM_SW>(0.0));
                     vsum += kernel * c;
+                    */
                   }
               }
-            else
-            if (mp.Kappa() < 1e-8)
+            else if (mp.Kappa() < 1e-8)
               for (auto [x,c] : simd_charges)
                 {
                   auto rho = L2Norm(p-x);
@@ -895,7 +924,7 @@ namespace ngsbem
                   kernel = If(rho > 0.0, kernel, SIMD<Complex,FMM_SW>(0.0));
                   vsum += kernel * c;
                 }
-              
+            
             sum += HSum(vsum);
           }
         else
@@ -911,43 +940,43 @@ namespace ngsbem
                 if (double rho = L2Norm(p-x); rho > 0)
                   sum += (1/(4*M_PI))*exp(Complex(0,rho*mp.Kappa())) / rho * c;
           }
-        }
 
         if (simd_dipoles.Size())
-        {
-          // static Timer t("mptool singmp, evaluate, simd dipoles"); RegionTimer r(t);
-          
-          simd_entry_type vsum{0.0};
-          for (auto [x,d,c] : simd_dipoles)
           {
-            auto rho = L2Norm(p-x);
-            auto drhodp = (1.0/rho) * (p-x);
-            auto [si,co] = sincos(rho*mp.Kappa());
-            auto dGdrho = (1/(4*M_PI))*SIMD<Complex,FMM_SW>(co,si) * 
-                          (-1.0/(rho*rho) + SIMD<Complex,FMM_SW>(0, mp.Kappa())/rho);
-            auto kernel = dGdrho * InnerProduct(drhodp, d);
-            kernel = If(rho > 0.0, kernel, SIMD<Complex,FMM_SW>(0.0));
-            vsum += kernel * c;
+            // static Timer t("mptool singmp, evaluate, simd dipoles"); RegionTimer r(t);
+            
+            simd_entry_type vsum{0.0};
+            for (auto [x,d,c] : simd_dipoles)
+              {
+                auto rho = L2Norm(p-x);
+                auto drhodp = (1.0/rho) * (p-x);
+                auto [si,co] = sincos(rho*mp.Kappa());
+                auto dGdrho = (1/(4*M_PI))*SIMD<Complex,FMM_SW>(co,si) * 
+                  (-1.0/(rho*rho) + SIMD<Complex,FMM_SW>(0, mp.Kappa())/rho);
+                auto kernel = dGdrho * InnerProduct(drhodp, d);
+                kernel = If(rho > 0.0, kernel, SIMD<Complex,FMM_SW>(0.0));
+                vsum += kernel * c;
+              }
+            sum += HSum(vsum);
           }
-          sum += HSum(vsum);
-        }
         else
-        {
-          for (auto [x,d,c] : dipoles)
-          if (double rho = L2Norm(p-x); rho > 0)
           {
-              Vec<3> drhodp = 1.0/rho * (p-x);
-              Complex dGdrho = (1/(4*M_PI))*exp(Complex(0,rho*mp.Kappa())) *
-              (Complex(0, mp.Kappa())/rho - 1.0/sqr(rho));
-              sum += dGdrho * InnerProduct(drhodp, d) * c;
+            for (auto [x,d,c] : dipoles)
+              if (double rho = L2Norm(p-x); rho > 0)
+                {
+                  Vec<3> drhodp = 1.0/rho * (p-x);
+                  Complex dGdrho = (1/(4*M_PI))*exp(Complex(0,rho*mp.Kappa())) *
+                    (Complex(0, mp.Kappa())/rho - 1.0/sqr(rho));
+                  sum += dGdrho * InnerProduct(drhodp, d) * c;
+                }
           }
-        }
-
-
-        
-        if (simd_chargedipoles.Size())
+      
+      
+      
+      if (simd_chargedipoles.Size())
         {
           // static Timer t("mptool singmp, evaluate, simd chargedipoles"); RegionTimer r(t);
+          // t.AddFlops (simd_chargedipoles.Size()*FMM_SW);
           
           simd_entry_type vsum{0.0};
           for (auto [x,c,d,c2] : simd_chargedipoles)
@@ -956,10 +985,10 @@ namespace ngsbem
               auto rhokappa = rho*mp.Kappa();
               auto invrho = If(rho>0.0, 1.0/rho, SIMD<double,FMM_SW>(0.0));
               auto [si,co] = sincos(rhokappa);
-
+              
               auto kernelc = (1/(4*M_PI))*invrho*SIMD<Complex,FMM_SW>(co,si);
               vsum += kernelc * c;   
-
+              
               auto kernel = 
                 invrho*invrho * InnerProduct(p-x, d) * 
                 kernelc * SIMD<Complex,FMM_SW>(-1.0, rhokappa);
@@ -968,15 +997,16 @@ namespace ngsbem
             }
           sum += HSum(vsum);
         }
-        else
+      else
         {
           // static Timer t("mptool singmp, evaluate, chargedipoles"); RegionTimer r(t);
+          // t.AddFlops (chargedipoles.Size());
           
           for (auto [x,c,d,c2] : chargedipoles)
             if (double rho = L2Norm(p-x); rho > 0)
               {
                 sum += (1/(4*M_PI))*exp(Complex(0,rho*mp.Kappa())) / rho * c;
-                  
+                
                 Vec<3> drhodp = 1.0/rho * (p-x);
                 Complex dGdrho = (1/(4*M_PI))*exp(Complex(0,rho*mp.Kappa())) *
                   (Complex(0, mp.Kappa())/rho - 1.0/sqr(rho));
@@ -1025,7 +1055,14 @@ namespace ngsbem
         }
 
         if (dipoles.Size())
-            throw Exception("EvaluateDeriv not implemented for dipoles in SingularMLExpansion");
+          {
+            static int cnt = 0;
+            cnt++;
+            if (cnt < 3)
+              cout << "we know what we do - evaluateDeriv not implemented for dipoles in SingularMLExpansion" << endl;
+            // return sum;
+            // throw Exception("EvaluateDeriv not implemented for dipoles in SingularMLExpansion");
+          }
         if (chargedipoles.Size())
             throw Exception("EvaluateDeriv not implemented for dipoles in SingularMLExpansion");
 
@@ -1134,8 +1171,6 @@ namespace ngsbem
                 simd_chargedipoles[ii] = MakeSimd(di);
               }
 
-
-
             
             if (nodes_to_process)
                 *nodes_to_process += this;
@@ -1224,6 +1259,14 @@ namespace ngsbem
             num += ch->NumCoefficients();
         return num;
       }
+
+      void TraverseTree (const std::function<void(Node&)> & func)
+      {
+        func(*this);
+        for (auto & child : childs)
+          if (child)
+            child->TraverseTree(func);
+      }
     };
     
     Node root;
@@ -1313,89 +1356,93 @@ namespace ngsbem
       
       root.CalcTotalSources();
 
-      if (false)
+      if constexpr (false)
         // direct evaluation of S->S
         root.CalcMP(nullptr, nullptr);
       else
         {
           
-      Array<RecordingSS> recording;
-      Array<Node*> nodes_to_process;
+          Array<RecordingSS> recording;
+          Array<Node*> nodes_to_process;
 
-      {
-        RegionTimer reg(trec);
-      root.CalcMP(&recording, &nodes_to_process);
-      }
+          {
+            RegionTimer reg(trec);
+            root.CalcMP(&recording, &nodes_to_process);
+          }
       
-      {
-        RegionTimer rs2mp(ts2mp);
-        ParallelFor(nodes_to_process.Size(), [&](int i){
-          auto node = nodes_to_process[i];
-          for (auto [x,c]: node->charges)
-            node->mp.AddCharge(x-node->center, c);
-          for (auto [x,d,c]: node->dipoles)
-            node->mp.AddDipole(x-node->center, d, c);
-          for (auto [x,c,d,c2]: node->chargedipoles)
-            node->mp.AddChargeDipole(x-node->center, c, d, c2);
-          for (auto [sp,ep,j,num]: node->currents)
-            node->mp.AddCurrent(sp-node->center, ep-node->center, j, num);
-        }, TasksPerThread(4));
-      }
-
-      {
-      RegionTimer reg(tsort);
-      QuickSort (recording, [] (auto & a, auto & b)
-      {
-        if (a.len < (1-1e-8) * b.len) return true;
-        if (a.len > (1+1e-8) * b.len) return false;
-        return a.theta < b.theta;
-      });
-      }
+          {
+            RegionTimer rs2mp(ts2mp);
+            ParallelFor(nodes_to_process.Size(), [&](int i)
+            {
+              auto node = nodes_to_process[i];
+              for (auto [x,c]: node->charges)
+                node->mp.AddCharge(x-node->center, c);
+              for (auto [x,d,c]: node->dipoles)
+                node->mp.AddDipole(x-node->center, d, c);
+              for (auto [x,c,d,c2]: node->chargedipoles)
+                node->mp.AddChargeDipole(x-node->center, c, d, c2);
+              for (auto [sp,ep,j,num]: node->currents)
+                node->mp.AddCurrent(sp-node->center, ep-node->center, j, num);
+            }, TasksPerThread(4));
+          }
+          
+          {
+            RegionTimer reg(tsort);
+            QuickSort (recording, [] (auto & a, auto & b)
+            {
+              if (a.len < (1-1e-8) * b.len) return true;
+              if (a.len > (1+1e-8) * b.len) return false;
+              return a.theta < b.theta;
+            });
+          }
       
-      double current_len = -1e100;
-      double current_theta = -1e100;
-      Array<RecordingSS*> current_batch;
-      Array<Array<RecordingSS*>> batch_group;
-      Array<double> group_lengths;
-      Array<double> group_thetas;
-      for (auto & record : recording)
-        {
-          bool len_changed = fabs(record.len - current_len) > 1e-8;
-          bool theta_changed = fabs(record.theta - current_theta) > 1e-8;
-          if ((len_changed || theta_changed) && current_batch.Size() > 0) {
+          double current_len = -1e100;
+          double current_theta = -1e100;
+          Array<RecordingSS*> current_batch;
+          Array<Array<RecordingSS*>> batch_group;
+          Array<double> group_lengths;
+          Array<double> group_thetas;
+          for (auto & record : recording)
+            {
+              bool len_changed = fabs(record.len - current_len) > 1e-8;
+              bool theta_changed = fabs(record.theta - current_theta) > 1e-8;
+              if ((len_changed || theta_changed) && current_batch.Size() > 0) {
+                batch_group.Append(current_batch);
+                group_lengths.Append(current_len);
+                group_thetas.Append(current_theta);
+                current_batch.SetSize(0);
+              }
+              
+              current_len = record.len;
+              current_theta = record.theta;
+              current_batch.Append(&record);
+            }
+          
+          if (current_batch.Size() > 0) {
             batch_group.Append(current_batch);
             group_lengths.Append(current_len);
             group_thetas.Append(current_theta);
-            current_batch.SetSize(0);
-            }
+          }
 
-          current_len = record.len;
-          current_theta = record.theta;
-          current_batch.Append(&record);
+          {
+            RegionTimer rS2S(tS2S);
+            // ParallelFor(batch_group.Size(), [&](int i) {
+            for (int i = 0; i < batch_group.Size(); i++){
+              // *testout << "Processing batch " << i << " of size " << batch_group[i].Size() << ", with len = " << group_lengths[i] << ", theta = " << group_thetas[i] << endl;
+              int chunk_size = 24;
+              if (batch_group[i].Size() < chunk_size)
+                ProcessBatchSS(batch_group[i], group_lengths[i], group_thetas[i]);
+              else
+                ParallelForRange(IntRange(batch_group[i].Size()), [&](IntRange range) {
+                  auto sub_batch = batch_group[i].Range(range.First(), range.Next());
+                  ProcessBatchSS(sub_batch, group_lengths[i], group_thetas[i]);
+                }, TasksPerThread(4));
+            }
+          }
         }
-      if (current_batch.Size() > 0) {
-        batch_group.Append(current_batch);
-        group_lengths.Append(current_len);
-        group_thetas.Append(current_theta);
-      }
 
-      {
-        RegionTimer rS2S(tS2S);
-      // ParallelFor(batch_group.Size(), [&](int i) {
-      for (int i = 0; i < batch_group.Size(); i++){
-          // *testout << "Processing batch " << i << " of size " << batch_group[i].Size() << ", with len = " << group_lengths[i] << ", theta = " << group_thetas[i] << endl;
-        int chunk_size = 24;
-        if (batch_group[i].Size() < chunk_size)
-            ProcessBatch(batch_group[i], group_lengths[i], group_thetas[i]);
-        else
-          ParallelForRange(IntRange(batch_group[i].Size()), [&](IntRange range) {
-              auto sub_batch = batch_group[i].Range(range.First(), range.Next());
-              ProcessBatch(sub_batch, group_lengths[i], group_thetas[i]);
-          }, TasksPerThread(4));
-      }
-      }
-        }
-      
+      // cout << "have singular:" << endl;
+      // PrintStatistics (cout);
       havemp = true;
     }
 
@@ -1407,6 +1454,41 @@ namespace ngsbem
         return root.Evaluate(p);
     }
 
+
+    void PrintStatistics (ostream & ost)
+    {
+      int levels = 0;
+      int cnt = 0;
+      root.TraverseTree( [&](Node & node) {
+        levels = max(levels, node.level);
+        cnt++;
+      });
+      ost << "levels: " << levels << endl;
+      ost << "nodes: " << cnt << endl;
+
+      Array<int> num_on_level(levels+1);
+      Array<int> order_on_level(levels+1);
+      Array<size_t> coefs_on_level(levels+1);            
+      num_on_level = 0;
+      order_on_level = 0;
+      root.TraverseTree( [&](Node & node) {
+        num_on_level[node.level]++;
+        order_on_level[node.level] = max(order_on_level[node.level],node.mp.Order());
+        coefs_on_level[node.level] += node.mp.SH().Coefs().Size();
+      });
+
+      cout << "num on level" << endl;
+      for (int i = 0; i < num_on_level.Size(); i++)
+        cout << i << ": " << num_on_level[i] << ", order = " << order_on_level[i] << ", coefs " << coefs_on_level[i] << endl;
+
+      size_t totcoefs = 0;
+      for (auto n : coefs_on_level)
+        totcoefs += n;
+      cout << "total mem in coefs: " << sizeof(entry_type)*totcoefs / sqr(1024) << " MB" << endl;
+    }
+
+
+    
     template <typename entry_type2>
     friend class RegularMLExpansion;
   };
@@ -1420,6 +1502,9 @@ namespace ngsbem
   }
 
 
+  // *********************************** Regular multilevel Expansion
+  
+
   template <typename elem_type=Complex>
   class NGS_DLL_HEADER RegularMLExpansion
   {
@@ -1456,28 +1541,28 @@ namespace ngsbem
         }
       }
       else if (N <= 3) {
-        ProcessVectorizedBatch<3, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchRS<3, vec_length>(batch, len, theta);
       }
       else if (N <= 4) {
-        ProcessVectorizedBatch<4, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchRS<4, vec_length>(batch, len, theta);
       }
       else if (N <= 6) {
-        ProcessVectorizedBatch<6, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchRS<6, vec_length>(batch, len, theta);
       }
       else if (N <= 12) {
-        ProcessVectorizedBatch<12, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchRS<12, vec_length>(batch, len, theta);
       }
       else if (N <= 24) {
-        ProcessVectorizedBatch<24, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchRS<24, vec_length>(batch, len, theta);
       }
       else if (N <= 48) {
-        ProcessVectorizedBatch<48, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchRS<48, vec_length>(batch, len, theta);
       }
       else if (N <= 96) {
-        ProcessVectorizedBatch<96, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchRS<96, vec_length>(batch, len, theta);
       }
       else if (N <= 192) {
-        ProcessVectorizedBatch<192, vec_length>(batch, len, theta);
+        ProcessVectorizedBatchRS<192, vec_length>(batch, len, theta);
       }
       else {
         // Split large batches
@@ -1509,7 +1594,7 @@ namespace ngsbem
 
 
     template<int N, int vec_length>
-    static void ProcessVectorizedBatch(FlatArray<RecordingRS*> batch, double len, double theta) {
+    static void ProcessVectorizedBatchRS(FlatArray<RecordingRS*> batch, double len, double theta) {
 
       // static Timer t("ProcessVectorizedBatch, N = "+ToString(N) + ", vec_len = " + ToString(vec_length));
       // RegionTimer reg(t, batch[0]->mpS->SH().Order());
@@ -1570,6 +1655,7 @@ namespace ngsbem
       std::array<unique_ptr<Node>,8> childs;
       SphericalExpansion<Regular,elem_type> mp;
       Array<Vec<3>> targets;
+      Array<tuple<Vec<3>,double>> vol_targets;      
       int total_targets;
       std::mutex node_mutex;
       atomic<bool> have_childs{false};
@@ -1577,15 +1663,22 @@ namespace ngsbem
       Array<const typename SingularMLExpansion<elem_type>::Node*> singnodes;
 
       Node (Vec<3> acenter, double ar, int alevel, double kappa)
-        : center(acenter), r(ar), level(alevel), mp(MPOrder(ar*kappa), kappa, ar) // 1.0/min(1.0, 0.25*r*kappa))
+        : center(acenter), r(ar), level(alevel),
+          // mp(MPOrder(ar*kappa), kappa, ar) // 1.0/min(1.0, 0.25*r*kappa))
+          mp(-1, kappa, ar)
           // : center(acenter), r(ar), level(alevel), mp(MPOrder(ar*kappa), kappa, 1.0)
       {
         if (level < nodes_on_level.Size())
           nodes_on_level[level]++;
       }
 
-
-      void CreateChilds()
+      void Allocate()
+      {
+        mp = SphericalExpansion<Regular,elem_type>(MPOrder(r*mp.Kappa()), mp.Kappa(), r);
+      }
+      
+      
+      void CreateChilds(bool allocate = false)
       {
         if (childs[0]) throw Exception("have already childs");
         // create children nodes:
@@ -1596,6 +1689,8 @@ namespace ngsbem
             cc(1) += (i&2) ? r/2 : -r/2;
             cc(2) += (i&4) ? r/2 : -r/2;
             childs[i] = make_unique<Node> (cc, r/2, level+1, mp.Kappa());
+            if (allocate)
+              childs[i] -> Allocate();
           }
         have_childs = true;
       }
@@ -1648,7 +1743,7 @@ namespace ngsbem
             if (allow_refine)
               {
                 if (!childs[0])
-                  CreateChilds();
+                  CreateChilds(true);
                 
                 for (auto & ch : childs)
                   ch -> AddSingularNode (singnode, allow_refine, recording);
@@ -1668,7 +1763,7 @@ namespace ngsbem
                                    childs[nr] -> AddSingularNode (singnode, allow_refine, recording);
                                });
                 
-                if (targets.Size())
+                if (targets.Size()+vol_targets.Size())
                   singnodes.Append(&singnode);
               }
           }
@@ -1683,7 +1778,7 @@ namespace ngsbem
       {
         if (allow_refine)
           if (mp.Order() > 30 && !childs[0])
-            CreateChilds();
+            CreateChilds(allow_refine);
 
         if (childs[0])
           {
@@ -1754,6 +1849,14 @@ namespace ngsbem
         return sum;
       }
 
+      void TraverseTree (const std::function<void(Node&)> & func)
+      {
+        func(*this);
+        for (auto & child : childs)
+          if (child)
+            child->TraverseTree(func);
+      }
+      
       double Norm() const
       {
         double norm = L2Norm(mp.SH().Coefs());
@@ -1771,17 +1874,23 @@ namespace ngsbem
             num += ch->NumCoefficients();
         return num;
       }
-      
+
+      int GetChildNum (Vec<3> x) const
+      {
+        int childnum  = 0;
+        if (x(0) > center(0)) childnum += 1;
+        if (x(1) > center(1)) childnum += 2;
+        if (x(2) > center(2)) childnum += 4;
+        return childnum;
+      }
+
       void AddTarget (Vec<3> x)
       {
         // if (childs[0])
         if (have_childs) // quick check without locking
           {
             // directly send to childs:
-            int childnum  = 0;
-            if (x(0) > center(0)) childnum += 1;
-            if (x(1) > center(1)) childnum += 2;
-            if (x(2) > center(2)) childnum += 4;
+            int childnum  = GetChildNum(x);
             childs[childnum] -> AddTarget( x );
             return;
           }
@@ -1791,32 +1900,74 @@ namespace ngsbem
         if (have_childs) // test again after locking
         {
           // directly send to childs:
-          int childnum  = 0;
-          if (x(0) > center(0)) childnum += 1;
-          if (x(1) > center(1)) childnum += 2;
-          if (x(2) > center(2)) childnum += 4;
+          int childnum  = GetChildNum(x);
           childs[childnum] -> AddTarget(x);
           return;
         }
 
-
         targets.Append( x );
 
         // if (r*mp.Kappa() < 1e-8) return;
         if (level > 20) return;        
-        if (targets.Size() < maxdirect && r*mp.Kappa() < 1)
+        if (targets.Size() < maxdirect && r*mp.Kappa() < 5)
+          return;
+
+        CreateChilds();
+
+        for (auto t : targets)
+          AddTarget (t);
+        for (auto [x,r] : vol_targets)
+          AddVolumeTarget (x,r);
+
+        targets.SetSize0();
+        vol_targets.SetSize0();
+      }
+
+
+      void AddVolumeTarget (Vec<3> x, double tr)
+      {
+        if (MaxNorm(x-center) > r+tr) return;
+
+        if (have_childs)
+          {
+            for (auto & child : childs)
+              child->AddVolumeTarget(x, tr);
+            return;
+          }
+
+        
+        lock_guard<mutex> guard(node_mutex);
+        
+        if (have_childs)
+          {
+            for (auto & child : childs)
+              child->AddVolumeTarget(x, tr);
+            return;
+          }
+
+        
+        vol_targets.Append (tuple(x,tr));
+
+        if (level > 20) return;
+        if (vol_targets.Size() < maxdirect && (r*mp.Kappa() < 5))
           return;
 
         CreateChilds();
 
         for (auto t : targets)
           AddTarget (t);
+        for (auto [x,r] : vol_targets)
+          AddVolumeTarget (x,r);
+
         targets.SetSize0();
+        vol_targets.SetSize0();
       }
 
+      
+
       void CalcTotalTargets()
       {
-        total_targets = targets.Size();
+        total_targets = targets.Size() + vol_targets.Size();
         for (auto & child : childs)
           if (child)
             {
@@ -1838,6 +1989,19 @@ namespace ngsbem
         if (total_targets == 0)
           mp = SphericalExpansion<Regular,elem_type>(-1, mp.Kappa(),1.);
       }
+
+      void AllocateMemory()
+      {
+        for (auto & child : childs)
+          if (child)
+            child->AllocateMemory();
+
+        if (total_targets > 0)
+          Allocate();
+        // mp = SphericalExpansion<Regular,elem_type>(MPOrder(r*mp.Kappa()), mp.Kappa(), r); // -1, mp.Kappa(),1.);
+      }
+
+
       
 
       void Print (ostream & ost, size_t childnr = -1) const
@@ -1863,7 +2027,8 @@ namespace ngsbem
       : root(center, r, 0, asingmp->Kappa()), singmp(asingmp)
     {
       if (!singmp->havemp) throw Exception("first call Calc for singular MP");
-
+      root.Allocate();
+      
       nodes_on_level = 0;
       nodes_on_level[0] = 1;
       {
@@ -1901,73 +2066,98 @@ namespace ngsbem
       root.AddTarget (t);
     }
 
+    void AddVolumeTarget (Vec<3> t, double r)
+    {
+      root.AddVolumeTarget (t, r);
+    }
+
     void CalcMP(shared_ptr<SingularMLExpansion<elem_type>> asingmp, bool onlytargets = true)
     {
       static Timer t("mptool regular MLMP"); RegionTimer rg(t);
+      static Timer tremove("removeempty");
       static Timer trec("mptool regular MLMP - recording");
       static Timer tsort("mptool regular MLMP - sort");       
       
       singmp = asingmp;
 
+      
       root.CalcTotalTargets();
+      // cout << "before remove empty trees:" << endl;
+      // PrintStatistics(cout);
+
+      /*
+      tremove.Start();
       if (onlytargets)
         root.RemoveEmptyTrees();
-
+      tremove.Stop();
+      */
       
-      // root.AddSingularNode(singmp->root, !onlytargets, nullptr);
+      root.AllocateMemory();
 
-      // /*
-      Array<RecordingRS> recording;
-      {
-        RegionTimer rrec(trec);
-        root.AddSingularNode(singmp->root, !onlytargets, &recording);
-      }
-      
-      // cout << "recorded: " << recording.Size() << endl;
-      {
-      RegionTimer reg(tsort);
-      QuickSort (recording, [] (auto & a, auto & b)
-      {
-        if (a.len < (1-1e-8) * b.len) return true;
-        if (a.len > (1+1e-8) * b.len) return false;
-        return a.theta < b.theta;
-      });
-      }
-      
-      double current_len = -1e100;
-      double current_theta = -1e100;
-      Array<RecordingRS*> current_batch;
-      Array<Array<RecordingRS*>> batch_group;
-      Array<double> group_lengths;
-      Array<double> group_thetas;
-      for (auto & record : recording)
+      // cout << "after allocating regular:" << endl;
+      // PrintStatistics(cout);
+
+      // cout << "starting S-R converion" << endl;
+      // PrintStatistics(cout);
+
+
+      if constexpr (false)
         {
-          bool len_changed = fabs(record.len - current_len) > 1e-8;
-          bool theta_changed = fabs(record.theta - current_theta) > 1e-8;
-          if ((len_changed || theta_changed) && current_batch.Size() > 0) {
+          root.AddSingularNode(singmp->root, !onlytargets, nullptr);
+        }
+      else
+        {  // use recording
+          Array<RecordingRS> recording;
+          {
+            RegionTimer rrec(trec);
+            root.AddSingularNode(singmp->root, !onlytargets, &recording);
+          }
+          
+          // cout << "recorded: " << recording.Size() << endl;
+          {
+            RegionTimer reg(tsort);
+            QuickSort (recording, [] (auto & a, auto & b)
+            {
+              if (a.len < (1-1e-8) * b.len) return true;
+              if (a.len > (1+1e-8) * b.len) return false;
+              return a.theta < b.theta;
+            });
+          }
+          
+          double current_len = -1e100;
+          double current_theta = -1e100;
+          Array<RecordingRS*> current_batch;
+          Array<Array<RecordingRS*>> batch_group;
+          Array<double> group_lengths;
+          Array<double> group_thetas;
+          for (auto & record : recording)
+            {
+              bool len_changed = fabs(record.len - current_len) > 1e-8;
+              bool theta_changed = fabs(record.theta - current_theta) > 1e-8;
+              if ((len_changed || theta_changed) && current_batch.Size() > 0) {
+                // ProcessBatch(current_batch, current_len, current_theta);
+                batch_group.Append(current_batch);
+                group_lengths.Append(current_len);
+                group_thetas.Append(current_theta);
+                current_batch.SetSize(0);
+              }
+              
+              current_len = record.len;
+              current_theta = record.theta;
+              current_batch.Append(&record);
+            }
+          if (current_batch.Size() > 0) {
             // ProcessBatch(current_batch, current_len, current_theta);
             batch_group.Append(current_batch);
             group_lengths.Append(current_len);
             group_thetas.Append(current_theta);
-            current_batch.SetSize(0);
-            }
-
-          current_len = record.len;
-          current_theta = record.theta;
-          current_batch.Append(&record);
+          }
+          
+          ParallelFor(batch_group.Size(), [&](int i) {
+            ProcessBatchRS(batch_group[i], group_lengths[i], group_thetas[i]);
+          }, TasksPerThread(4));
         }
-      if (current_batch.Size() > 0) {
-        // ProcessBatch(current_batch, current_len, current_theta);
-        batch_group.Append(current_batch);
-        group_lengths.Append(current_len);
-        group_thetas.Append(current_theta);
-      }
-
-      ParallelFor(batch_group.Size(), [&](int i) {
-          ProcessBatchRS(batch_group[i], group_lengths[i], group_thetas[i]);
-      }, TasksPerThread(4));
-      // */
-
+          
       
       /*
       int maxlevel = 0;
@@ -1978,10 +2168,49 @@ namespace ngsbem
         cout << "reg " << i << ": " << RegularMLExpansion::nodes_on_level[i] << endl;
       */
 
+      // cout << "starting R-R converion" << endl;
+      // PrintStatistics(cout);
+      
       static Timer tloc("mptool regular localize expansion"); RegionTimer rloc(tloc);
       root.LocalizeExpansion(!onlytargets);
+
+
+      // cout << "R-R conversion done" << endl;
+      // PrintStatistics(cout);      
     }
 
+    void PrintStatistics (ostream & ost)
+    {
+      int levels = 0;
+      int cnt = 0;
+      root.TraverseTree( [&](Node & node) {
+        levels = max(levels, node.level);
+        cnt++;
+      });
+      ost << "levels: " << levels << endl;
+      ost << "nodes: " << cnt << endl;
+
+      Array<int> num_on_level(levels+1);
+      Array<int> order_on_level(levels+1);
+      Array<size_t> coefs_on_level(levels+1);            
+      num_on_level = 0;
+      order_on_level = 0;
+      root.TraverseTree( [&](Node & node) {
+        num_on_level[node.level]++;
+        order_on_level[node.level] = max(order_on_level[node.level],node.mp.Order());
+        coefs_on_level[node.level] += node.mp.SH().Coefs().Size();
+      });
+
+      cout << "num on level" << endl;
+      for (int i = 0; i < num_on_level.Size(); i++)
+        cout << i << ": " << num_on_level[i] << ", order = " << order_on_level[i] << ", coefs " << coefs_on_level[i] << endl;
+
+      size_t totcoefs = 0;
+      for (auto n : coefs_on_level)
+        totcoefs += n;
+      cout << "total mem in coefs: " << sizeof(elem_type)*totcoefs / sqr(1024) << " MB" << endl;
+    }
+    
     void Print (ostream & ost) const
     {
       root.Print(ost);
@@ -2001,6 +2230,7 @@ namespace ngsbem
     {
       // static Timer t("mptool Eval MLMP regular"); RegionTimer r(t);
       // if (L2Norm(p-root.center) > root.r) return elem_type{0.0};
+      
       if (MaxNorm(p-root.center) > root.r)
         return singmp->Evaluate(p);
       return root.Evaluate(p);