robotic 0.2.9.dev1__cp311-cp311-manylinux2014_x86_64.whl → 0.3.4.dev5__cp311-cp311-manylinux2014_x86_64.whl

robotic/include/rai/Core/array.ipp

@@ -11,14 +11,13 @@
 #include "array.h"
 
 #include <algorithm>
+#include <type_traits>
 
 #define ARRAY_flexiMem true
 
 namespace rai {
 
 //fwd declarations
-extern int64_t globalMemoryTotal, globalMemoryBound;
-extern bool globalMemoryStrict;
 
 extern uint lineCount;
 char skip(std::istream& is, const char* skipSymbols, const char* stopSymbols, bool skipCommentLines);
@@ -42,7 +41,7 @@ template<class T> Array<T>::Array()
     N(0),
     nd(0),
     d0(0), d1(0), d2(0),
-    d(&d0),
+    _shape(&d0),
     isReference(false),
     M(0),
     special(0) {
@@ -73,14 +72,18 @@ template<class T> Array<T>::Array(Array<T>&& a)
     N(a.N),
     nd(a.nd),
     d0(a.d0), d1(a.d1), d2(a.d2),
-    d(&d0),
+    _shape(&d0),
     isReference(a.isReference),
     M(a.M),
     special(a.special) {
-  //if(a.jac) jac = std::move(a.jac);
-  CHECK_EQ(a.d, &a.d0, "");
+  if constexpr(std::is_same_v<T, double>){
+    if(a.jac) jac = std::move(a.jac);
+  }
+  // CHECK_EQ(a.shape, &a.d0, "NIY for larger tensors");
+  if(a._shape!=&a.d0) { _shape=a._shape; a._shape=&a.d0; }
   a.p=NULL;
   a.N=a.nd=a.d0=a.d1=a.d2=0;
+  a.resetShape();
   a.isReference=false;
   a.special=NULL;
 }
@@ -100,11 +103,14 @@ template<class T> Array<T>::Array(std::initializer_list<T> values) : Array() { o
 /// initialization via {1., 2., 3., ...} lists, with certain dimensionality
 template<class T> Array<T>::Array(std::initializer_list<uint> dim, std::initializer_list<T> values) : Array() { operator=(values); reshape(dim); }
 
+template<class T> Array<T>::Array(const T* p, uint size, bool byReference) : Array() { if(byReference) referTo(p, size); else setCarray(p, size); }
+
 template<class T> Array<T>::~Array() {
 #if 0
   clear();
 #else //faster (leaves members non-zeroed..)
   if(special) { delete special; special=NULL; }
+  if(_shape!=&d0) { delete[] _shape; }
   if(M) {
     globalMemoryTotal -= M*sizeT;
     if(memMove==1) free(p); else delete[] p;
@@ -120,57 +126,56 @@ template<class T> Array<T>&  Array<T>::clear() {
 }
 
 /// resize 1D array, discard the previous contents
-template<class T> Array<T>& Array<T>::resize(uint D0) { nd=1; d0=D0; resetD(); resizeMEM(d0, false); return *this; }
+template<class T> Array<T>& Array<T>::resize(uint D0) { nd=1; d0=D0; resetShape(); resizeMEM(d0, false); return *this; }
 
 /// resize but copy the previous contents
-template<class T> Array<T>& Array<T>::resizeCopy(uint D0) { nd=1; d0=D0; resetD(); resizeMEM(d0, true); return *this; }
+template<class T> Array<T>& Array<T>::resizeCopy(uint D0) { nd=1; d0=D0; resetShape(); resizeMEM(d0, true); return *this; }
 
 /// reshape the dimensionality (e.g. from 2D to 1D); throw an error if this actually requires to resize the memory
 template<class T> Array<T>& Array<T>::reshape(int D0) {
   if(D0<0) D0=N;
   CHECK_EQ((int)N, D0, "reshape must preserve total memory size");
-  nd=1; d0=D0; d1=d2=0; resetD();
+  nd=1; d0=D0; d1=d2=0; resetShape();
   return *this;
 }
 
 /// same for 2D ...
-template<class T> Array<T>& Array<T>::resize(uint D0, uint D1) { nd=2; d0=D0; d1=D1; resetD(); resizeMEM(d0*d1, false); return *this; }
+template<class T> Array<T>& Array<T>::resize(uint D0, uint D1) { nd=2; d0=D0; d1=D1; resetShape(); resizeMEM(d0*d1, false); return *this; }
 
 /// ...
-template<class T> Array<T>& Array<T>::resizeCopy(uint D0, uint D1) { nd=2; d0=D0; d1=D1; resetD(); resizeMEM(d0*d1, true); return *this; }
+template<class T> Array<T>& Array<T>::resizeCopy(uint D0, uint D1) { nd=2; d0=D0; d1=D1; resetShape(); resizeMEM(d0*d1, true); return *this; }
 
 /// ...
 template<class T> Array<T>& Array<T>::reshape(int D0, int D1) {
   if(D0<0) D0=N/D1; else if(D1<0) D1=N/D0;
   CHECK_EQ((int)N, D0*D1, "reshape must preserve total memory size");
   nd=2; d0=D0; d1=D1; d2=0;
-  resetD();
+  resetShape();
   return *this;
 }
 
 /// same for 3D ...
-template<class T> Array<T>& Array<T>::resize(uint D0, uint D1, uint D2) { nd=3; d0=D0; d1=D1; d2=D2; resetD(); resizeMEM(d0*d1*d2, false); return *this; }
+template<class T> Array<T>& Array<T>::resize(uint D0, uint D1, uint D2) { nd=3; d0=D0; d1=D1; d2=D2; resetShape(); resizeMEM(d0*d1*d2, false); return *this; }
 
 /// ...
-template<class T> Array<T>& Array<T>::resizeCopy(uint D0, uint D1, uint D2) { nd=3; d0=D0; d1=D1; d2=D2; resetD(); resizeMEM(d0*d1*d2, true); return *this; }
+template<class T> Array<T>& Array<T>::resizeCopy(uint D0, uint D1, uint D2) { nd=3; d0=D0; d1=D1; d2=D2; resetShape(); resizeMEM(d0*d1*d2, true); return *this; }
 
 /// ...
 template<class T> Array<T>& Array<T>::reshape(int D0, int D1, int D2) {
   if(D0<0) D0=N/(D1*D2); else if(D1<0) D1=N/(D0*D2); else if(D2<0) D2=N/(D0*D1);
   CHECK_EQ((int)N, D0*D1*D2, "reshape must preserve total memory size");
   nd=3; d0=D0; d1=D1; d2=D2;
-  resetD();
+  resetShape();
   return *this;
 }
 
 /// resize to multi-dimensional tensor
 template<class T> Array<T>& Array<T>::resize(uint ND, uint* dim) {
-  nd=ND; d0=d1=d2=0; resetD();
-  uint j;
-  for(j=0; j<nd && j<3; j++) {(&d0)[j]=dim[j]; }
-  if(nd>3) { d=new uint[nd];  memmove(d, dim, nd*sizeof(uint)); }
-  uint64_t S;
-  for(S=1, j=0; j<nd; j++) S*=dim[j];
+  nd=ND; d0=d1=d2=0;
+  if(nd>0){ d0=dim[0]; if(nd>1){ d1=dim[1]; if(nd>2){ d2=dim[2]; } } }
+  resetShape(dim);
+  uint64_t S=(nd>0?1:0);
+  for(uint j=0; j<nd; j++) S*=dim[j];
   if(S>=(1ull <<32)) HALT("Array #elements " <<(S>>30) <<"G is >= 2^32");
   resizeMEM((uint)S, false);
   return *this;
@@ -178,12 +183,11 @@ template<class T> Array<T>& Array<T>::resize(uint ND, uint* dim) {
 
 /// resize to multi-dimensional tensor
 template<class T> Array<T>& Array<T>::resizeCopy(uint ND, uint* dim) {
-  nd=ND; d0=d1=d2=0; resetD();
-  uint j;
-  for(j=0; j<nd && j<3; j++) {(&d0)[j]=dim[j]; }
-  if(nd>3) { d=new uint[nd];  memmove(d, dim, nd*sizeof(uint)); }
-  uint64_t S;
-  for(S=1, j=0; j<nd; j++) S*=dim[j];
+  nd=ND; d0=d1=d2=0;
+  if(nd>0){ d0=dim[0]; if(nd>1){ d1=dim[1]; if(nd>2){ d2=dim[2]; } } }
+  resetShape(dim);
+  uint64_t S=(nd>0?1:0);
+  for(uint j=0; j<nd; j++) S*=dim[j];
   if(S>=(1ull <<32)) HALT("Array #elements " <<(S>>30) <<"G is >= 2^32");
   resizeMEM((uint)S, true);
   return *this;
@@ -191,19 +195,10 @@ template<class T> Array<T>& Array<T>::resizeCopy(uint ND, uint* dim) {
 
 /// resize to multi-dimensional tensor
 template<class T> Array<T>& Array<T>::reshape(uint ND, uint* dim) {
-  nd=ND; d0=d1=d2=0; resetD();
-  if(nd>0){
-    d0=dim[0];
-    if(nd>1){
-      d1=dim[1];
-      if(nd>2){
-        d2=dim[2];
-        if(nd>3) { d=new uint[nd];  memmove(d, dim, nd*sizeof(uint)); }
-      }
-    }
-  }
-  //for(uint j=0; j<nd && j<3; j++) {(&d0)[j]=dim[j]; }
-  uint S=(nd>0?1:0);
+  nd=ND; d0=d1=d2=0;
+  if(nd>0){ d0=dim[0]; if(nd>1){ d1=dim[1]; if(nd>2){ d2=dim[2]; } } }
+  resetShape(dim);
+  uint64_t S=(nd>0?1:0);
   for(uint j=0; j<nd; j++) S*=dim[j];
   CHECK_EQ(N, S, "reshape must preserve total memory size");
   return *this;
@@ -224,8 +219,7 @@ template<class T> Array<T>& Array<T>::resizeAs(const Array<T>& a) {
   CHECK(this!=&a, "never do this!!!");
   if(isReference) CHECK_EQ(N, a.N, "resize of a reference (e.g. subarray) is not allowed! (only a resize without changing memory size)");
   nd=a.nd; d0=a.d0; d1=a.d1; d2=a.d2;
-  resetD();
-  if(nd>3) { d=new uint[nd];  memmove(d, a.d, nd*sizeof(uint)); }
+  resetShape(a._shape);
   resizeMEM(a.N, false);
   return *this;
 }
@@ -233,8 +227,8 @@ template<class T> Array<T>& Array<T>::resizeAs(const Array<T>& a) {
 /// make it the same size as \c a and copy previous content
 template<class T> Array<T>& Array<T>::resizeCopyAs(const Array<T>& a) {
   CHECK(this!=&a, "never do this!!!");
-  nd=a.nd; d0=a.d0; d1=a.d1; d2=a.d2; resetD();
-  if(nd>3) { d=new uint[nd];  memmove(d, a.d, nd*sizeof(uint)); }
+  nd=a.nd; d0=a.d0; d1=a.d1; d2=a.d2;
+  resetShape(a._shape);
   resizeMEM(a.N, true);
   return *this;
 }
@@ -242,20 +236,15 @@ template<class T> Array<T>& Array<T>::resizeCopyAs(const Array<T>& a) {
 template<class T> Array<T>& Array<T>::reshapeAs(const Array<T>& a) {
   CHECK(this!=&a, "never do this!!!");
   CHECK_EQ(N, a.N, "reshape must preserve total memory size");
-  nd=a.nd; d0=a.d0; d1=a.d1; d2=a.d2; resetD();
-  if(nd>3) { d=new uint[nd];  memmove(d, a.d, nd*sizeof(uint)); }
-  return *this;
-}
-
-template<class T> Array<T>& Array<T>::reshapeFlat() {
-  reshape(N);
+  nd=a.nd; d0=a.d0; d1=a.d1; d2=a.d2;
+  resetShape(a._shape);
   return *this;
 }
 
 /// return the k-th dimensionality
 template<class T> uint Array<T>::dim(uint k) const {
   CHECK(k<nd, "dimensionality range check error: " <<k <<"!<" <<nd);
-  if(!d && k<3) return (&d0)[k]; else return d[k];
+  if(!_shape && k<3) return (&d0)[k]; else return _shape[k];
 }
 
 #ifdef RAI_CLANG
@@ -363,10 +352,9 @@ template<class T> void Array<T>::freeMEM() {
     M=0;
   }
 #endif
-  if(d && d!=&d0) { delete[] d; d=NULL; }
+  if(_shape!=&d0) { delete[] _shape; _shape=&d0; }
   p=NULL;
   N=nd=d0=d1=d2=0;
-  d=&d0;
   isReference=false;
 }
 
@@ -386,9 +374,13 @@ template<class T> Array<T>& Array<T>::dereference() {
 }
 
 /// reset the dimensionality pointer d to point to &d0
-template<class T> void Array<T>::resetD() {
-  if(d && d!=&d0) { delete[] d; d=NULL; }
-  d=&d0;
+template<class T> void Array<T>::resetShape(uint* dim) {
+  if(_shape!=&d0) { delete[] _shape; _shape=&d0; }
+  if(nd>3) {
+    CHECK(dim, "need shape tuple");
+    _shape=new uint[nd];
+    memmove(_shape, dim, nd*sizeof(uint));
+  }
 }
 
 //***** append, insert & remove
@@ -641,10 +633,10 @@ template<class T> void Array<T>::delColumns(int i, uint k) {
 }
 
 /// inserts k columns at the i-th column [must be 2D]
-template<class T> void Array<T>::insColumns(int i, uint k) {
+template<class T> Array<T>& Array<T>::insColumns(int i, uint k) {
   CHECK(memMove, "only with memMove");
   CHECK_EQ(nd, 2, "only for matricies");
-  if(!k) return;
+  if(!k) return *this;
   if(i<0) i+=d1+1;
   CHECK_LE(i, (int)d1, "range check error");
   uint n=d1;
@@ -654,6 +646,7 @@ template<class T> void Array<T>::insColumns(int i, uint k) {
     memset(p+j*d1+i, 0, sizeT*k);
     if(i) memmove(p+j*d1, p+j*n, sizeT*i);
   }
+  return *this;
 }
 
 /// changes the range of one dimension (generalization of ins/delColumn to arbitrary tensors)
@@ -687,7 +680,7 @@ template<class T> void Array<T>::resizeDim(uint k, uint dk) {
 /// return a uint-Array that contains (acutally refers to) the dimensions of 'this'
 template<class T> Array<uint> Array<T>::dim() const {
   Array<uint> dims;
-  dims.setCarray(d, nd);
+  dims.setCarray(_shape, nd);
   return dims;
 }
 
@@ -710,8 +703,15 @@ template<class T> T& Array<T>::elem(int i, int j) {
   if(j<0) j += d1;
   CHECK(nd==2 && (uint)i<d0 && (uint)j<d1,
         "2D range error (" <<nd <<"=2, " <<i <<"<" <<d0 <<", " <<j <<"<" <<d1 <<")");
+  if constexpr(std::is_same_v<T, double>){
+    if(isSparseMatrix(*this)) {
+      return sparse().addEntry(i, j);
+    }
+    if(isRowShifted(*this)) {
+      return rowShifted().elemNew(i, j);
+    }
+  }
   return p[i*d1+j];
-
 }
 
 /// multi-dimensional (tensor) access
@@ -767,7 +767,7 @@ template<class T> Array<T> Array<T>::ref() const {
 
 template<class T> Array<T> Array<T>::operator()(std::pair<int, int> I) const {
   Array<T> z;
-  z.referToRange(*this, I.first, I.second);
+  z.referToRange(*this, I);
   //  if(I.size()==2) z.referToRange(*this, I.begin()[0], I.begin()[1]);
   //  else if(I.size()==0) z.referTo(*this);
   //  else if(I.size()==1) z.referToDim(*this, I.begin()[0]);
@@ -777,9 +777,8 @@ template<class T> Array<T> Array<T>::operator()(std::pair<int, int> I) const {
 
 /// range reference access
 template<class T> Array<T> Array<T>::operator()(int i, std::pair<int, int> J) const {
-  if(i<0) i += d0;
   Array<T> z;
-  z.referToRange(*this, i, J.first, J.second);
+  z.referToRange(*this, i, J);
 //  if(J.size()==2)
 //  else if(J.size()==0) z.referToDim(*this, i);
 //  else if(J.size()==1) z.referToDim(*this, i, J.begin()[0]);
@@ -789,10 +788,8 @@ template<class T> Array<T> Array<T>::operator()(int i, std::pair<int, int> J) co
 
 /// range reference access
 template<class T> Array<T> Array<T>::operator()(int i, int j, std::initializer_list<int> K) const {
-  if(i<0) i += d0;
-  if(j<0) j += d1;
   Array<T> z;
-  if(K.size()==2) z.referToRange(*this, i, j, K.begin()[0], K.begin()[1]);
+  if(K.size()==2) z.referToRange(*this, i, j, {K.begin()[0], K.begin()[1]});
   else if(K.size()==0) z.referToDim(*this, i, j);
   else if(K.size()==1) z.referToDim(*this, i, j, K.begin()[0]);
   else HALT("range list needs 0,1, or 2 entries exactly");
@@ -827,8 +824,9 @@ template<class T> Array<T> Array<T>::copy() const { return Array<T>(*this); }
 /** @brief a sub array of a 1D Array (corresponds to matlab [i:I]); when
   the upper limit I is -1, it is replaced by the max limit (like
   [i:]) */
-template<class T> Array<T> Array<T>::sub(int i, int I) const {
+template<class T> Array<T> Array<T>::sub(std::pair<int, int> _I) const {
   CHECK_EQ(nd, 1, "1D range error ");
+  int i=_I.first, I=_I.second-1;
   Array<T> x;
   if(i<0) i+=d0;
   if(I<0) I+=d0;
@@ -845,8 +843,9 @@ template<class T> Array<T> Array<T>::sub(int i, int I) const {
 /** @brief copies a sub array of a 2D Array (corresponds to matlab [i:I, j:J]);
   when the upper limits I or J are -1, they are replaced by the
   max limit (like [i:, j:]) */
-template<class T> Array<T> Array<T>::sub(int i, int I, int j, int J) const {
+template<class T> Array<T> Array<T>::sub(std::pair<int, int> _I, std::pair<int, int> _J) const {
   CHECK_EQ(nd, 2, "2D range error ");
+  int i=_I.first, I=_I.second-1, j=_J.first, J=_J.second-1;
   Array<T> x;
   if(i<0) i+=d0;
   if(j<0) j+=d1;
@@ -865,8 +864,9 @@ template<class T> Array<T> Array<T>::sub(int i, int I, int j, int J) const {
 /** @brief copies a sub array of a 3D Array (corresponds to matlab [i:I, j:J]);
   when the upper limits I or J are -1, they are replaced by the
   max limit (like [i:, j:]) */
-template<class T> Array<T> Array<T>::sub(int i, int I, int j, int J, int k, int K) const {
+template<class T> Array<T> Array<T>::sub(std::pair<int, int> _I, std::pair<int, int> _J, std::pair<int, int> _K) const {
   CHECK_EQ(nd, 3, "3D range error ");
+  int i=_I.first, I=_I.second-1, j=_J.first, J=_J.second-1, k=_K.first, K=_K.second-1;
   Array<T> x;
   if(i<0) i+=d0;
   if(j<0) j+=d1;
@@ -891,8 +891,9 @@ template<class T> Array<T> Array<T>::sub(int i, int I, int j, int J, int k, int
   runs from i to I (as explained above) while the second index runs
   over the columns explicitly referred to by cols. (col doesn't have
   to be ordered or could also contain some columns multiply) */
-template<class T> Array<T> Array<T>::sub(int i, int I, Array<uint> cols) const {
+template<class T> Array<T> Array<T>::pick(std::pair<int, int> _I, Array<uint> cols) const {
   CHECK_EQ(nd, 2, "2D range error ");
+  int i=_I.first, I=_I.second-1;
   Array<T> x;
   if(i<0) i+=d0;
   if(I<0) I+=d0;
@@ -902,7 +903,7 @@ template<class T> Array<T> Array<T>::sub(int i, int I, Array<uint> cols) const {
   return x;
 }
 
-template<class T> Array<T> Array<T>::sub(Array<uint> elems) const {
+template<class T> Array<T> Array<T>::pick(Array<uint> elems) const {
   Array<T> x;
   if(nd==1) {
     x.resize(elems.N);
@@ -930,7 +931,7 @@ template<class T> Array<T> Array<T>::sub(Array<uint> elems) const {
  */
 template<class T>
 Array<T> Array<T>::row(uint row_index) const {
-  return sub(row_index, row_index, 0, d1 - 1);
+  return sub({row_index, row_index+1},{ 0, d1 - 1+1});
 }
 
 /**
@@ -945,7 +946,7 @@ Array<T> Array<T>::row(uint row_index) const {
  */
 template<class T>
 Array<T> Array<T>::col(uint col_index) const {
-  return sub(0, d0 - 1, col_index, col_index).reshape(d0);
+  return sub({0, d0 - 1+1},{ col_index, col_index+1}).reshape(d0);
 }
 
 /**
@@ -961,7 +962,7 @@ Array<T> Array<T>::col(uint col_index) const {
  */
 template<class T>
 Array<T> Array<T>::rows(uint start_row, uint end_row) const {
-  return sub(start_row, end_row - 1, 0, d1 - 1);
+  return sub({start_row, end_row - 1+1},{ 0, d1 - 1+1});
 }
 
 /**
@@ -977,7 +978,7 @@ Array<T> Array<T>::rows(uint start_row, uint end_row) const {
  */
 template<class T>
 Array<T> Array<T>::cols(uint start_col, uint end_col) const {
-  return sub(0, d0 - 1, start_col, end_col - 1);
+  return sub({0, d0 - 1+1},{ start_col, end_col - 1+1});
 }
 
 /// makes this array a reference to the C buffer
@@ -1013,6 +1014,10 @@ template<class T> Array<T>& Array<T>::operator=(const Array<T>& a) {
   if(memMove) memmove(p, a.p, sizeT*N);
   else for(uint i=0; i<N; i++) p[i]=a.p[i];
   if(special) { delete special; special=NULL; }
+  if constexpr(std::is_same_v<T, double>){
+    if(isSpecial(a)) special_copy(*this, a);
+    if(a.jac) jac = std::make_unique<arr>(*a.jac);
+  }
   return *this;
 }
 
@@ -1047,7 +1052,7 @@ template<class T> Array<T> catCol(const Array<Array<T>*>& X) {
 //          d1+=x->d1;
 //      }
   } else {
-    z.resize(d0, d1);
+    z.resize(d0, d1).setZero();
     d1=0;
     for(const Array<T>* x:  X) { z.setMatrixBlock(*x, 0, d1); d1+=x->nd==2?x->d1:1; }
   }
@@ -1068,7 +1073,7 @@ template<class T> Array<T> catCol(const Array<Array<T>>& X) {
 }
 
 /// set all entries to same value x [default: don't change dimension]
-template<class T> void Array<T>::setUni(const T& x, int d) {
+template<class T> void Array<T>::setConst(const T& x, int d) {
   if(d!=-1) resize(d);
   uint i;
   for(i=0; i<N; i++) elem(i)=x;
@@ -1080,7 +1085,9 @@ template<class T> void Array<T>::setId(int d) {
   CHECK(d!=-1 || (nd==2 && d0==d1), "need squared matrix to set to identity");
   if(d!=-1) resize(d, d);
   setZero();
-  for(uint i=0; i<d0; i++) operator()(i, i)=(T)1;
+  if constexpr(std::is_scalar_v<T>){
+    for(uint i=0; i<d0; i++) operator()(i, i)=(T)1;
+  }else NIY;
 }
 
 template<class T> void Array<T>::setDiag(const T& x, int d) {
@@ -1117,13 +1124,38 @@ template<class T> void Array<T>::setBlockMatrix(const Array<T>& A, const Array<T
 template<class T> void Array<T>::setBlockMatrix(const Array<T>& A, const Array<T>& B) {
   CHECK(A.nd==2 && B.nd==2, "");
   CHECK(A.d1==B.d1, "");
-  resize(A.d0+B.d0, A.d1);
+
+  if constexpr(std::is_same_v<T, double>){
+    if(isSparse(A)){
+      CHECK(isSparse(B), "");
+      sparse().resize(A.d0+B.d0, A.d1, 0);
+    }else{
+      resize(A.d0+B.d0, A.d1).setZero();
+    }
+  }else{
+    resize(A.d0+B.d0, A.d1).setZero();
+  }
+
   setMatrixBlock(A, 0, 0);
   setMatrixBlock(B, A.d0, 0);
 }
 
 /// constructs a vector x=[a, b]
 template<class T> void Array<T>::setBlockVector(const Array<T>& a, const Array<T>& b) {
+
+  if constexpr(std::is_same_v<T, double>){
+    if(a.jac || b.jac) {
+      const Array<T>& A=*a.jac;
+      const Array<T>& B=*b.jac;
+      if(isSparse(A)){
+        CHECK(isSparse(B), "");
+        J().sparse().resize(A.d0+B.d0, A.d1, 0);
+      }else{
+        CHECK(!isSparse(B), "");
+        J().resize(A.d0+B.d0, A.d1).setZero();
+      }
+    }
+  }
   CHECK(a.nd==1 && b.nd==1, "");
   resize(a.N+b.N);
   setVectorBlock(a, 0);   //for(i=0;i<a.N;i++) operator()(i    )=a(i);
@@ -1132,6 +1164,13 @@ template<class T> void Array<T>::setBlockVector(const Array<T>& a, const Array<T
 
 /// write the matrix B into 'this' matrix at location lo0, lo1
 template<class T> void Array<T>::setMatrixBlock(const Array<T>& B, uint lo0, uint lo1) {
+  if constexpr(std::is_same_v<T, double>){
+    if(isSparse(*this)){
+      sparse().add(B, lo0, lo1);
+      return;
+    }
+  }
+
   CHECK(!special && !B.special, "");
   CHECK(B.nd==1 || B.nd==2, "");
   if(B.nd==2) {
@@ -1155,18 +1194,29 @@ template<class T> void Array<T>::setVectorBlock(const Array<T>& B, uint lo) {
   CHECK(nd==1 && B.nd==1 && lo+B.N<=N, "");
   uint i;
   for(i=0; i<B.N; i++) elem(lo+i)=B.elem(i);
+  if constexpr(std::is_same_v<T, double>){
+    if(B.jac) {
+      CHECK(jac && jac->d1==B.jac->d1, "Jacobian needs to be pre-sized");
+      CHECK(!B.jac->jac, "NOT HANDLED YET");
+      jac->setMatrixBlock(*B.jac, lo, 0);
+    }
+  }
 }
 
 /// sorted permutation of length \c n
 template<class T> void Array<T>::setStraightPerm(int n) {
   if(n!=-1) resize(n);
-  for(uint i=0; i<N; i++) elem(i)=(T)i;
+  if constexpr(std::is_arithmetic_v<T>){
+    for(uint i=0; i<N; i++) elem(i)=static_cast<T>(i);
+  } else NIY;
 }
 
 /// reverse sorted permutation of lenth \c N
 template<class T> void Array<T>::setReversePerm(int n) {
   if(n!=-1) resize(n);
-  for(uint i=0; i<N; i++) elem(N-1-i)=(T)i;
+  if constexpr(std::is_arithmetic_v<T>){
+    for(uint i=0; i<N; i++) elem(N-1-i)=static_cast<T>(i);
+  } else NIY;
 }
 
 /// permute all elements randomly
@@ -1190,7 +1240,7 @@ template<class T> Array<T>& Array<T>::setCarray(const T* buffer, uint D0) {
 }
 
 /// 'this' becomes a copy (not reference to!) of the 2D C array
-template<class T> Array<T>& Array<T>::setCarray(const T** buffer, uint D0, uint D1) {
+template<class T> Array<T>& Array<T>::setCarray(T* const* buffer, uint D0, uint D1) {
   resize(D0, D1);
   uint i, j;
   for(i=0; i<d0; i++) {
@@ -1209,8 +1259,9 @@ template<class T> void Array<T>::referTo(const Array<T>& a) {
 }
 
 /// make this array a subarray reference to \c a
-template<class T> void Array<T>::referToRange(const Array<T>& a, int i_lo, int i_up) {
+template<class T> void Array<T>::referToRange(const Array<T>& a, std::pair<int, int> I) {
   CHECK_LE(a.nd, 3, "not implemented yet");
+  int i_lo=I.first, i_up=I.second-1;
   if(i_lo<0) i_lo+=a.d0;
   if(i_up<0) i_up+=a.d0;
   if(i_lo>i_up) { clear(); return; }
@@ -1228,9 +1279,10 @@ template<class T> void Array<T>::referToRange(const Array<T>& a, int i_lo, int i
 }
 
 /// make this array a subarray reference to \c a
-template<class T> void Array<T>::referToRange(const Array<T>& a, int i, int j_lo, int j_up) {
+template<class T> void Array<T>::referToRange(const Array<T>& a, int i, std::pair<int, int> J) {
   CHECK(a.nd>1, "does not make sense");
   CHECK_LE(a.nd, 3, "not implemented yet");
+  int j_lo=J.first, j_up=J.second-1;
   if(i<0) i+=a.d0;
   if(j_lo<0) j_lo+=a.d1;
   if(j_up<0) j_up+=a.d1;
@@ -1247,9 +1299,10 @@ template<class T> void Array<T>::referToRange(const Array<T>& a, int i, int j_lo
 }
 
 /// make this array a subarray reference to \c a
-template<class T> void Array<T>::referToRange(const Array<T>& a, int i, int j, int k_lo, int k_up) {
+template<class T> void Array<T>::referToRange(const Array<T>& a, int i, int j, std::pair<int, int> K) {
   CHECK(a.nd>2, "does not make sense");
   CHECK_LE(a.nd, 3, "not implemented yet");
+  int k_lo=K.first, k_up=K.second-1;
   if(i<0) i+=a.d0;
   if(j<0) j+=a.d1;
   if(k_lo<0) k_lo+=a.d2;
@@ -1279,8 +1332,8 @@ template<class T> void Array<T>::referToDim(const Array<T>& a, int i) {
   } else if(a.nd>3) {
     uint n=a.N/a.d0;
     referTo(a.p+i*n, n);
-    nd=a.nd-1;  d0=a.d1;  d1=a.d2;  d2=a.d[3];
-    if(nd>3) { d=new uint[nd];  memmove(d, a.d+1, nd*sizeof(uint)); }
+    nd=a.nd-1;  d0=a.d1;  d1=a.d2;  d2=a._shape[3];
+    resetShape(a._shape+1);
   }
 }
 
@@ -1302,14 +1355,14 @@ template<class T> void Array<T>::referToDim(const Array<T>& a, uint i, uint j, u
   CHECK(i<a.d0 && j<a.d1 && k<a.d2, "SubDim range error (" <<i <<"<" <<a.d0 <<", " <<j <<"<" <<a.d1 <<", " <<k <<"<" <<a.d2 << ")");
 
   if(a.nd==4) {
-    referTo(&a(i, j, k), a.d[3]);
+    referTo(&a(i, j, k), a._shape[3]);
   } else if(a.nd==5) {
     NIY;
 //    nd=2; d0=a.d[3]; d1=a.d[4]; d2=0; N=d0*d1;
   } else if(a.nd>5) {
     NIY;
 //    nd=a.nd-3; d0=a.d[3]; d1=a.d[4]; d2=a.d[5]; N=a.N/(a.d0*a.d1*a.d2);
-//    resetD();
+//    resetShape();
 //    if(nd>3) { d=new uint[nd];  memmove(d, a.d+3, nd*sizeof(uint)); }
   }
 //  p=a.p+(i*a.N+(j*a.N+(k*a.N/a.d2))/a.d1)/a.d0;
@@ -1322,6 +1375,7 @@ template<class T> void Array<T>::takeOver(Array<T>& a) {
   freeMEM();
   memMove=a.memMove;
   N=a.N; nd=a.nd; d0=a.d0; d1=a.d1; d2=a.d2;
+  resetShape(a._shape);
   p=a.p; M=a.M;
   special=a.special;
 #if 0 //a remains reference on this
@@ -1330,53 +1384,12 @@ template<class T> void Array<T>::takeOver(Array<T>& a) {
 #else //a is cleared
   a.p=NULL;
   a.M=a.N=a.nd=a.d0=a.d1=a.d2=0;
-  if(a.d && a.d!=&a.d0) { delete[] a.d; a.d=NULL; }
+  a.resetShape();
   a.special=0;
   a.isReference=false;
 #endif
 }
 
-/** @brief return a `dim'-dimensional grid with `steps' intervals
-  filling the range [lo, hi] in each dimension. Note: returned array is
-  `flat', rather than grid-shaped. */
-template<class T> Array<T>& Array<T>::setGrid(uint dim, T lo, T hi, uint steps) {
-  CHECK(steps, "steps needs to be >0");
-  uint i, j, k;
-  if(dim==1) {
-    resize(steps+1, 1);
-    for(i=0; i<d0; i++) elem(i)=lo+(hi-lo)*i/steps;
-    return *this;
-  }
-  if(dim==2) {
-    resize(steps+1, steps+1, 2);
-    for(i=0; i<d0; i++) for(j=0; j<d1; j++) {
-        operator()(i, j, 0)=lo+(hi-lo)*i/steps;
-        operator()(i, j, 1)=lo+(hi-lo)*j/steps;
-      }
-    reshape(d0*d1, 2);
-    return *this;
-  }
-  if(dim==3) {
-    resize(uintA{steps+1, steps+1, steps+1, 3});
-    T dx = (hi-lo)/steps;
-    for(i=0; i<d0; i++) for(j=0; j<d1; j++) {
-        T* p = &elem(uintA{i, j, 0, 0});
-        for(k=0; k<d2; k++) {
-          *(p++) = lo+dx*i;
-          *(p++) = lo+dx*j;
-          *(p++) = lo+dx*k;
-//        elem(uintA{i, j, k, 0}) = lo+dx*i;
-//        elem(uintA{i, j, k, 1}) = lo+dx*j;
-//        elem(uintA{i, j, k, 2}) = lo+dx*k;
-        }
-      }
-    reshape(d0*d1*d2, 3);
-    return *this;
-  }
-  NIY;
-  return *this;
-}
-
 template<class T> T rai::Array<T>::median_nonConst() {
   CHECK_GE(N, 1, "");
   std::nth_element(p, p+N/2, p+N);
@@ -1498,7 +1511,7 @@ template<class T> void Array<T>::permuteInv(const Array<uint>& permutation) {
 template<class T> void Array<T>::permuteRowsInv(const Array<uint>& permutation) {
   CHECK_LE(permutation.N, d0, "array smaller than permutation ("<<N<<"<"<<permutation.N<<")");
   Array<T> b=(*this);
-  for(uint i=0; i<d0; i++) operator[](permutation(i))()=b[i];
+  for(uint i=0; i<d0; i++) operator[](permutation(i))=b[i];
 }
 
 /// randomly permute all entries of 'this'
@@ -1531,9 +1544,28 @@ template<class T> void Array<T>::shift(int offset, bool wrapAround) {
 
 //==================================================================================
 
+/// return fraction of non-zeros in the array
+template<class T> double Array<T>::sparsity() {
+  uint i, m=0;
+  for(i=0; i<N; i++) if(elem(i)) m++;
+  return ((double)m)/N;
+}
+
+//==================================================================================
+
 /** @brief prototype for operator<<, writes the array by separating elements with ELEMSEP, separating rows with LINESEP, using BRACKETS[0] and BRACKETS[1] to brace the data, optionally writs a dimensionality tag before the data (see below), and optinally in binary format */
 template<class T> void Array<T>::write(std::ostream& os, const char* ELEMSEP, const char* LINESEP, const char* BRACKETS, bool dimTag, bool binary) const {
-  CHECK(!special, "");
+
+  if constexpr(std::is_same_v<T, double>){
+    if(special){
+      special_write(os, *this);
+      if(jac) os <<" -- JACOBIAN:\n" <<*jac <<endl;
+      return;
+    }
+  }else{
+    CHECK(!special, "");
+  }
+
   CHECK(!binary || memMove, "binary write works only for memMoveable data");
   uint i, j, k;
   if(!ELEMSEP) ELEMSEP=arrayElemsep;
@@ -1572,15 +1604,19 @@ template<class T> void Array<T>::write(std::ostream& os, const char* ELEMSEP, co
         }
       }
     if(nd>3) {
-      CHECK(d && d!=&d0, "");
+      CHECK(_shape && _shape!=&d0, "");
       for(i=0; i<N; i++) {
-        if(i && !(i%d[nd-1])) os <<LINESEP;
-        if(nd>1 && !(i%(d[nd-2]*d[nd-1]))) os <<LINESEP;
+        if(i && !(i%_shape[nd-1])) os <<LINESEP;
+        if(nd>1 && !(i%(_shape[nd-2]*_shape[nd-1]))) os <<LINESEP;
         os <<(i?ELEMSEP:"") <<elem(i);
       }
     }
     if(BRACKETS[1]) os <<BRACKETS[1];
   }
+
+  if constexpr(std::is_same_v<T, double>){
+    if(jac) os <<" -- JACOBIAN:\n" <<*jac <<endl;
+  }
 }
 
 /** @brief prototype for operator>>, if there is a dimensionality tag: fast reading of ascii (if there is brackets[]) or binary (if there is \\0\\0 brackets) data; otherwise slow ascii read */
@@ -1607,7 +1643,9 @@ template<class T> Array<T>& Array<T>::read(std::istream& is) {
     } else { //fast ascii read
       for(uint i=0; i<N; i++) {
         if(is.fail()) PARSERR("could not read " <<i <<"-th element of an array");
-        is >>p[i];
+        if constexpr(!std::is_pointer_v<T>){
+          is >>p[i];
+        } else NIY;
       }
     }
     if(expectBracket) {
@@ -1632,7 +1670,9 @@ template<class T> Array<T>& Array<T>::read(std::istream& is) {
         continue;
       }
       if(c!=',') is.putback(c);
-      is >>x;
+      if constexpr(!std::is_pointer_v<T>){
+        is >>x;
+      }else NIY;
       if(!is.good()) {
         if(!expectBracket) is.clear(); //ok
         else PARSERR("failed reading ending bracket ]");
@@ -1825,6 +1865,7 @@ template<class T> void writeConsecutiveConstant(std::ostream& os, const Array<T>
 
 /// contatenation of two arrays
 template<class T> Array<T> operator, (const Array<T>& y, const Array<T>& z) { Array<T> x(y); x.append(z); return x; }
+
 /// calls Array<T>::read
 template<class T> std::istream& operator>>(std::istream& is, Array<T>& x) { x.read(is); return is; }
 
@@ -1869,4 +1910,114 @@ template<class T> bool operator<(const Array<T>& v, const Array<T>& w) {
   return v.N<w.N;
 }
 
+//core for matrix-matrix (elem-wise) update
+#define UpdateOperator_MM( op ) \
+if constexpr(std::is_same_v<T, double>){ \
+      if(isNoArr(x)){ return; } \
+      if(isSparseMatrix(x) && isSparseMatrix(y)){ x.sparse() op y.sparse(); return; }  \
+      if(isRowShifted(x) && isRowShifted(y)){ x.rowShifted() op y.rowShifted(); return; }  \
+} \
+    CHECK(!x.special, "");  \
+    CHECK(!y.special, "");  \
+    CHECK_EQ(x.N, y.N, "update operator on different array dimensions (" <<x.N <<", " <<y.N <<")"); \
+    T *xp=x.p, *xstop=xp+x.N; \
+    const T *yp=y.p; \
+    for(; xp!=xstop; xp++, yp++) *xp op *yp;
+
+//core for matrix-scalar update
+#define UpdateOperator_MS( op ) \
+if constexpr(std::is_same_v<T, double>){ \
+      if(isNoArr(x)){ return; } \
+      if(isSparseMatrix(x)){ x.sparse() op y; return; }  \
+      if(isRowShifted(x)){ x.rowShifted() op y; return; }  \
+} \
+    CHECK(!x.special, "");  \
+    T *xp=x.p, *xstop=xp+x.N; \
+    for(; xp!=xstop; xp++) *xp op y;
+
+
+template<class T> void operator+=(Array<T>& x, const Array<T>& y) {
+  UpdateOperator_MM(+=);
+  if constexpr(std::is_same_v<T, double>){
+    if(y.jac) {
+      if(x.jac) *x.jac += *y.jac;
+      else x.J() = *y.jac;
+    }
+  }
+}
+template<class T> void operator+=(Array<T>& x, const T& y) {
+  UpdateOperator_MS(+=);
+}
+template<class T> void operator-=(Array<T>& x, const Array<T>& y) {
+  UpdateOperator_MM(-=);
+  if constexpr(std::is_same_v<T, double>){
+    if(y.jac) {
+      if(x.jac) *x.jac -= *y.jac;
+      else x.J() = -(*y.jac);
+    }
+  }
+}
+template<class T> void operator-=(Array<T>& x, const T& y) {
+  UpdateOperator_MS(-=);
+}
+template<class T> void operator*=(Array<T>& x, const T& y) {
+  if constexpr(std::is_same_v<T, double>){
+    if(x.jac) *x.jac *= y;
+  }
+  UpdateOperator_MS(*=);
+}
+template<class T> void operator/=(Array<T>& x, const T& y) {
+  if constexpr(std::is_same_v<T, double>){
+    if(x.jac) *x.jac /= y;
+  }
+  UpdateOperator_MS(/=);
+}
+
+#undef UpdateOperator_MM
+#undef UpdateOperator_MS
+
+//===========================================================================
+
+/** @brief return a `dim'-dimensional grid with `steps' intervals
+  filling the range [lo, hi] in each dimension. Note: returned array is
+  `flat', rather than grid-shaped. */
+template<class T> Array<T> grid(uint dim, T lo, T hi, uint steps) {
+  Array<T> x;
+  CHECK(steps, "steps needs to be >0");
+  uint i, j, k;
+  if(dim==1) {
+    x.resize(steps+1, 1);
+    for(i=0; i<x.d0; i++) x.elem(i)=lo+(hi-lo)*i/steps;
+    return x;
+  }
+  if(dim==2) {
+    x.resize(steps+1, steps+1, 2);
+    for(i=0; i<x.d0; i++) for(j=0; j<x.d1; j++) {
+        x(i, j, 0)=lo+(hi-lo)*i/steps;
+        x(i, j, 1)=lo+(hi-lo)*j/steps;
+      }
+    x.reshape(x.d0*x.d1, 2);
+    return x;
+  }
+  if(dim==3) {
+    x.resize(uintA{steps+1, steps+1, steps+1, 3});
+    T dx = (hi-lo)/steps;
+    for(i=0; i<x.d0; i++) for(j=0; j<x.d1; j++) {
+        T* p = &x.elem(uintA{i, j, 0, 0});
+        for(k=0; k<x.d2; k++) {
+          *(p++) = lo+dx*i;
+          *(p++) = lo+dx*j;
+          *(p++) = lo+dx*k;
+          //        elem(uintA{i, j, k, 0}) = lo+dx*i;
+          //        elem(uintA{i, j, k, 1}) = lo+dx*j;
+          //        elem(uintA{i, j, k, 2}) = lo+dx*k;
+        }
+      }
+    x.reshape(x.d0*x.d1*x.d2, 3);
+    return x;
+  }
+  NIY;
+  return x;
+}
+
 } //namespace