foscat 3.0.13__tar.gz → 3.0.16__tar.gz

{foscat-3.0.13 → foscat-3.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: foscat
-Version: 3.0.13
+Version: 3.0.16
 Summary: Generate synthetic Healpix or 2D data using Cross Scattering Transform
 Home-page: https://github.com/jmdelouis/FOSCAT
 Author: Jean-Marc DELOUIS
@@ -9,6 +9,12 @@ Maintainer: Theo Foulquier
 Maintainer-email: theo.foulquier@ifremer.fr
 License: MIT
 Keywords: Scattering transform,Component separation,denoising
+Requires-Dist: imageio
+Requires-Dist: imagecodecs
+Requires-Dist: matplotlib
+Requires-Dist: numpy
+Requires-Dist: tensorflow
+Requires-Dist: healpy
 
 Utilize the Cross Scattering Transform (described in https://arxiv.org/abs/2207.12527) to synthesize Healpix or 2D data that is suitable for component separation purposes, such as denoising. 
  A demo package for this process can be found at https://github.com/jmdelouis/FOSCAT_DEMO. 

{foscat-3.0.13 → foscat-3.0.16}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='foscat',
-    version='3.0.13',
+    version='3.0.16',
     description='Generate synthetic Healpix or 2D data using Cross Scattering Transform' ,
     long_description='Utilize the Cross Scattering Transform (described in https://arxiv.org/abs/2207.12527) to synthesize Healpix or 2D data that is suitable for component separation purposes, such as denoising. \n A demo package for this process can be found at https://github.com/jmdelouis/FOSCAT_DEMO. \n Complete doc can be found at https://foscat-documentation.readthedocs.io/en/latest/index.html. \n\n List of developers : J.-M. Delouis, T. Foulquier, L. Mousset, T. Odaka, F. Paul, E. Allys ' ,
     license='MIT',

{foscat-3.0.13 → foscat-3.0.16}/src/foscat/FoCUS.py

@@ -4,6 +4,7 @@ import os, sys
 import foscat.backend as bk
 from scipy.interpolate import griddata
 
+
 TMPFILE_VERSION='V2_6'
 
 class FoCUS:
@@ -27,9 +28,11 @@ class FoCUS:
                  JmaxDelta=0,
                  DODIV=False,
                  InitWave=None,
+                 silent=False,
                  mpi_size=1,
                  mpi_rank=0):
 
+        self.__version__ = '3.0.16'
         # P00 coeff for normalization for scat_cov
         self.TMPFILE_VERSION=TMPFILE_VERSION
         self.P1_dic = None
@@ -42,20 +45,25 @@ class FoCUS:
         self.mpi_size=mpi_size
         self.mpi_rank=mpi_rank
         self.return_data=return_data
-        
-        print('================================================')
-        print('          START FOSCAT CONFIGURATION')
-        print('================================================')
-        sys.stdout.flush()
+        self.silent=silent
+
+        if not self.silent:
+            print('================================================')
+            print('          START FOSCAT CONFIGURATION')
+            print('================================================')
+            sys.stdout.flush()
 
         self.TEMPLATE_PATH=TEMPLATE_PATH
         if os.path.exists(self.TEMPLATE_PATH)==False:
-            print('The directory %s to store temporary information for FoCUS does not exist: Try to create it'%(self.TEMPLATE_PATH))
+            if not self.silent:
+                print('The directory %s to store temporary information for FoCUS does not exist: Try to create it'%(self.TEMPLATE_PATH))
             try:
                 os.system('mkdir -p %s'%(self.TEMPLATE_PATH))
-                print('The directory %s is created')
+                if not self.silent:
+                    print('The directory %s is created')
             except:
-                print('Impossible to create the directory %s'%(self.TEMPLATE_PATH))
+                if not self.silent:
+                    print('Impossible to create the directory %s'%(self.TEMPLATE_PATH))
                 exit(0)
                 
         self.number_of_loss=0
@@ -65,13 +73,15 @@ class FoCUS:
         self.padding=padding
             
         if OSTEP!=0:
-            print('OPTION option is deprecated after version 2.0.6. Please use Jmax option')
+            if not self.silent:
+                print('OPTION option is deprecated after version 2.0.6. Please use Jmax option')
             JmaxDelta=OSTEP
         else:
             OSTEP=JmaxDelta
             
         if JmaxDelta<-1:
-            print('Warning : Jmax can not be smaller than -1')
+            if not self.silent:
+                print('Warning : Jmax can not be smaller than -1')
             exit(0)
             
         self.OSTEP=JmaxDelta
@@ -93,7 +103,8 @@ class FoCUS:
         self.backend=bk.foscat_backend(BACKEND,
                                        all_type=all_type,
                                        mpi_rank=mpi_rank,
-                                       gpupos=gpupos)
+                                       gpupos=gpupos,
+                                       silent=self.silent)
 
         self.all_bk_type=self.backend.all_bk_type
         self.all_cbk_type=self.backend.all_cbk_type
@@ -103,14 +114,15 @@ class FoCUS:
         
         self.gpupos=(gpupos+mpi_rank)%self.backend.ngpu
 
-        print('============================================================')
-        print('==                                                        ==')
-        print('==                                                        ==')
-        print('==     RUN ON GPU Rank %d : %s                          =='%(mpi_rank,self.gpulist[self.gpupos%self.ngpu]))
-        print('==                                                        ==')
-        print('==                                                        ==')
-        print('============================================================')
-        sys.stdout.flush()
+        if not self.silent:
+            print('============================================================')
+            print('==                                                        ==')
+            print('==                                                        ==')
+            print('==     RUN ON GPU Rank %d : %s                          =='%(mpi_rank,self.gpulist[self.gpupos%self.ngpu]))
+            print('==                                                        ==')
+            print('==                                                        ==')
+            print('============================================================')
+            sys.stdout.flush()
 
         l_NORIENT=NORIENT
         if DODIV:
@@ -212,7 +224,8 @@ class FoCUS:
 
             for i in range(1,6):
                 lout=(2**i)
-                print('Init Wave ',lout)
+                if not self.silent:
+                    print('Init Wave ',lout)
                 
                 if self.InitWave is None:
                     wr,wi,ws,widx=self.init_index(lout)
@@ -322,7 +335,8 @@ class FoCUS:
             indices=np.zeros([12*nside*nside,l_kernel,2],dtype='int')
             for k in range(12*nside*nside):
                 if k%(nside*nside)==0:
-                    print('Pre-compute nside=%6d %.2f%%'%(nside,100*k/(12*nside*nside)))
+                    if not self.silent:
+                        print('Pre-compute nside=%6d %.2f%%'%(nside,100*k/(12*nside*nside)))
         
                 rot=[po[k]/np.pi*180.0,90+(-to[k])/np.pi*180.0]
                 r=hp.Rotator(rot=rot).get_inverse()            
@@ -334,10 +348,12 @@ class FoCUS:
             if transpose:
                 indices[:,:,1]=indices[:,:,1]//4
                 np.save('%s/FOSCAT_%s_W%d_%d_%d_CNN_Transpose.npy'%(self.TEMPLATE_PATH,TMPFILE_VERSION,l_kernel,self.NORIENT,nside),indices)
-                print('Write %s/FOSCAT_%s_W%d_%d_%d_CNN_Transpose.npy'%(self.TEMPLATE_PATH,TMPFILE_VERSION,l_kernel,self.NORIENT,nside))
+                if not self.silent:
+                    print('Write %s/FOSCAT_%s_W%d_%d_%d_CNN_Transpose.npy'%(self.TEMPLATE_PATH,TMPFILE_VERSION,l_kernel,self.NORIENT,nside))
             else:
                 np.save('%s/FOSCAT_%s_W%d_%d_%d_CNNnpy'%(self.TEMPLATE_PATH,TMPFILE_VERSION,l_kernel,self.NORIENT,nside),indices)
-                print('Write %s/FOSCAT_%s_W%d_%d_%d_CNN.npy'%(self.TEMPLATE_PATH,TMPFILE_VERSION,l_kernel,self.NORIENT,nside))
+                if not self.silent:
+                    print('Write %s/FOSCAT_%s_W%d_%d_%d_CNN.npy'%(self.TEMPLATE_PATH,TMPFILE_VERSION,l_kernel,self.NORIENT,nside))
 
                 
         if transpose:
@@ -355,7 +371,8 @@ class FoCUS:
         l_kernel=self.KERNELSZ*self.KERNELSZ
             
         if im.shape[1]!=ww.shape[1]:
-            print('Weights channels should be equal to the input image channels')
+            if not self.silent:
+                print('Weights channels should be equal to the input image channels')
             return -1
         
         if self.ww_CNN_Transpose[nside] is None:
@@ -374,7 +391,8 @@ class FoCUS:
         l_kernel=self.KERNELSZ*self.KERNELSZ
             
         if im.shape[1]!=ww.shape[1]:
-            print('Weights channels should be equal to the input image channels')
+            if not self.silent:
+                print('Weights channels should be equal to the input image channels')
             return -1
         
         if self.ww_CNN[nside] is None:
@@ -421,7 +439,8 @@ class FoCUS:
         if self.use_2D:
             ishape=list(im.shape)
             if len(ishape)<axis+2:
-                print('Use of 2D scat with data that has less than 2D')
+                if not self.silent:
+                    print('Use of 2D scat with data that has less than 2D')
                 exit(0)
                 
             npix=im.shape[axis]
@@ -481,7 +500,8 @@ class FoCUS:
         if self.use_2D:
             ishape=list(im.shape)
             if len(ishape)<axis+2:
-                print('Use of 2D scat with data that has less than 2D')
+                if not self.silent:
+                    print('Use of 2D scat with data that has less than 2D')
                 exit(0)
                 
             if nouty is None:
@@ -523,7 +543,8 @@ class FoCUS:
             lout=int(np.sqrt(im.shape[axis]//12))
             
             if self.pix_interp_val[lout][nout] is None:
-                print('compute lout nout',lout,nout)
+                if not self.silent:
+                    print('compute lout nout',lout,nout)
                 th,ph=hp.pix2ang(nout,np.arange(12*nout**2,dtype='int'),nest=True)
                 p, w = hp.get_interp_weights(lout,th,ph,nest=True)
                 del th
@@ -880,7 +901,8 @@ class FoCUS:
                     
                 for k in range(12*nside*nside):
                     if k%(nside*nside)==0:
-                        print('Pre-compute nside=%6d %.2f%%'%(nside,100*k/(12*nside*nside)))
+                        if not self.silent:
+                            print('Pre-compute nside=%6d %.2f%%'%(nside,100*k/(12*nside*nside)))
                     if nside>scale*2:
                         lidx=hp.get_all_neighbours(nside//scale,th[k//(scale*scale)],ph[k//(scale*scale)],nest=True)
                         lidx=np.concatenate([lidx,np.array([(k//(scale*scale))])],0)
@@ -932,7 +954,8 @@ class FoCUS:
                 wav=w.flatten()
                 wwav=wwav.flatten()
                 
-                print('Write FOSCAT_%s_W%d_%d_%d_PIDX.npy'%(TMPFILE_VERSION,self.KERNELSZ**2,self.NORIENT,nside))
+                if not self.silent:
+                    print('Write FOSCAT_%s_W%d_%d_%d_PIDX.npy'%(TMPFILE_VERSION,self.KERNELSZ**2,self.NORIENT,nside))
                 np.save('%s/FOSCAT_%s_W%d_%d_%d_PIDX.npy'%(self.TEMPLATE_PATH,TMPFILE_VERSION,self.KERNELSZ**2,self.NORIENT,nside),indice)
                 np.save('%s/FOSCAT_%s_W%d_%d_%d_WAVE.npy'%(self.TEMPLATE_PATH,TMPFILE_VERSION,self.KERNELSZ**2,self.NORIENT,nside),wav)
                 np.save('%s/FOSCAT_%s_W%d_%d_%d_PIDX2.npy'%(self.TEMPLATE_PATH,TMPFILE_VERSION,self.KERNELSZ**2,self.NORIENT,nside),indice2)
@@ -946,7 +969,8 @@ class FoCUS:
                         self.comp_idx_w25(nside)
                 else:
                     if self.rank==0:
-                        print('Only 3x3 and 5x5 kernel have been developped for Healpix and you ask for %dx%d'%(KERNELSZ,KERNELSZ))
+                        if not self.silent:
+                            print('Only 3x3 and 5x5 kernel have been developped for Healpix and you ask for %dx%d'%(KERNELSZ,KERNELSZ))
                         exit(0)
 
         self.barrier()  
@@ -1230,7 +1254,8 @@ class FoCUS:
             
             ishape=list(in_image.shape)
             if len(ishape)<axis+2:
-                print('Use of 2D scat with data that has less than 2D')
+                if not self.silent:
+                    print('Use of 2D scat with data that has less than 2D')
                 exit(0)
                 
             npix=ishape[axis]
@@ -1354,7 +1379,8 @@ class FoCUS:
             
             ishape=list(in_image.shape)
             if len(ishape)<axis+2:
-                print('Use of 2D scat with data that has less than 2D')
+                if not self.silent:
+                    print('Use of 2D scat with data that has less than 2D')
                 exit(0)
                 
             npix=ishape[axis]

{foscat-3.0.13 → foscat-3.0.16}/src/foscat/GCNN.py

@@ -11,6 +11,7 @@ class GCNN:
                  nscale=1,
                  chanlist=[],
                  in_nside=1,
+                 n_chan_out=1,
                  nbatch=1,
                  SEED=1234,
                  filename=None):
@@ -27,12 +28,14 @@ class GCNN:
             self.chanlist=outlist[0]
             self.in_nside=outlist[4] 
             self.nbatch=outlist[1]
+            self.n_chan_out=outlist[8]
         
             self.x=self.scat_operator.backend.bk_cast(outlist[6])
         else:
             self.nscale=nscale
             self.nbatch=nbatch
             self.npar=nparam
+            self.n_chan_out=n_chan_out
             self.scat_operator=scat_operator
         
             if len(chanlist)!=nscale+1:
@@ -56,7 +59,9 @@ class GCNN:
                  self.in_nside, \
                  self.nscale, \
                  self.get_weights().numpy(), \
-                 self.all_type]
+                 self.all_type, \
+                 self.n_chan_out]
+        
         myout=open("%s.pkl"%(filename),"wb")
         pickle.dump(outlist,myout)
         myout.close()
@@ -66,7 +71,7 @@ class GCNN:
         for i in range(self.nscale):
             totnchan=totnchan+self.chanlist[i]*self.chanlist[i+1]
         return self.npar*12*self.in_nside**2*self.chanlist[0] \
-            +totnchan*self.KERNELSZ*self.KERNELSZ+self.chanlist[self.nscale]
+            +totnchan*self.KERNELSZ*self.KERNELSZ+self.chanlist[self.nscale]*self.n_chan_out
 
     def set_weights(self,x):
         self.x=x
@@ -93,8 +98,8 @@ class GCNN:
             im=self.scat_operator.healpix_layer_transpose(im,ww)
             im=self.scat_operator.backend.bk_relu(im)
             
-        ww=self.scat_operator.backend.bk_reshape(x[nn:],[self.chanlist[self.nscale],1])
+        ww=self.scat_operator.backend.bk_reshape(x[nn:],[self.chanlist[self.nscale],self.n_chan_out])
         im=self.scat_operator.backend.bk_matmul(im,ww)
         
-        return self.scat_operator.backend.bk_reshape(im,[im.shape[0]])
+        return im
      

{foscat-3.0.13 → foscat-3.0.16}/src/foscat/backend.py

@@ -3,7 +3,7 @@ import numpy as np
 
 class foscat_backend:
     
-    def __init__(self,name,mpi_rank=0,all_type='float64',gpupos=0):
+    def __init__(self,name,mpi_rank=0,all_type='float64',gpupos=0,silent=False):
         
         self.TENSORFLOW=1
         self.TORCH=2
@@ -71,7 +71,7 @@ class foscat_backend:
         #===========================================================================
         # INIT 
         if mpi_rank==0:
-            if self.BACKEND==self.TENSORFLOW:
+            if self.BACKEND==self.TENSORFLOW and silent==False:
                 print("Num GPUs Available: ", len(self.backend.config.experimental.list_physical_devices('GPU')))
             sys.stdout.flush()
         
@@ -264,7 +264,14 @@ class foscat_backend:
         if self.BACKEND==self.TORCH:
             return x
         if self.BACKEND==self.NUMPY:
-            return x
+            res=np.zeros([x.shape[0],x.shape[1],x.shape[2],w.shape[3]],dtype=x.dtype)
+            for k in range(w.shape[2]):
+                for l in range(w.shape[3]):
+                    for j in range(res.shape[0]):
+                        tmp=self.scipy.signal.convolve2d(x[j,:,:,k],w[:,:,k,l], mode='same', boundary='fill', fillvalue=0.0)
+                        res[j,:,:,l]+=tmp
+                        del tmp
+            return res
 
     def bk_threshold(self,x,threshold,greater=True):
 
@@ -297,9 +304,14 @@ class foscat_backend:
         if self.BACKEND==self.TENSORFLOW:
             return self.backend.nn.conv1d(x,w, stride=[1,1,1], padding='SAME')
         if self.BACKEND==self.TORCH:
+            # Torch not yet done !!!
             return self.backend.nn.conv1d(x,w, stride=1, padding='SAME')
         if self.BACKEND==self.NUMPY:
-            return self.backend.nn.conv1d(x,w, stride=1, padding='SAME')
+            res=np.zeros([x.shape[0],x.shape[1],w.shape[1]],dtype=x.dtype)
+            for k in range(w.shape[1]):
+                for l in range(w.shape[2]):
+                    res[:,:,l]+=self.scipy.ndimage.convolve1d(x[:,:,k],w[:,k,l],axis=1,mode='constant',cval=0.0)
+            return res
 
     def bk_flattenR(self,x):
         if self.BACKEND==self.TENSORFLOW or self.BACKEND==self.TORCH:

foscat-3.0.16/src/foscat/backend_tens.py

@@ -0,0 +1,49 @@
+import tensorflow as tf
+import numpy as np
+
+class foscat_backend_tens:
+    
+    def __init__(self,backend):
+        
+        self.bk=backend
+    # ---------------------------------------------−---------
+    
+    @tf.function
+    def loss(self,x,batch,loss_function):
+
+        operation=loss_function.scat_operator
+
+        nx=1
+        if len(x.shape)>1:
+            nx=x.shape[0]
+            
+        with tf.device(operation.gpulist[(operation.gpupos+self.curr_gpu)%operation.ngpu]):
+            print('%s Run [PROC=%04d] on GPU %s'%(loss_function.name,self.mpi_rank,
+                                      operation.gpulist[(operation.gpupos+self.curr_gpu)%operation.ngpu]))
+            sys.stdout.flush()
+
+            l_x=x
+            """
+            if nx>1:
+                l_x={}
+            for i in range(nx):
+            """
+            
+            if nx==1:    
+                ndata=x.shape[0]
+            else:
+                ndata=x.shape[0]*x.shape[1]
+                
+            if self.KEEP_TRACK is not None:
+                l,linfo=loss_function.eval(l_x,batch,return_all=True)
+            else:
+                l=loss_function.eval(l_x,batch)
+                
+            g=tf.gradients(l,x)[0]
+            g=self.backend.check_dense(g,ndata)
+            self.curr_gpu=self.curr_gpu+1   
+            
+        if self.KEEP_TRACK is not None:
+            return l,g,linfo
+        else:
+            return l,g

{foscat-3.0.13 → foscat-3.0.16}/src/foscat/scat_cov.py

@@ -1493,11 +1493,15 @@ class funct(FOC.FoCUS):
             scat_cov(sS0, sP00, sC01, sC11, s1=sS1,c10=sC10,backend=self.backend)
 
     # compute local direction to make the statistical analysis more efficient
-    def stat_cfft(self,im,upscale=False,smooth_scale=0):
+    def stat_cfft(self,im,image2=None,upscale=False,smooth_scale=0):
         tmp=im
+        if image2 is not None:
+            tmpi2=image2
         if upscale:
             l_nside=int(np.sqrt(tmp.shape[1]//12))
             tmp=self.up_grade(tmp,l_nside*2,axis=1)
+            if image2 is not None:
+                tmpi2=self.up_grade(tmpi2,l_nside*2,axis=1)
             
         l_nside=int(np.sqrt(tmp.shape[1]//12))
         nscale=int(np.log(l_nside)/np.log(2))
@@ -1505,6 +1509,11 @@ class funct(FOC.FoCUS):
         cmat2={}
         for k in range(nscale):
             sim=self.backend.bk_abs(self.convol(tmp,axis=1))
+            if image2 is not None:
+                sim=self.backend.bk_real(self.backend.bk_L1(self.convol(tmp,axis=1)*self.backend.bk_conjugate(self.convol(tmpi2,axis=1))))
+            else:
+                sim=self.backend.bk_abs(self.convol(tmp,axis=1))
+                
             cc=self.backend.bk_reduce_mean(sim[:,:,0]-sim[:,:,2],0)
             ss=self.backend.bk_reduce_mean(sim[:,:,1]-sim[:,:,3],0)
             for m in range(smooth_scale):
@@ -1582,6 +1591,8 @@ class funct(FOC.FoCUS):
         
             if k<l_nside-1:
                 tmp=self.ud_grade_2(tmp,axis=1)
+                if image2 is not None:
+                    tmpi2=self.ud_grade_2(tmpi2,axis=1)
         return cmat,cmat2
     
     def eval(self, image1, image2=None, mask=None, norm=None, Auto=True, calc_var=False,cmat=None,cmat2=None):
@@ -2410,13 +2421,13 @@ class funct(FOC.FoCUS):
     """
     @tf.function
     """
-    def eval_comp_fast(self, image1, image2=None,mask=None,norm=None, Auto=True):
+    def eval_comp_fast(self, image1, image2=None,mask=None,norm=None, Auto=True,cmat=None,cmat2=None):
 
-        res=self.eval(image1, image2=image2,mask=mask,Auto=Auto)
+        res=self.eval(image1, image2=image2,mask=mask,Auto=Auto,cmat=cmat,cmat2=cmat2)
         return res.S0,res.P00,res.S1,res.C01,res.C11,res.C10
 
-    def eval_fast(self, image1, image2=None,mask=None,norm=None, Auto=True):
-        s0,p0,s1,c01,c11,c10=self.eval_comp_fast(image1, image2=image2,mask=mask,Auto=Auto)
+    def eval_fast(self, image1, image2=None,mask=None,norm=None, Auto=True,cmat=None,cmat2=None):
+        s0,p0,s1,c01,c11,c10=self.eval_comp_fast(image1, image2=image2,mask=mask,Auto=Auto,cmat=cmat,cmat2=cmat2)
         return scat_cov(s0, p0,  c01, c11, s1=s1,c10=c10,backend=self.backend)
         
         

{foscat-3.0.13 → foscat-3.0.16}/src/foscat.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: foscat
-Version: 3.0.13
+Version: 3.0.16
 Summary: Generate synthetic Healpix or 2D data using Cross Scattering Transform
 Home-page: https://github.com/jmdelouis/FOSCAT
 Author: Jean-Marc DELOUIS
@@ -9,6 +9,12 @@ Maintainer: Theo Foulquier
 Maintainer-email: theo.foulquier@ifremer.fr
 License: MIT
 Keywords: Scattering transform,Component separation,denoising
+Requires-Dist: imageio
+Requires-Dist: imagecodecs
+Requires-Dist: matplotlib
+Requires-Dist: numpy
+Requires-Dist: tensorflow
+Requires-Dist: healpy
 
 Utilize the Cross Scattering Transform (described in https://arxiv.org/abs/2207.12527) to synthesize Healpix or 2D data that is suitable for component separation purposes, such as denoising. 
  A demo package for this process can be found at https://github.com/jmdelouis/FOSCAT_DEMO. 

{foscat-3.0.13 → foscat-3.0.16}/src/foscat.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ src/foscat/Spline1D.py
 src/foscat/Synthesis.py
 src/foscat/__init__.py
 src/foscat/backend.py
+src/foscat/backend_tens.py
 src/foscat/loss_backend_tens.py
 src/foscat/loss_backend_torch.py
 src/foscat/scat.py