dsipts 1.1.12__py3-none-any.whl → 1.1.15__py3-none-any.whl

dsipts/__init__.py

@@ -26,6 +26,9 @@ from .models.TTM import TTM
 from .models.Samformer import Samformer
 from .models.Duet import Duet
 from .models.Simple import Simple
+from .models.TimesNet import TimesNet
+from .models.TimeKAN import TimeKAN
+from .version import __version__
 try:
     import lightning.pytorch as pl
     from .models.base_v2 import Base
@@ -44,5 +47,5 @@ __all__ = [
     "RNN", "LinearTS", "Persistent", "D3VAE", "DilatedConv", "TFT",
     "Informer", "VVA", "VQVAEA", "CrossFormer", "Autoformer", "PatchTST",
     "Diffusion", "DilatedConvED", "TIDE", "ITransformer", "TimeXER",
-    "TTM", "Samformer", "Duet", "Base", "Simple"
+    "TTM", "Samformer", "Duet", "Base", "Simple","TimesNet","TimeKAN"
 ]

dsipts/data_structure/data_structure.py

@@ -6,7 +6,7 @@ from sklearn.preprocessing import LabelEncoder, OrdinalEncoder
 from sklearn.preprocessing import * 
 from torch.utils.data import DataLoader
 from .utils import extend_time_df,MetricsCallback, MyDataset, ActionEnum,beauty_string
-
+from torch.utils.data.sampler import WeightedRandomSampler
 try:
 
     #new version of lightning
@@ -249,7 +249,8 @@ class TimeSeries():
                     check_past:bool=True,
                     group:Union[None,str]=None,
                     check_holes_and_duplicates:bool=True,
-                    silly_model:bool=False)->None:
+                    silly_model:bool=False,
+                    sampler_weights:Union[None,str]=None)->None:
         """ This is a crucial point in the data structure. We expect here to have a dataset with time as timestamp.
             There are some checks:
                 1- the duplicates will tbe removed taking the first instance
@@ -270,6 +271,7 @@ class TimeSeries():
             group (str or None, optional): if not None the time serie dataset is considered composed by omogeneus timeseries coming from different realization (for example point of sales, cities, locations) and the relative series are not splitted during the sample generation. Defaults to None
             check_holes_and_duplicates (bool, optional): if False duplicates or holes will not checked, the dataloader can not correctly work, disable at your own risk. Defaults True
             silly_model (bool, optional): if True, target variables will be added to the pool of the future variables. This can be useful to see if information passes throught the decoder part of your model (if any)
+            sampler_weights  group (str or None, optional): if it is a column name it will be used as weight for the sampler. Careful that the weight of the sample is the weight value of the fist target value (index)
         """
         
         
@@ -322,7 +324,7 @@ class TimeSeries():
         if group is not None:
             if group not in cat_past_var:
                 beauty_string(f'I will add {group} to the categorical past/future variables','info',self.verbose)
-                self.cat_var.append(group)
+                self.cat_past_var.append(group)
             if group not in cat_fut_var:
                 beauty_string(f'I will add {group} to the categorical past/future variables','info',self.verbose)
                 self.cat_fut_var.append(group)   
@@ -350,7 +352,7 @@ class TimeSeries():
         if silly_model:
             beauty_string('YOU ARE TRAINING A SILLY MODEL WITH THE TARGETS IN THE INPUTS','section',self.verbose) 
             self.future_variables+=self.target_variables
-            
+        self.sampler_weights = sampler_weights
     def plot(self):
         """  
         Easy way to control the loaded data
@@ -409,6 +411,7 @@ class TimeSeries():
         y_samples = []
         t_samples = []
         g_samples = []
+        sampler_weights_samples = []
         
         if starting_point is not None:
             kk = list(starting_point.keys())[0]
@@ -448,7 +451,8 @@ class TimeSeries():
 
         idx_target = []
         for c in self.target_variables:
-            idx_target.append(self.past_variables.index(c))
+            if c in self.past_variables:
+                idx_target.append(self.past_variables.index(c))
             
         idx_target_future = []
         
@@ -475,7 +479,8 @@ class TimeSeries():
             if len(self.cat_fut_var)>0:
                 x_fut_cat = tmp[self.cat_fut_var].values
             y_target = tmp[self.target_variables].values
-
+            if self.sampler_weights is not None:
+                sampler_weights = tmp[self.sampler_weights].values.flatten()
         
             if starting_point is not None:
                 check = tmp[list(starting_point.keys())[0]].values == starting_point[list(starting_point.keys())[0]]
@@ -512,6 +517,8 @@ class TimeSeries():
                                 x_cat_future_samples.append(x_fut_cat[i-shift+skip_stacked:i+future_steps-shift+skip_stacked])
                         
                         y_samples.append(y_target[i+skip_stacked:i+future_steps+skip_stacked])
+                        if self.sampler_weights is not None:
+                            sampler_weights_samples.append(sampler_weights[i+skip_stacked])
                         t_samples.append(t[i+skip_stacked:i+future_steps+skip_stacked])
                         g_samples.append(groups[i])
 
@@ -524,6 +531,8 @@ class TimeSeries():
                 beauty_string('WARNING x_num_future_samples is empty and it should not','info',True)
         
         y_samples = np.stack(y_samples)
+        if self.sampler_weights is not None:
+            sampler_weights_samples = np.stack(sampler_weights_samples)
         t_samples = np.stack(t_samples)   
         g_samples = np.stack(g_samples)
 
@@ -537,7 +546,6 @@ class TimeSeries():
         else:
             mod = 1.0
         dd = {'y':y_samples.astype(np.float32),
-
               'x_num_past':(x_num_past_samples*mod).astype(np.float32)}
         if len(self.cat_past_var)>0:
             dd['x_cat_past'] = x_cat_past_samples
@@ -545,7 +553,10 @@ class TimeSeries():
             dd['x_cat_future'] = x_cat_future_samples
         if len(self.future_variables)>0:
             dd['x_num_future'] = x_num_future_samples.astype(np.float32)
-        
+        if self.sampler_weights is not None:
+            dd['sampler_weights'] = sampler_weights_samples.astype(np.float32)
+        else:
+            dd['sampler_weights'] = np.ones(len(y_samples)).astype(np.float32)
         return MyDataset(dd,t_samples,g_samples,idx_target,idx_target_future)
     
           
@@ -660,7 +671,7 @@ class TimeSeries():
                         if c!=self.group:                               
                             self.scaler_cat[f'{c}_{group}'] =  OrdinalEncoder(dtype=np.int32,handle_unknown= 'use_encoded_value',unknown_value=train[c].nunique())
                             self.scaler_cat[f'{c}_{group}'].fit(tmp[c].values.reshape(-1,1))  
-
+ 
         dl_train = self.create_data_loader(train,past_steps,future_steps,shift,keep_entire_seq_while_shifting,starting_point,skip_step)
         dl_validation = self.create_data_loader(validation,past_steps,future_steps,shift,keep_entire_seq_while_shifting,starting_point,skip_step)
         if test.shape[0]>0:
@@ -753,10 +764,33 @@ class TimeSeries():
         else:
             self.modifier = None
         
+        if self.sampler_weights is not None:
+            beauty_string(f'USING SAMPLER IN TRAIN {min(train.sampler_weights)}-{max(train.sampler_weights)}','section',self.verbose)
+
+            sampler = WeightedRandomSampler(train.sampler_weights, num_samples= len(train))
+            train_dl = DataLoader(train, batch_size = batch_size , shuffle=False,sampler=sampler,drop_last=True,num_workers=num_workers,persistent_workers=persistent_workers)
 
-        train_dl = DataLoader(train, batch_size = batch_size , shuffle=True,drop_last=True,num_workers=num_workers,persistent_workers=persistent_workers)
+        else:
+            train_dl = DataLoader(train, batch_size = batch_size , shuffle=True,drop_last=True,num_workers=num_workers,persistent_workers=persistent_workers)
         valid_dl = DataLoader(validation, batch_size = batch_size , shuffle=False,drop_last=True,num_workers=num_workers,persistent_workers=persistent_workers)
-   
+        debug_prediction = True
+
+        if debug_prediction:
+            dl = DataLoader(test, batch_size = batch_size , shuffle=False,drop_last=True,num_workers=num_workers,persistent_workers=persistent_workers)
+            res = []
+            real = []
+            for batch in dl:
+        
+
+                res.append(self.model.inference(batch).cpu().detach().numpy())
+                real.append(batch['y'].cpu().detach().numpy())
+
+            res = np.vstack(res)
+            real = np.vstack(real)
+            with open('/home/agobbi/Projects/ExpTS/tmp_beginning.pkl','wb') as f:
+                import pickle
+                pickle.dump([res,real],f)
+        
         checkpoint_callback = ModelCheckpoint(dirpath=dirpath,
                                      monitor='val_loss',
                                       save_last = True,
@@ -851,23 +885,24 @@ class TimeSeries():
 
 
       
-
+     
         if auto_lr_find and (weight_exists is False):
-            if OLD_PL:
-                lr_tuner = trainer.tune(self.model,train_dataloaders=train_dl,val_dataloaders = valid_dl)
-                files = os.listdir(dirpath)
-                for f in files:
-                    if '.lr_find' in f:
-                        os.remove(os.path.join(dirpath,f))
-                self.model.optim_config['lr'] = lr_tuner['lr_find'].suggestion()
-            else:
-                from lightning.pytorch.tuner import Tuner
-                tuner = Tuner(trainer)
-                lr_finder = tuner.lr_find(self.model,train_dataloaders=train_dl,val_dataloaders = valid_dl)
-                self.model.optim_config['lr'] = lr_finder.suggestion() ## we are using it as optim key
+            try:
+                if OLD_PL:
+                    lr_tuner = trainer.tune(self.model,train_dataloaders=train_dl,val_dataloaders = valid_dl)
+                    files = os.listdir(dirpath)
+                    for f in files:
+                        if '.lr_find' in f:
+                            os.remove(os.path.join(dirpath,f))
+                    self.model.optim_config['lr'] = lr_tuner['lr_find'].suggestion()
+                else:
+                    from lightning.pytorch.tuner import Tuner
+                    tuner = Tuner(trainer)
+                    lr_finder = tuner.lr_find(self.model,train_dataloaders=train_dl,val_dataloaders = valid_dl)
+                    self.model.optim_config['lr'] = lr_finder.suggestion() ## we are using it as optim key
+            except Exception as e:
+                beauty_string(f'There is a problem with the finding LR routine {e}','section',self.verbose)
         
- 
-
         if OLD_PL:
             if weight_exists:
                 trainer.fit(self.model, train_dl,valid_dl,ckpt_path=os.path.join(dirpath,'last.ckpt'))
@@ -878,17 +913,36 @@ class TimeSeries():
                 trainer.fit(self.model, train_dataloaders = train_dl,val_dataloaders = valid_dl,ckpt_path=os.path.join(dirpath,'last.ckpt'))
             else:
                 trainer.fit(self.model, train_dataloaders = train_dl,val_dataloaders = valid_dl)
+
         self.checkpoint_file_best = checkpoint_callback.best_model_path
         self.checkpoint_file_last = checkpoint_callback.last_model_path 
         if self.checkpoint_file_last=='':
             beauty_string('There is a bug on saving last model I will try to fix it','info',self.verbose)
-            self.checkpoint_file_last = checkpoint_callback.best_model_path.replace('checkpoint','last')
+            self.checkpoint_file_last = os.path.join(dirpath, "last.ckpt")
+            trainer.save_checkpoint(os.path.join(dirpath, "last.ckpt"))
+        if self.checkpoint_file_best=='':
+            beauty_string('There is a bug on saving best model I will try to fix it','info',self.verbose)
+            self.checkpoint_file_best = os.path.join(dirpath, "checkpoint.ckpt")
+            trainer.save_checkpoint(os.path.join(dirpath, "checkpoint.ckpt"))
 
         self.dirpath = dirpath
         
         self.losses = mc.metrics
 
         files = os.listdir(dirpath)
+        if debug_prediction:
+            res = []
+            real = []
+            for batch in dl:
+                res.append(self.model.inference(batch).cpu().detach().numpy())
+                real.append(batch['y'].cpu().detach().numpy())
+
+            res = np.vstack(res)
+            real = np.vstack(real)
+            with open('/home/agobbi/Projects/ExpTS/tmp_after_training.pkl','wb') as f:
+                import pickle
+                pickle.dump([res,real],f)
+        
         ##accrocchio per multi gpu
         for f in files:
             if '__losses__.csv' in f:
@@ -901,7 +955,6 @@ class TimeSeries():
             self.losses = pd.DataFrame()
 
         try:
-
             if OLD_PL:
                 if isinstance(self.model, torch._dynamo.eval_frame.OptimizedModule):
                     self.model = self.model._orig_mod 
@@ -914,7 +967,21 @@ class TimeSeries():
                     self.model = mm.__class__.load_from_checkpoint(self.checkpoint_file_last)
                 else:
                     self.model = self.model.__class__.load_from_checkpoint(self.checkpoint_file_last)
+            if debug_prediction:
+                res = []
+                real = []
+                for batch in dl:
+            
+
+                    res.append(self.model.inference(batch).cpu().detach().numpy())
+                    real.append(batch['y'].cpu().detach().numpy())
 
+                res = np.vstack(res)
+                real = np.vstack(real)
+                with open('/home/agobbi/Projects/ExpTS/tmp_after_loading.pkl','wb') as f:
+                    import pickle
+                    pickle.dump([res,real],f)
+            
         except Exception as _:
             beauty_string(f'There is a problem loading the weights on file MAYBE CHANGED HOW WEIGHTS ARE LOADED {self.checkpoint_file_last}','section',self.verbose)
 
@@ -992,12 +1059,15 @@ class TimeSeries():
         beauty_string(f'Device used: {self.model.device}','info',self.verbose)
 
         for batch in dl:
+    
+
             res.append(self.model.inference(batch).cpu().detach().numpy())
             real.append(batch['y'].cpu().detach().numpy())
-       
+
         res = np.vstack(res)
- 
         real = np.vstack(real)
+
+
         time = dl.dataset.t
         groups = dl.dataset.groups
         #import pdb
@@ -1026,7 +1096,7 @@ class TimeSeries():
             
             if self.group is not None:
                 time[self.group] = groups
-                time = time.melt(id_vars=['region'])
+                time = time.melt(id_vars=[self.group])
             else:
                 time = time.melt()
             time.rename(columns={'value':'time','variable':'lag'},inplace=True)
@@ -1048,7 +1118,8 @@ class TimeSeries():
 
             if self.group is not None:
                 time[self.group] = groups
-                time = time.melt(id_vars=['region'])
+
+                time = time.melt(id_vars=[self.group])
             else:
                 time = time.melt()
             time.rename(columns={'value':'time','variable':'lag'},inplace=True)
@@ -1186,15 +1257,21 @@ class TimeSeries():
                 
                 try:
                     tmp_path = os.path.join(directory,self.checkpoint_file_last.split('/')[-1])
+                    beauty_string(f"Loading {tmp_path}",'section',self.verbose)
+
                 except Exception as _:
                     beauty_string('checkpoint_file_last not defined try to load best','section',self.verbose)
                     tmp_path = os.path.join(directory,self.checkpoint_file_best.split('/')[-1])
+                    beauty_string(f"Loading {tmp_path}",'section',self.verbose)
             else:
                 try:
+                    
                     tmp_path = os.path.join(directory,self.checkpoint_file_best.split('/')[-1])
+                    beauty_string(f"Loading {tmp_path}",'section',self.verbose)
                 except Exception as _:
                     beauty_string('checkpoint_file_best not defined try to load best','section',self.verbose)
                     tmp_path = os.path.join(directory,self.checkpoint_file_last.split('/')[-1])
+                    beauty_string(f"Loading {tmp_path}",'section',self.verbose)
         try:
             #with torch.serialization.add_safe_globals([ListConfig]):
             if OLD_PL:
@@ -1202,5 +1279,6 @@ class TimeSeries():
             else:
                 self.model = self.model.__class__.load_from_checkpoint(tmp_path,verbose=self.verbose,)
             self.model.to(torch.device("cuda:0" if torch.cuda.is_available() else "cpu"))
+
         except Exception as e:
             beauty_string(f'There is a problem loading the weights on file {tmp_path} {e}','section',self.verbose)

dsipts/data_structure/utils.py

@@ -142,12 +142,13 @@ class MyDataset(Dataset):
         Returns:
             torch.utils.data.Dataset: a torch Dataset to be used in a Dataloader
         """
+
         self.data = data
         self.t = t
         self.groups = groups
         self.idx_target = np.array(idx_target) if idx_target is not None else None
         self.idx_target_future = np.array(idx_target_future) if idx_target_future is not None else None
-
+        self.sampler_weights = data['sampler_weights']
   
 
     def __len__(self):
@@ -157,7 +158,8 @@ class MyDataset(Dataset):
     def __getitem__(self, idxs):
         sample = {}
         for k in self.data:
-            sample[k] = self.data[k][idxs]
+            if k!='sampler_weights':
+                sample[k] = self.data[k][idxs]
         if self.idx_target is not None:
             sample['idx_target'] = self.idx_target
         if self.idx_target_future is not None:

dsipts/models/Persistent.py

@@ -30,6 +30,8 @@ class Persistent(Base):
         self.save_hyperparameters(logger=False)
         self.fake = nn.Linear(1,1)
         self.use_quantiles = False
+    def can_be_compiled(self):
+        return False
     
     def forward(self, batch):
      

dsipts/models/TTM.py

@@ -38,6 +38,8 @@ class TTM(Base):
                 fcm_mix_layers,
                 fcm_prepend_past,
                 enable_forecast_channel_mixing,
+                force_return,
+                few_shot = True,
                 **kwargs)->None:
    
         super().__init__(**kwargs)
@@ -48,7 +50,9 @@ class TTM(Base):
         self.index_fut = list(exogenous_channel_indices_cont)
 
         if len(exogenous_channel_indices_cat)>0:
-            self.index_fut_cat = (self.past_channels+len(self.embs_past))+list(exogenous_channel_indices_cat)
+
+            self.index_fut_cat = [self.past_channels+c for c in list(exogenous_channel_indices_cat)]
+
         else:
             self.index_fut_cat = []
         self.freq = freq
@@ -75,6 +79,7 @@ class TTM(Base):
             fcm_use_mixer=fcm_use_mixer,
             fcm_mix_layers=fcm_mix_layers,
             freq=freq,
+            force_return=force_return,
             freq_prefix_tuning=freq_prefix_tuning,
             fcm_prepend_past=fcm_prepend_past,
             enable_forecast_channel_mixing=enable_forecast_channel_mixing,
@@ -82,8 +87,9 @@ class TTM(Base):
         )
         hidden_size =  self.model.config.hidden_size
         self.model.prediction_head = torch.nn.Linear(hidden_size, self.out_channels*self.mul)
-        self._freeze_backbone()
-
+        if few_shot:
+            self._freeze_backbone()
+        self.zero_pad = (force_return=='zeropad') 
     def _freeze_backbone(self):
         """
         Freeze the backbone of the model.
@@ -108,29 +114,44 @@ class TTM(Base):
         return input
 
     def can_be_compiled(self):
-        return True  
+        
+        return True#True#not self.zero_pad  
   
     def forward(self, batch):
         x_enc = batch['x_num_past'].to(self.device)
+
+        
+        if self.zero_pad:
+            B,L,C = batch['x_num_past'].shape
+            x_enc = torch.zeros((B,512,C)).to(self.device)
+            x_enc[:,-L:,:] = batch['x_num_past'].to(self.device)
+        else:
+            x_enc = batch['x_num_past'].to(self.device)
         original_indexes = batch['idx_target'][0].tolist()
 
 
         if 'x_cat_past' in batch.keys():
-            x_mark_enc = batch['x_cat_past'].to(torch.float32).to(self.device)
-            x_mark_enc = self._scaler_past(x_mark_enc)
+            if self.zero_pad:
+                B,L,C = batch['x_cat_past'].shape
+                x_mark_enc = torch.zeros((B,512,C)).to(self.device)
+                x_mark_enc[:,-L:,:] = batch['x_cat_past'].to(torch.float32).to(self.device)
+            else:
+                x_mark_enc = batch['x_cat_past'].to(torch.float32).to(self.device)
+                x_mark_enc = self._scaler_past(x_mark_enc)
             past_values = torch.cat((x_enc,x_mark_enc), axis=-1).type(torch.float32)
         else:
             past_values = x_enc
+        B,L,C = past_values.shape
+        future_values = torch.zeros((B,self.future_steps,C)).to(self.device)
         
-        future_values = torch.zeros_like(past_values).to(self.device)
-        future_values = future_values[:,:self.future_steps,:]
 
+   
         if 'x_num_future' in batch.keys(): 
             future_values[:,:,self.index_fut] = batch['x_num_future'].to(self.device)
         if 'x_cat_future' in batch.keys():
             x_mark_dec = batch['x_cat_future'].to(torch.float32).to(self.device)
             x_mark_dec = self._scaler_fut(x_mark_dec)
-            future_values[:,:,self.index_cat_fut] = x_mark_dec
+            future_values[:,:,self.index_fut_cat] = x_mark_dec
         
 
         #investigating!! problem with dynamo!
@@ -153,6 +174,7 @@ class TTM(Base):
 
 
         BS = res.shape[0]
+
         return res.reshape(BS,self.future_steps,-1,self.mul)
         
     

dsipts/models/TimeKAN.py

@@ -0,0 +1,123 @@
+## Copyright https://github.com/huangst21/TimeKAN/blob/main/models/TimeKAN.py
+## Modified for notation alignmenet and batch structure
+## extended to what inside itransformer folder
+
+import torch
+import torch.nn as nn
+import numpy as np
+from .timekan.Layers import FrequencyDecomp,FrequencyMixing,series_decomp,Normalize
+from ..data_structure.utils import beauty_string
+from .utils import  get_scope,get_activation,Embedding_cat_variables
+
+try:
+    import lightning.pytorch as pl
+    from .base_v2 import Base
+    OLD_PL = False
+except:
+    import pytorch_lightning as pl
+    OLD_PL = True
+    from .base import Base
+
+
+
+class TimeKAN(Base):
+    handle_multivariate = True
+    handle_future_covariates = True
+    handle_categorical_variables = True
+    handle_quantile_loss = True
+    description = get_scope(handle_multivariate,handle_future_covariates,handle_categorical_variables,handle_quantile_loss)
+    
+    def __init__(self, 
+                 # specific params
+                 down_sampling_window:int,
+                 e_layers:int,
+                 moving_avg:int,
+                 down_sampling_layers: int,
+                 d_model: int,
+                 begin_order: int,
+                 use_norm:bool,
+                 **kwargs)->None:
+             
+        
+        
+        
+        super().__init__(**kwargs)
+       
+        self.save_hyperparameters(logger=False)
+        self.e_layers = e_layers
+        self.emb_past = Embedding_cat_variables(self.past_steps,self.emb_dim,self.embs_past, reduction_mode=self.reduction_mode,use_classical_positional_encoder=self.use_classical_positional_encoder,device = self.device)
+        #self.emb_fut = Embedding_cat_variables(self.future_steps,self.emb_dim,self.embs_fut, reduction_mode=self.reduction_mode,use_classical_positional_encoder=self.use_classical_positional_encoder,device = self.device)
+        emb_past_out_channel = self.emb_past.output_channels
+        
+        
+        
+        self.res_blocks = nn.ModuleList([FrequencyDecomp( self.past_steps,down_sampling_window,down_sampling_layers) for _ in range(e_layers)])
+        self.add_blocks = nn.ModuleList([FrequencyMixing(d_model,self.past_steps,begin_order,down_sampling_window,down_sampling_layers) for _ in range(e_layers)])
+
+        self.preprocess = series_decomp(moving_avg)
+        self.enc_in = self.past_channels + emb_past_out_channel
+        self.project = nn.Linear(self.enc_in,d_model)
+        self.layer = e_layers
+        self.normalize_layers = torch.nn.ModuleList(
+            [
+                Normalize(self.enc_in, affine=True, non_norm=use_norm)
+                for i in range(down_sampling_layers + 1)
+            ]
+        )
+        self.predict_layer =nn. Linear(
+                        self.past_steps,
+                        self.future_steps,
+                    )
+        self.final_layer = nn.Linear(d_model, self.mul)
+        self.down_sampling_layers = down_sampling_layers
+        self.down_sampling_window = down_sampling_window
+    def can_be_compiled(self):
+        return True#True  
+
+
+    def __multi_level_process_inputs(self, x_enc):
+        down_pool = torch.nn.AvgPool1d(self.down_sampling_window)
+        # B,T,C -> B,C,T
+        x_enc = x_enc.permute(0, 2, 1)
+        x_enc_ori = x_enc
+        x_enc_sampling_list = []
+        x_enc_sampling_list.append(x_enc.permute(0, 2, 1))
+        for i in range(self.down_sampling_layers):
+            x_enc_sampling = down_pool(x_enc_ori)
+            x_enc_sampling_list.append(x_enc_sampling.permute(0, 2, 1))
+            x_enc_ori = x_enc_sampling
+        x_enc = x_enc_sampling_list
+        return x_enc
+
+
+    def forward(self, batch:dict)-> float:
+
+        x_enc = batch['x_num_past'].to(self.device)
+        BS = x_enc.shape[0]
+        if 'x_cat_past' in batch.keys():
+            emb_past = self.emb_past(BS,batch['x_cat_past'].to(self.device))
+        else:
+            emb_past = self.emb_past(BS,None)
+
+        x_past = torch.cat([x_enc,emb_past],2)
+
+        x_enc = self.__multi_level_process_inputs(x_past)
+
+        x_list = []
+        for i, x in zip(range(len(x_enc)), x_enc, ):
+            B, T, N = x.size()
+            x = self.normalize_layers[i](x, 'norm')
+            x = x.permute(0, 2, 1).contiguous().reshape(B * N, T, 1)
+            x_list.append(self.project(x.reshape(B, T, N)))
+
+
+
+        for i in range(self.layer):
+            x_list = self.res_blocks[i](x_list)
+            x_list = self.add_blocks[i](x_list)
+
+        dec_out = x_list[0]
+        dec_out = self.predict_layer(dec_out.permute(0, 2, 1)).permute( 0, 2, 1)  
+        dec_out = self.final_layer(dec_out)
+
+        return dec_out.reshape(BS,self.future_steps,self.out_channels,self.mul)