dsipts 1.1.5__py3-none-any.whl

dsipts/models/LinearTS.py

@@ -0,0 +1,222 @@
+
+## Copyright 2022 DLinear Authors (https://github.com/cure-lab/LTSF-Linear/tree/main?tab=Apache-2.0-1-ov-file#readme)
+## Code modified for align the notation and the batch generation
+## extended to all present in informer, autoformer folder
+
+from torch import nn
+import torch
+
+try:
+    import lightning.pytorch as pl
+    from .base_v2 import Base
+    OLD_PL = False
+except:
+    import pytorch_lightning as pl
+    OLD_PL = True
+    from .base import Base
+from .utils import QuantileLossMO, get_activation
+from typing import List, Union
+from ..data_structure.utils import beauty_string
+from .utils import  get_scope
+from .utils import Embedding_cat_variables
+    
+
+
+class moving_avg(nn.Module):
+    """
+    Moving average block to highlight the trend of time series
+    """
+    def __init__(self, kernel_size, stride):
+        super(moving_avg, self).__init__()
+        self.kernel_size = kernel_size
+        self.avg = nn.AvgPool1d(kernel_size=kernel_size, stride=stride, padding=0)
+
+    def forward(self, x):
+        # padding on the both ends of time series
+        front = x[:, 0:1, :].repeat(1, (self.kernel_size - 1) // 2, 1)
+        end = x[:, -1:, :].repeat(1, (self.kernel_size - 1) // 2, 1)
+        x = torch.cat([front, x, end], dim=1)
+        x = self.avg(x.permute(0, 2, 1))
+        x = x.permute(0, 2, 1)
+        return x
+
+
+class series_decomp(nn.Module):
+    """
+    Series decomposition block
+    """
+    def __init__(self, kernel_size):
+        super(series_decomp, self).__init__()
+        self.moving_avg = moving_avg(kernel_size, stride=1)
+
+    def forward(self, x):
+        moving_mean = self.moving_avg(x)
+        res = x - moving_mean
+        return res, moving_mean
+
+
+class LinearTS(Base):
+    handle_multivariate = True
+    handle_future_covariates = True
+    handle_categorical_variables = True
+    handle_quantile_loss = True
+    description = get_scope(handle_multivariate,handle_future_covariates,handle_categorical_variables,handle_quantile_loss)
+    description+='\n THE SIMPLE IMPLEMENTATION DOES NOT USE CATEGORICAL NOR FUTURE VARIABLES'
+    
+    def __init__(self, 
+               
+                 kernel_size:int,
+                 hidden_size:int,
+                 dropout_rate:float=0.1,
+                 activation:str='torch.nn.ReLU',
+                 kind:str='linear',
+                 use_bn:bool=False,
+                 simple:bool=False,
+                 **kwargs)->None:
+        """Initialize the model with specified parameters. Linear model from https://github.com/cure-lab/LTSF-Linear/blob/main/run_longExp.py
+        
+        Args:
+            kernel_size (int): Kernel dimension for the initial moving average.
+            hidden_size (int): Hidden size of the linear block.
+            dropout_rate (float, optional): Dropout rate in Dropout layers. Default is 0.1.
+            activation (str, optional): Activation function in PyTorch. Default is 'torch.nn.ReLU'.
+            kind (str, optional): Type of model, can be 'linear', 'dlinear' (de-trending), or 'nlinear' (differential). Defaults to 'linear'.
+            use_bn (bool, optional): If True, Batch Normalization layers will be added and Dropouts will be removed. Default is False.
+            simple (bool, optional): If True, the model used is the same as illustrated in the paper; otherwise, a more complex model with the same idea is used. Default is False.
+            **kwargs: Additional keyword arguments for the parent class.
+        
+        Raises:
+            ValueError: If an invalid activation function is provided.
+        """
+        
+        super().__init__(**kwargs)
+
+        if activation == 'torch.nn.SELU':
+            beauty_string('SELU do not require BN','info',self.verbose)
+            use_bn = False
+            
+        if isinstance(activation, str):
+            activation = get_activation(activation)
+        else:
+            beauty_string('There is a bug in pytorch lightening, the constructior is called twice','info',self.verbose)
+        
+        self.save_hyperparameters(logger=False)
+      
+        self.kind = kind
+       
+
+        self.simple = simple
+       
+        self.emb_past = Embedding_cat_variables(self.past_steps,self.emb_dim,self.embs_past, reduction_mode=self.reduction_mode,use_classical_positional_encoder=self.use_classical_positional_encoder,device = self.device)
+        self.emb_fut = Embedding_cat_variables(self.future_steps,self.emb_dim,self.embs_fut, reduction_mode=self.reduction_mode,use_classical_positional_encoder=self.use_classical_positional_encoder,device = self.device)
+        emb_past_out_channel = self.emb_past.output_channels
+        emb_fut_out_channel = self.emb_fut.output_channels
+
+
+
+        ## ne faccio uno per ogni canale
+        self.linear =  nn.ModuleList()
+
+            
+        if kind=='dlinear':
+            self.decompsition = series_decomp(kernel_size)    
+            self.Linear_Trend = nn.ModuleList()
+            for _ in range(self.out_channels):
+                self.Linear_Trend.append(nn.Linear(self.past_steps,self.future_steps))
+            
+        
+        for _ in range(self.out_channels):
+            if simple:
+                self.linear.append(nn.Linear(self.past_steps,self.future_steps*self.mul))
+                                               
+            else:
+                self.linear.append(nn.Sequential(nn.Linear(emb_past_out_channel*self.past_steps+emb_fut_out_channel*self.future_steps+self.past_steps*self.past_channels+self.future_channels*self.future_steps,hidden_size),
+                                                    activation(),
+                                                    nn.BatchNorm1d(hidden_size) if use_bn else nn.Dropout(dropout_rate) ,    
+                                                    nn.Linear(hidden_size,hidden_size//2), 
+                                                    activation(),
+                                                    nn.BatchNorm1d(hidden_size//2) if use_bn else nn.Dropout(dropout_rate) ,    
+                                                    nn.Linear(hidden_size//2,hidden_size//4),
+                                                    activation(),
+                                                    nn.BatchNorm1d(hidden_size//4) if use_bn else nn.Dropout(dropout_rate) ,    
+                                                    nn.Linear(hidden_size//4,hidden_size//8),
+                                                    activation(),
+                                                    nn.BatchNorm1d(hidden_size//8) if use_bn else nn.Dropout(dropout_rate) ,    
+                                                    nn.Linear(hidden_size//8,self.future_steps*self.mul)))
+                               
+    def forward(self, batch):
+      
+        x =  batch['x_num_past'].to(self.device)
+        idx_target = batch['idx_target'][0]
+        
+        BS = x.shape[0]
+        if 'x_cat_future' in batch.keys():
+            emb_fut = self.emb_fut(BS,batch['x_cat_future'].to(self.device))
+        else:
+            emb_fut = self.emb_fut(BS,None)
+        if 'x_cat_past' in batch.keys():
+            emb_past = self.emb_past(BS,batch['x_cat_past'].to(self.device))
+        else:
+            emb_past = self.emb_past(BS,None)
+        
+        if self.kind=='nlinear':
+            
+            x_start = x[:,-1,idx_target].unsqueeze(1)
+            x[:,:,idx_target]-=x_start
+        
+        if self.kind=='alinear':
+            x[:,:,idx_target] = 0
+        
+        if self.kind=='dlinear':
+            x_start = x[:,:,idx_target]
+            seasonal_init, trend_init = self.decompsition(x_start)
+            seasonal_init, trend_init = seasonal_init.permute(0,2,1), trend_init.permute(0,2,1)
+            x[:,:,idx_target] = seasonal_init.permute(0,2,1)
+            tmp = []
+            for j in range(len(self.Linear_Trend)):
+               
+                tmp.append(self.Linear_Trend[j](trend_init[:,j,:]))
+
+            trend = torch.stack(tmp,2)
+            
+        if self.simple is False:
+            if 'x_num_future' in batch.keys():
+                x_future = batch['x_num_future'].to(self.device)
+            else:
+                x_future = None
+                
+            tmp = [x,emb_past]
+            tot_past = torch.cat(tmp,2).flatten(1)
+        
+
+
+            tmp = [emb_fut]
+                          
+            if x_future is not None:
+                tmp.append(x_future)
+           
+            tot_future = torch.cat(tmp,2).flatten(1)
+            tot = torch.cat([tot_past,tot_future],1)
+                
+            tot = tot.unsqueeze(2).repeat(1,1,len(self.linear)).permute(0,2,1)
+        else:
+            tot = x.permute(0,2,1)
+        res = []
+
+        for j in range(len(self.linear)):
+            res.append(self.linear[j](tot[:,j,:]).reshape(BS,-1,self.mul))
+        ## BxLxCxMUL
+        res = torch.stack(res,2)
+
+        if self.kind=='nlinear':
+            #res BxLxCx3
+            #start BxCx1
+            res+=x_start.unsqueeze(1)
+        
+
+        if self.kind=='dlinear':
+            res = res+trend.unsqueeze(3)
+        
+            
+        return res
+    

dsipts/models/PatchTST.py

@@ -0,0 +1,181 @@
+## Copyright https://github.com/yuqinie98/PatchTST/blob/main/LICENSE
+## Modified for notation alignmenet and batch structure
+## extended to what inside patchtst folder
+
+
+
+from torch import  nn
+import torch
+
+try:
+    import lightning.pytorch as pl
+    from .base_v2 import Base
+    OLD_PL = False
+except:
+    import pytorch_lightning as pl
+    OLD_PL = True
+    from .base import Base
+from typing import List,Union
+from ..data_structure.utils import beauty_string
+from .utils import  get_scope
+from .utils import  get_activation
+from .patchtst.layers import series_decomp, PatchTST_backbone
+from .utils import Embedding_cat_variables
+
+
+
+  
+class PatchTST(Base):
+    handle_multivariate = True
+    handle_future_covariates = False
+    handle_categorical_variables = True
+    handle_quantile_loss = True
+    description = get_scope(handle_multivariate,handle_future_covariates,handle_categorical_variables,handle_quantile_loss)
+    
+    
+    def __init__(self, 
+              
+                 d_model:int,
+                 patch_len:int,
+                 kernel_size:int,
+                 decomposition:bool=True,
+                 activation:str='torch.nn.ReLU',
+                 n_head:int=1,
+                 n_layer:int=2,
+                 stride:int=8,
+                 remove_last:bool = False,
+                 hidden_size:int=1048,
+                 dropout_rate:float=0.1,
+                 **kwargs)->None:
+        """Initializes the model with specified parameters.https://github.com/yuqinie98/PatchTST/blob/main/
+        
+        Args:
+            d_model (int): The dimensionality of the model.
+            patch_len (int): The length of the patches.
+            kernel_size (int): The size of the kernel for convolutional layers.
+            decomposition (bool, optional): Whether to use decomposition. Defaults to True.
+            activation (str, optional): The activation function to use. Defaults to 'torch.nn.ReLU'.
+            n_head (int, optional): The number of attention heads. Defaults to 1.
+            n_layer (int, optional): The number of layers in the model. Defaults to 2.
+            stride (int, optional): The stride for convolutional layers. Defaults to 8.
+            remove_last (bool, optional): Whether to remove the last layer. Defaults to False.
+            hidden_size (int, optional): The size of the hidden layers. Defaults to 1048.
+            dropout_rate (float, optional): The dropout rate for regularization. Defaults to 0.1.
+            **kwargs: Additional keyword arguments.
+        
+        Raises:
+            ValueError: If the activation function is not recognized.
+        
+
+        """
+        super().__init__(**kwargs)
+
+        if activation == 'torch.nn.SELU':
+            beauty_string('SELU do not require BN','info',self.verbose)
+        if isinstance(activation, str):
+            activation = get_activation(activation)
+        else:
+            beauty_string('There is a bug in pytorch lightening, the constructior is called twice ','info',self.verbose)
+        
+   
+        self.save_hyperparameters(logger=False)
+      
+        self.remove_last = remove_last
+     
+        self.emb_past = Embedding_cat_variables(self.past_steps,self.emb_dim,self.embs_past, reduction_mode=self.reduction_mode,use_classical_positional_encoder=self.use_classical_positional_encoder,device = self.device)
+        self.emb_fut = Embedding_cat_variables(self.future_steps,self.emb_dim,self.embs_fut, reduction_mode=self.reduction_mode,use_classical_positional_encoder=self.use_classical_positional_encoder,device = self.device)
+        emb_past_out_channel = self.emb_past.output_channels
+        emb_fut_out_channel = self.emb_fut.output_channels
+
+     
+    
+        self.past_channels+=emb_past_out_channel
+        
+    
+        # model
+        self.decomposition = decomposition
+        if self.decomposition:
+            self.decomp_module = series_decomp(kernel_size)
+            self.model_trend = PatchTST_backbone(c_in=self.past_channels, context_window = self.past_steps, target_window=self.future_steps, patch_len=patch_len, stride=stride, 
+                                  max_seq_len=self.past_steps+self.future_steps, n_layers=n_layer, d_model=d_model,
+                                  n_heads=n_head, d_k=None, d_v=None, d_ff=hidden_size, norm='BatchNorm', attn_dropout=dropout_rate,
+                                  dropout=dropout_rate, act=activation(), key_padding_mask='auto', padding_var=None, 
+                                  attn_mask=None, res_attention=True, pre_norm=False, store_attn=False,
+                                  pe='zeros', learn_pe=True, fc_dropout=dropout_rate, head_dropout=dropout_rate, padding_patch = 'end',
+                                  pretrain_head=False, head_type='flatten', individual=False, revin=True, affine=False,
+                                  subtract_last=remove_last, verbose=False)
+            self.model_res = PatchTST_backbone(c_in=self.past_channels, context_window = self.past_steps, target_window=self.future_steps, patch_len=patch_len, stride=stride, 
+                                  max_seq_len=self.past_steps+self.future_steps, n_layers=n_layer, d_model=d_model,
+                                  n_heads=n_head, d_k=None, d_v=None, d_ff=hidden_size, norm='BatchNorm', attn_dropout=dropout_rate,
+                                  dropout=dropout_rate, act=activation(), key_padding_mask='auto', padding_var=None, 
+                                  attn_mask=None, res_attention=True, pre_norm=False, store_attn=False,
+                                  pe='zeros', learn_pe=True, fc_dropout=dropout_rate, head_dropout=dropout_rate, padding_patch = 'end',
+                                  pretrain_head=False, head_type='flatten', individual=False, revin=True, affine=False,
+                                  subtract_last=remove_last, verbose=False)
+        else:
+            self.model = PatchTST_backbone(c_in=self.past_channels, context_window = self.past_steps, target_window=self.future_steps, patch_len=patch_len, stride=stride, 
+                                  max_seq_len=self.past_steps+self.future_steps, n_layers=n_layer, d_model=d_model,
+                                  n_heads=n_head, d_k=None, d_v=None, d_ff=hidden_size, norm='BatchNorm', attn_dropout=dropout_rate,
+                                  dropout=dropout_rate, act=activation(), key_padding_mask='auto', padding_var=None, 
+                                  attn_mask=None, res_attention=True, pre_norm=False, store_attn=False,
+                                  pe='zeros', learn_pe=True, fc_dropout=dropout_rate, head_dropout=dropout_rate, padding_patch = 'end',
+                                  pretrain_head=False, head_type='flatten', individual=False, revin=True, affine=False,
+                                  subtract_last=remove_last, verbose=False)
+    
+    
+        dim = self.past_channels+emb_fut_out_channel+self.future_channels
+        self.final_layer = nn.Sequential(activation(),
+                                         nn.Linear(dim, dim*2),
+                                         activation(),
+                                         nn.Linear(dim*2,self.out_channels*self.mul  ))
+
+
+    
+        #self.final_linear = nn.Sequential(nn.Linear(past_channels,past_channels//2),activation(),nn.Dropout(dropout_rate), nn.Linear(past_channels//2,out_channels)  )
+    
+    def forward(self, batch):           # x: [Batch, Input length, Channel]
+        
+
+        x_seq = batch['x_num_past'].to(self.device)#[:,:,idx_target]
+        BS = x_seq.shape[0]
+        if 'x_cat_future' in batch.keys():
+            emb_fut = self.emb_fut(BS,batch['x_cat_future'].to(self.device))
+        else:
+            emb_fut = self.emb_fut(BS,None)
+        if 'x_cat_past' in batch.keys():
+            emb_past = self.emb_past(BS,batch['x_cat_past'].to(self.device))
+        else:
+            emb_past = self.emb_past(BS,None)
+            
+        tmp_future = [emb_fut]
+        if 'x_num_future' in batch.keys():
+            x_future = batch['x_num_future'].to(self.device)
+            tmp_future.append(x_future)
+        
+        
+        tot = [x_seq,emb_past]
+    
+        x_seq = torch.cat(tot,axis=2)
+
+        if self.decomposition:
+            res_init, trend_init = self.decomp_module(x_seq)
+            res_init, trend_init = res_init.permute(0,2,1), trend_init.permute(0,2,1)  # x: [Batch, Channel, Input length]
+            res = self.model_res(res_init)
+            trend = self.model_trend(trend_init)
+            x = res + trend
+            x = x.permute(0,2,1)    # x: [Batch, Input length, Channel]
+        else:
+            x = x_seq.permute(0,2,1)# x: [Batch, Channel, Input length]
+            x = self.model(x)
+            x = x.permute(0,2,1)    # x: [Batch, Input length, Channel]
+        
+        
+        tmp_future.append(x)
+        tmp_future = torch.cat(tmp_future,2)
+        output = self.final_layer(tmp_future)
+        return output.reshape(BS,self.future_steps,self.out_channels,self.mul)
+
+        
+        
+        
+        

dsipts/models/Persistent.py

@@ -0,0 +1,44 @@
+
+from torch import nn
+
+try:
+    import lightning.pytorch as pl
+    from .base_v2 import Base
+    OLD_PL = False
+except:
+    import pytorch_lightning as pl
+    OLD_PL = True
+    from .base import Base
+from .utils import L1Loss
+from ..data_structure.utils import beauty_string
+from .utils import  get_scope
+
+class Persistent(Base):
+    handle_multivariate = True
+    handle_future_covariates = False
+    handle_categorical_variables = False
+    handle_quantile_loss = False
+    description = get_scope(handle_multivariate,handle_future_covariates,handle_categorical_variables,handle_quantile_loss)
+    
+    def __init__(self, 
+                 **kwargs)->None:
+        """
+        Simple persistent model aligned with all the other
+        """
+    
+        super().__init__(**kwargs)
+        self.save_hyperparameters(logger=False)
+        self.fake = nn.Linear(1,1)
+        self.use_quantiles = False
+    
+    def forward(self, batch):
+     
+        x =  batch['x_num_past'].to(self.device)
+        idx_target = batch['idx_target'][0]
+        x_start = x[:,-1,idx_target].unsqueeze(1)
+        #this is B,1,C
+        #[B,L,C,1] remember the outoput size
+        res = x_start.repeat(1,self.future_steps,1).unsqueeze(3)
+        
+        return res
+    

dsipts/models/RNN.py

@@ -0,0 +1,213 @@
+
+from torch import  nn
+import torch
+
+try:
+    import lightning.pytorch as pl
+    from .base_v2 import Base
+    OLD_PL = False
+except:
+    import pytorch_lightning as pl
+    OLD_PL = True
+    from .base import Base
+from .utils import QuantileLossMO,Permute,get_activation
+from typing import List,Union
+from ..data_structure.utils import beauty_string
+from .utils import  get_scope
+from .xlstm.xLSTM import xLSTM
+from .utils import Embedding_cat_variables
+
+
+class MyBN(nn.Module):
+    def __init__(self,channels):
+        super(MyBN, self).__init__()
+        self.BN = nn.BatchNorm1d(channels)
+    def forward(self,x):
+        return self.BN(x.permute(0,2,1)).permute(0,2,1)
+
+class RNN(Base):
+    handle_multivariate = True
+    handle_future_covariates = True
+    handle_categorical_variables = True
+    handle_quantile_loss = True
+    
+    
+    
+    def __init__(self, 
+                
+                 hidden_RNN:int,
+                 num_layers_RNN:int,
+                 kind:str,
+                 kernel_size:int,
+                 activation:str='torch.nn.ReLU',
+                 remove_last = False,
+                 dropout_rate:float=0.1,
+                 use_bn:bool=False,
+                 num_blocks:int=4, 
+                 bidirectional:bool=True,
+                 lstm_type:str='slstm',
+                
+                 **kwargs)->None:
+        """Initialize a recurrent model with an encoder-decoder structure.
+        
+        Args:
+            hidden_RNN (int): Hidden size of the RNN block.
+            num_layers_RNN (int): Number of RNN layers.
+            kind (str): Type of RNN to use, either 'gru' or 'lstm' or `xlstm`.
+            kernel_size (int): Kernel size in the encoder convolutional block.
+            activation (str, optional): Activation function from PyTorch. Default is 'torch.nn.ReLU'.
+            remove_last (bool, optional): If True, the model learns the difference with respect to the last seen point. Default is False.
+            dropout_rate (float, optional): Dropout rate in Dropout layers. Default is 0.1.
+            use_bn (bool, optional): If True, Batch Normalization layers will be added and Dropouts will be removed. Default is False.
+            num_blocks (int, optional): Number of xLSTM blocks (only for xLSTM). Default is 4.
+            bidirectional (bool, optional): If True, the RNN is bidirectional. Default is True.
+            lstm_type (str, optional): Type of LSTM to use (only for xLSTM), either 'slstm' or 'mlstm'. Default is 'slstm'.
+            **kwargs: Additional keyword arguments.
+        
+        
+        Raises:
+            ValueError: If the specified kind is not 'lstm', 'gru', or 'xlstm'.
+        """
+        
+        
+        super().__init__(**kwargs)
+        
+        if activation == 'torch.nn.SELU':
+            beauty_string('SELU do not require BN','info',self.verbose)
+            use_bn = False
+        if isinstance(activation, str):
+            activation = get_activation(activation)
+        else:
+            beauty_string('There is a bug in pytorch lightening, the constructior is called twice ','info',self.verbose)
+        
+        self.save_hyperparameters(logger=False)
+
+        self.num_layers_RNN = num_layers_RNN
+        self.hidden_RNN = hidden_RNN
+
+        self.kind = kind
+        self.remove_last = remove_last
+        
+        
+        self.emb_past = Embedding_cat_variables(self.past_steps,self.emb_dim,self.embs_past, reduction_mode=self.reduction_mode,use_classical_positional_encoder=self.use_classical_positional_encoder,device = self.device)
+        self.emb_fut = Embedding_cat_variables(self.future_steps,self.emb_dim,self.embs_fut, reduction_mode=self.reduction_mode,use_classical_positional_encoder=self.use_classical_positional_encoder,device = self.device)
+        emb_past_out_channel = self.emb_past.output_channels
+        emb_fut_out_channel = self.emb_fut.output_channels
+
+        
+
+
+     
+        self.initial_linear_encoder =  nn.Sequential(nn.Linear(self.past_channels,4),
+                                                     activation(),
+                                                     
+                                                    MyBN(4) if use_bn else nn.Dropout(dropout_rate) ,
+                                                     nn.Linear(4,8),
+                                                     activation(),
+                                                    MyBN(8) if use_bn else nn.Dropout(dropout_rate) ,
+                                                     nn.Linear(8,hidden_RNN//8))
+        self.initial_linear_decoder =  nn.Sequential(nn.Linear(self.future_channels,4),
+                                                     activation(),
+                                                     MyBN(4) if use_bn else nn.Dropout(dropout_rate) ,
+                                                     nn.Linear(4,8),
+                                                     activation(),
+                                                     MyBN(8) if use_bn else nn.Dropout(dropout_rate) ,
+                                                     nn.Linear(8,hidden_RNN//8))
+        
+        
+        self.conv_encoder = nn.Sequential(Permute(), nn.Conv1d(emb_past_out_channel+hidden_RNN//8, hidden_RNN//8, kernel_size, stride=1,padding='same'),Permute(),nn.Dropout(0.3))
+        
+        if self.future_channels+emb_fut_out_channel==0:
+            ## occhio che vuol dire che non ho futuro , per ora ci metto una pezza e uso hidden dell'encoder
+            self.conv_decoder =  nn.Sequential(Permute(),nn.Conv1d(hidden_RNN, hidden_RNN//8, kernel_size=kernel_size, stride=1,padding='same'),   Permute())
+        else:
+            self.conv_decoder =  nn.Sequential(Permute(),nn.Conv1d(self.future_channels+emb_fut_out_channel, hidden_RNN//8, kernel_size=kernel_size, stride=1,padding='same'),   Permute())
+            
+            
+        if self.kind=='lstm':
+            self.Encoder = nn.LSTM(input_size= hidden_RNN//8,hidden_size=hidden_RNN,num_layers = num_layers_RNN,batch_first=True)
+            self.Decoder = nn.LSTM(input_size= hidden_RNN//8,hidden_size=hidden_RNN,num_layers = num_layers_RNN,batch_first=True)
+        elif self.kind=='gru':
+            self.Encoder = nn.GRU(input_size= hidden_RNN//8,hidden_size=hidden_RNN,num_layers = num_layers_RNN,batch_first=True)
+            self.Decoder = nn.GRU(input_size= hidden_RNN//8,hidden_size=hidden_RNN,num_layers = num_layers_RNN,batch_first=True)
+        elif self.kind=='xlstm':
+            self.Encoder = xLSTM(input_size= hidden_RNN//8,hidden_size=hidden_RNN,num_layers = num_layers_RNN,num_blocks=num_blocks,dropout=dropout_rate, bidirectional=bidirectional, lstm_type=lstm_type)
+            self.Decoder = xLSTM(input_size= hidden_RNN//8,hidden_size=hidden_RNN,num_layers = num_layers_RNN,num_blocks=num_blocks,dropout=dropout_rate, bidirectional=bidirectional, lstm_type=lstm_type)
+      
+        else:
+            beauty_string('Speciky kind= lstm or gru please','section',True)
+        self.final_linear = nn.ModuleList()
+        for _ in range(self.out_channels*self.mul):
+            self.final_linear.append(nn.Sequential(nn.Linear(hidden_RNN,hidden_RNN//2), 
+                                            activation(),
+                                            MyBN(hidden_RNN//2) if use_bn else nn.Dropout(dropout_rate) ,
+                                            nn.Linear(hidden_RNN//2,hidden_RNN//4),
+                                            activation(),
+                                            MyBN(hidden_RNN//4) if use_bn else nn.Dropout(dropout_rate) ,
+                                            nn.Linear(hidden_RNN//4,hidden_RNN//8),
+                                            activation(),
+                                            MyBN(hidden_RNN//8) if use_bn else nn.Dropout(dropout_rate) ,
+                                            nn.Linear(hidden_RNN//8,1)))
+
+  
+
+    def forward(self, batch):
+  
+        x =  batch['x_num_past'].to(self.device)
+
+        BS = x.shape[0]
+        if 'x_cat_future' in batch.keys():
+            emb_fut = self.emb_fut(BS,batch['x_cat_future'].to(self.device))
+        else:
+            emb_fut = self.emb_fut(BS,None)
+        if 'x_cat_past' in batch.keys():
+            emb_past = self.emb_past(BS,batch['x_cat_past'].to(self.device))
+        else:
+            emb_past = self.emb_past(BS,None)
+
+        if 'x_num_future' in batch.keys():
+            x_future = batch['x_num_future'].to(self.device)
+        else:
+            x_future = None
+        
+        if self.remove_last:
+            idx_target = batch['idx_target'][0]
+
+            x_start = x[:,-1,idx_target].unsqueeze(1)
+            ##BxC
+            x[:,:,idx_target]-=x_start        
+        
+        tmp = [self.initial_linear_encoder(x),emb_past]
+        
+        
+        tot = torch.cat(tmp,2)
+
+        out, hidden = self.Encoder(self.conv_encoder(tot))      
+
+        tmp = [emb_fut]
+                   
+        if x_future is not None:
+            tmp.append(x_future)
+            
+        if len(tmp)>0:
+            tot = torch.cat(tmp,2)
+        else:
+            tot = out
+        out, _ = self.Decoder(self.conv_decoder(tot[:,-1:,:].repeat(1,self.future_steps,1)),hidden)  
+        res = []
+
+      
+        for j in range(len(self.final_linear)):
+            res.append(self.final_linear[j](out))
+            
+        res = torch.cat(res,2)
+        ##BxLxC
+        B,L,_ = res.shape
+        res = res.reshape(B,L,-1,self.mul)
+        
+        if self.remove_last:
+            res+=x_start.unsqueeze(1)
+      
+        return res
+
+