torch-rechub 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

torch_rechub/models/ranking/dcn_v2.py

@@ -0,0 +1,69 @@
+"""
+Date: create on 09/01/2022
+References:
+    paper: (WWW'21) Dcn v2: Improved deep & cross network and practical lessons for web-scale learning to rank systems
+    url: https://arxiv.org/abs/2008.13535
+Authors: lailai, lailai_zxy@tju.edu.cn
+"""
+import torch
+from ...basic.layers import LR, MLP,CrossNetV2, CrossNetMix, EmbeddingLayer
+
+class DCNv2(torch.nn.Module):
+    """Deep & Cross Network with a mixture of low-rank architecture
+
+    Args:
+        features (list[Feature Class]): training by the whole module.
+        n_cross_layers (int) : the number of layers of feature intersection layers
+        mlp_params (dict): the params of the last MLP module, keys include:`{"dims":list, "activation":str, "dropout":float, "output_layer":bool`}
+        use_low_rank_mixture (bool): True, whether to use a mixture of low-rank architecture
+        low_rank (int): the rank size of low-rank matrices
+        num_experts (int): the number of expert networks
+    """
+    def __init__(self,
+                 features,
+                 n_cross_layers,
+                 mlp_params,
+                 model_structure="parallel",
+                 use_low_rank_mixture=True,
+                 low_rank=32,
+                 num_experts=4,
+                 **kwargs):
+        super(DCNv2, self).__init__()
+        self.features = features
+        self.dims = sum([fea.embed_dim for fea in features])
+        self.embedding = EmbeddingLayer(features)
+        if use_low_rank_mixture:
+            self.crossnet = CrossNetMix(self.dims, n_cross_layers, low_rank=low_rank, num_experts=num_experts)
+        else:
+            self.crossnet = CrossNetV2(self.dims, n_cross_layers)
+        self.model_structure = model_structure
+        assert self.model_structure in ["crossnet_only", "stacked", "parallel"], \
+               "model_structure={} not supported!".format(self.model_structure)
+        if self.model_structure == "stacked":
+            self.stacked_dnn = MLP(self.dims,
+                                   output_layer=False,
+                                   ** mlp_params)
+            final_dim = mlp_params["dims"][-1]
+        if self.model_structure == "parallel":
+            self.parallel_dnn =  MLP(self.dims,
+                                     output_layer = False,
+                                   ** mlp_params)
+            final_dim = mlp_params["dims"][-1] + self.dims
+        if self.model_structure == "crossnet_only": # only CrossNet
+            final_dim = self.dims
+        self.linear = LR(final_dim)
+
+
+    def forward(self, x):
+        embed_x = self.embedding(x, self.features, squeeze_dim=True)
+        cross_out = self.crossnet(embed_x)
+        if self.model_structure == "crossnet_only":
+            final_out = cross_out
+        elif self.model_structure == "stacked":
+            final_out = self.stacked_dnn(cross_out)
+        elif self.model_structure == "parallel":
+            dnn_out = self.parallel_dnn(embed_x)
+            final_out = torch.cat([cross_out, dnn_out], dim=1)
+        y_pred = self.linear(final_out)
+        y_pred =  torch.sigmoid(y_pred.squeeze(1))
+        return y_pred

torch_rechub/models/ranking/deepffm.py

@@ -0,0 +1,123 @@
+"""
+Date: created on 31/07/2022
+References: 
+    paper: FAT-DeepFFM: Field Attentive Deep Field-aware Factorization Machine
+    url: https://arxiv.org/abs/1905.06336    
+Authors: Bo Kang, klinux@live.com
+"""
+
+import torch
+import torch.nn as nn
+
+from ...basic.layers import CEN, EmbeddingLayer, FFM, MLP
+
+
+class DeepFFM(nn.Module):
+    """The DeepFFM model, mentioned on the `webpage 
+    <https://cs.nju.edu.cn/31/60/c1654a209248/page.htm>` which is the first 
+    work that introduces FFM model into neural CTR system. It is also described 
+    in the `FAT-DeepFFM paper <https://arxiv.org/abs/1905.06336>`. 
+
+    Args:
+        linear_features (list): the list of `Feature Class`, fed to the linear module.
+        cross_features (list): the list of `Feature Class`, fed to the ffm module.
+        embed_dim (int): the dimensionality of categorical value embedding.        
+        mlp_params (dict): the params of the last MLP module, keys include:`{"dims":list, "activation":str, "dropout":float, "output_layer":bool`}
+    """
+    def __init__(self, linear_features, cross_features, embed_dim, mlp_params):
+        super().__init__()        
+        self.linear_features = linear_features
+        self.cross_features = cross_features        
+
+        self.num_fields = len(cross_features)
+        self.num_field_cross = self.num_fields * (self.num_fields - 1) // 2
+
+        self.ffm = FFM(num_fields=self.num_fields, reduce_sum=False)                
+        self.mlp_out = MLP(self.num_field_cross * embed_dim, **mlp_params)
+
+        self.linear_embedding = EmbeddingLayer(linear_features)
+        self.ffm_embedding = EmbeddingLayer(cross_features)
+
+        self.b =torch.nn.Parameter(torch.zeros(1))
+        
+        # This keeping constant value in module on correct device 
+        # url: https://discuss.pytorch.org/t/keeping-constant-value-in-module-on-correct-device/10129
+        fields_offset = torch.arange(0, self.num_fields, dtype=torch.long)
+        self.register_buffer('fields_offset', fields_offset)
+
+
+    def forward(self, x):
+        # compute scores from the linear part of the model, where input is the raw features (Eq. 5, FAT-DeepFFM)
+        y_linear = self.linear_embedding(x, self.linear_features, squeeze_dim=True).sum(1, keepdim=True) #[batch_size, 1]        
+                
+        # gather the embeddings. Each feature value corresponds to multiple embeddings, with multiplicity equal to number of features/fields.
+        # output shape [batch_size, num_field, num_field, emb_dim]
+        x_ffm = {fea.name: x[fea.name].unsqueeze(1) * self.num_fields  + self.fields_offset for fea in self.cross_features}        
+        input_ffm = self.ffm_embedding(x_ffm, self.cross_features, squeeze_dim=False)         
+
+        # compute second order field-aware feature crossings, output shape [batch_size, num_field_cross, emb_dim]
+        em = self.ffm(input_ffm)                                                            
+
+        # compute scores from the ffm part of the model, output shape [batch_size, 1]
+        y_ffm = self.mlp_out(em.flatten(start_dim=1))
+
+        # compute final prediction
+        y = y_linear + y_ffm
+        return torch.sigmoid(y.squeeze(1) + self.b)
+
+
+class FatDeepFFM(nn.Module):
+    """The FAT-DeepFFM model, mentioned in the `FAT-DeepFFM paper
+    <https://arxiv.org/abs/1905.06336>`. It combines DeepFFM with
+    Compose-Excitation Network (CENet) field attention mechanism
+    to highlight the importance of second-order feature crosses.
+
+    Args:
+        linear_features (list): the list of `Feature Class`, fed to the linear module.
+        cross_features (list): the list of `Feature Class`, fed to the ffm module.
+        embed_dim (int): the dimensionality of categorical value embedding.
+        reduction_ratio (int): the between the dimensions of input layer and hidden layer of the CEN MLP module.
+        mlp_params (dict): the params of the last MLP module, keys include:`{"dims":list, "activation":str, "dropout":float, "output_layer":bool`}
+    """
+
+    def __init__(self, linear_features, cross_features, embed_dim, reduction_ratio, mlp_params):
+        super().__init__()        
+        self.linear_features = linear_features
+        self.cross_features = cross_features        
+
+        self.num_fields = len(cross_features)
+        self.num_field_cross = self.num_fields * (self.num_fields - 1) // 2
+
+        self.ffm = FFM(num_fields=self.num_fields, reduce_sum=False)        
+        self.cen = CEN(embed_dim, self.num_field_cross, reduction_ratio)
+        self.mlp_out = MLP(self.num_field_cross * embed_dim, **mlp_params)
+
+        self.linear_embedding = EmbeddingLayer(linear_features)
+        self.ffm_embedding = EmbeddingLayer(cross_features)
+
+        self.b =torch.nn.Parameter(torch.zeros(1))
+
+        fields_offset = torch.arange(0, self.num_fields, dtype=torch.long)
+        self.register_buffer('fields_offset', fields_offset)
+
+    def forward(self, x):
+        # compute scores from the linear part of the model, where input is the raw features (Eq. 5, FAT-DeepFFM)
+        y_linear = self.linear_embedding(x, self.linear_features, squeeze_dim=True).sum(1, keepdim=True) #[batch_size, 1]        
+        
+        # gather the embeddings. Each feature value corresponds to multiple embeddings, with multiplicity is equal to the number of features/fields.
+        # output shape [batch_size, num_field, num_field, emb_dim]
+        x_ffm = {fea.name: x[fea.name].unsqueeze(1) * self.num_fields + self.fields_offset for fea in self.cross_features}        
+        input_ffm = self.ffm_embedding(x_ffm, self.cross_features, squeeze_dim=False)
+
+        # compute second order field-aware feature crossings, output shape [batch_size, num_field_cross, emb_dim]
+        em = self.ffm(input_ffm)
+
+        # rescale FFM embeddings with field attention (Eq.10), output shape [batch_size, num_field_cross * emb_dim]                                                           
+        aem = self.cen(em)
+        
+        # compute scores from the ffm part of the model, output shape [batch_size, 1]
+        y_ffm = self.mlp_out(aem)                                                           
+                
+        # compute final prediction
+        y = y_linear + y_ffm
+        return torch.sigmoid(y.squeeze(1) + self.b)

torch_rechub/models/ranking/deepfm.py

@@ -1,42 +1,42 @@
-"""
-Date: create on 22/04/2022
-References: 
-    paper: (IJCAI'2017) DeepFM: A Factorization-Machine based Neural Network for CTR Prediction 
-    url: https://arxiv.org/abs/1703.04247
-Authors: Mincai Lai, laimincai@shanghaitech.edu.cn
-"""
-
-import torch
-
-from ...basic.layers import FM, MLP, LR, EmbeddingLayer
-
-
-class DeepFM(torch.nn.Module):
-    """Deep Factorization Machine Model
-
-    Args:
-        deep_features (list): the list of `Feature Class`, training by the deep part module.
-        fm_features (list): the list of `Feature Class`, training by the fm part module.
-        mlp_params (dict): the params of the last MLP module, keys include:`{"dims":list, "activation":str, "dropout":float, "output_layer":bool`}
-    """
-
-    def __init__(self, deep_features, fm_features, mlp_params):
-        super(DeepFM, self).__init__()
-        self.deep_features = deep_features
-        self.fm_features = fm_features
-        self.deep_dims = sum([fea.embed_dim for fea in deep_features])
-        self.fm_dims = sum([fea.embed_dim for fea in fm_features])
-        self.linear = LR(self.fm_dims)  # 1-odrder interaction
-        self.fm = FM(reduce_sum=True)  # 2-odrder interaction
-        self.embedding = EmbeddingLayer(deep_features + fm_features)
-        self.mlp = MLP(self.deep_dims, **mlp_params)
-
-    def forward(self, x):
-        input_deep = self.embedding(x, self.deep_features, squeeze_dim=True)  #[batch_size, deep_dims]
-        input_fm = self.embedding(x, self.fm_features, squeeze_dim=False)  #[batch_size, num_fields, embed_dim]
-
-        y_linear = self.linear(input_fm.flatten(start_dim=1))
-        y_fm = self.fm(input_fm)
-        y_deep = self.mlp(input_deep)  #[batch_size, 1]
-        y = y_linear + y_fm + y_deep
+"""
+Date: create on 22/04/2022
+References: 
+    paper: (IJCAI'2017) DeepFM: A Factorization-Machine based Neural Network for CTR Prediction 
+    url: https://arxiv.org/abs/1703.04247
+Authors: Mincai Lai, laimincai@shanghaitech.edu.cn
+"""
+
+import torch
+
+from ...basic.layers import FM, MLP, LR, EmbeddingLayer
+
+
+class DeepFM(torch.nn.Module):
+    """Deep Factorization Machine Model
+
+    Args:
+        deep_features (list): the list of `Feature Class`, training by the deep part module.
+        fm_features (list): the list of `Feature Class`, training by the fm part module.
+        mlp_params (dict): the params of the last MLP module, keys include:`{"dims":list, "activation":str, "dropout":float, "output_layer":bool`}
+    """
+
+    def __init__(self, deep_features, fm_features, mlp_params):
+        super(DeepFM, self).__init__()
+        self.deep_features = deep_features
+        self.fm_features = fm_features
+        self.deep_dims = sum([fea.embed_dim for fea in deep_features])
+        self.fm_dims = sum([fea.embed_dim for fea in fm_features])
+        self.linear = LR(self.fm_dims)  # 1-odrder interaction
+        self.fm = FM(reduce_sum=True)  # 2-odrder interaction
+        self.embedding = EmbeddingLayer(deep_features + fm_features)
+        self.mlp = MLP(self.deep_dims, **mlp_params)
+
+    def forward(self, x):
+        input_deep = self.embedding(x, self.deep_features, squeeze_dim=True)  #[batch_size, deep_dims]
+        input_fm = self.embedding(x, self.fm_features, squeeze_dim=False)  #[batch_size, num_fields, embed_dim]
+
+        y_linear = self.linear(input_fm.flatten(start_dim=1))
+        y_fm = self.fm(input_fm)
+        y_deep = self.mlp(input_deep)  #[batch_size, 1]
+        y = y_linear + y_fm + y_deep
         return torch.sigmoid(y.squeeze(1))

torch_rechub/models/ranking/dien.py

@@ -0,0 +1,191 @@
+"""
+Date: create on 01/05/2024
+References: 
+    paper: (AAAI'2019) Deep Interest Evolution Network for Click-Through Rate Prediction
+    url: https://arxiv.org/pdf/1809.03672
+Authors: Tao Fan, thisisevy@foxmail.com
+"""
+
+import torch
+from torch import nn
+from torch.nn import Parameter, init
+
+from ...basic.layers import MLP, EmbeddingLayer
+
+
+class AUGRU(nn.Module):
+
+    def __init__(self, embed_dim):
+        super(AUGRU, self).__init__()
+        self.embed_dim = embed_dim
+        # 初始化AUGRU单元
+        self.augru_cell = AUGRU_Cell(self.embed_dim)
+
+    def forward(self, x, item):
+        '''
+        :param x: 输入的序列向量，维度为 [ batch_size, seq_lens, embed_dim ]
+        :param item: 目标物品的向量
+        :return: outs: 所有AUGRU单元输出的隐藏向量[ batch_size, seq_lens, embed_dim ]
+                 h: 最后一个AUGRU单元输出的隐藏向量[ batch_size, embed_dim ]
+        '''
+        outs = []
+        h = None
+        # 开始循环，x.shape[1]是序列的长度
+        for i in range(x.shape[1]):
+            if h == None:
+                # 初始化第一层的输入h
+                h = Parameter(torch.rand(x.shape[0], self.embed_dim).to(x.device))
+            h = self.augru_cell(x[:, i], h, item)
+            outs.append(torch.unsqueeze(h, dim=1))
+        outs = torch.cat(outs, dim=1)
+        return outs, h
+
+
+# AUGRU单元
+class AUGRU_Cell(nn.Module):
+
+    def __init__(self, embed_dim):
+        """
+        :param embed_dim: 输入向量的维度
+        """
+        super(AUGRU_Cell, self).__init__()
+
+        # 初始化更新门的模型参数
+        self.Wu = Parameter(torch.rand(embed_dim, embed_dim))
+        self.Uu = Parameter(torch.rand(embed_dim, embed_dim))
+        self.bu = init.xavier_uniform_(Parameter(torch.empty(1, embed_dim)))
+
+        # 初始化重置门的模型参数
+        self.Wr = init.xavier_uniform_(Parameter(torch.empty(embed_dim, embed_dim)))
+        self.Ur = init.xavier_uniform_(Parameter(torch.empty(embed_dim, embed_dim)))
+        self.br = init.xavier_uniform_(Parameter(torch.empty(1, embed_dim)))
+
+        # 初始化计算h~的模型参数
+        self.Wh = init.xavier_uniform_(Parameter(torch.empty(embed_dim, embed_dim)))
+        self.Uh = init.xavier_uniform_(Parameter(torch.empty(embed_dim, embed_dim)))
+        self.bh = init.xavier_uniform_(Parameter(torch.empty(1, embed_dim)))
+
+        # 初始化注意计算里的模型参数
+        self.Wa = init.xavier_uniform_(Parameter(torch.empty(embed_dim, embed_dim)))
+
+    # 注意力的计算
+    def attention(self, x, item):
+        '''
+        :param x: 输入的序列中第t个向量 [ batch_size, embed_dim ]
+        :param item: 目标物品的向量 [ batch_size, embed_dim ]
+        :return: 注意力权重 [ batch_size, 1 ]
+        '''
+        hW = torch.matmul(x, self.Wa)
+        hWi = torch.sum(hW * item, dim=1)
+        hWi = torch.unsqueeze(hWi, 1)
+        return torch.softmax(hWi, dim=1)
+
+    def forward(self, x, h_1, item):
+        '''
+        :param x:  输入的序列中第t个物品向量 [ batch_size, embed_dim ]
+        :param h_1:  上一个AUGRU单元输出的隐藏向量 [ batch_size, embed_dim ]
+        :param item: 目标物品的向量 [ batch_size, embed_dim ]
+        :return: h 当前层输出的隐藏向量 [ batch_size, embed_dim ]
+        '''
+        # [ batch_size, embed_dim ]
+        u = torch.sigmoid(torch.matmul(x, self.Wu) + torch.matmul(h_1, self.Uu) + self.bu)
+        # [ batch_size, embed_dim ]
+        r = torch.sigmoid(torch.matmul(x, self.Wr) + torch.matmul(h_1, self.Ur) + self.br)
+        # [ batch_size, embed_dim ]
+        h_hat = torch.tanh(torch.matmul(x, self.Wh) + r * torch.matmul(h_1, self.Uh) + self.bh)
+        # [ batch_size, 1 ]
+        a = self.attention(x, item)
+        # [ batch_size, embed_dim ]
+        u_hat = a * u
+        # [ batch_size, embed_dim ]
+        h = (1 - u_hat) * h_1 + u_hat * h_hat
+        # [ batch_size, embed_dim ]
+        return h
+
+
+class DIEN(nn.Module):
+    """Deep Interest Evolution Network
+    Args:
+        features (list): the list of `Feature Class`. training by MLP. It means the user profile features and context features in origin paper, exclude history and target features.
+        history_features (list): the list of `Feature Class`,training by ActivationUnit. It means the user behaviour sequence features, eg.item id sequence, shop id sequence.
+        target_features (list): the list of `Feature Class`, training by ActivationUnit. It means the target feature which will execute target-attention with history feature.
+        mlp_params (dict): the params of the last MLP module, keys include:`{"dims":list, "activation":str, "dropout":float, "output_layer":bool`}
+        history_labels (list): the list of history_features whether it is clicked history or not. It should be 0 or 1.
+        alpha (float): the weighting of auxiliary loss.
+    """
+
+    def __init__(self, features, history_features, target_features, mlp_params, history_labels,
+                 alpha=0.2):
+        super().__init__()
+        self.alpha = alpha  # 计算辅助损失函数时的权重
+        self.features = features
+        self.history_features = history_features
+        self.target_features = target_features
+        self.num_history_features = len(history_features)
+        self.all_dims = sum([fea.embed_dim for fea in features + history_features + target_features])
+        # self.GRU = nn.GRU(batch_first=True)
+        self.embedding = EmbeddingLayer(features + history_features + target_features)
+        self.interest_extractor_layers = nn.ModuleList(
+            [nn.GRU(fea.embed_dim, fea.embed_dim, batch_first=True) for fea in self.history_features])
+        self.interest_evolving_layers = nn.ModuleList(
+            [AUGRU(fea.embed_dim) for fea in self.history_features])
+
+        self.mlp = MLP(self.all_dims, activation="dice", **mlp_params)
+        self.history_labels = torch.Tensor(history_labels)
+        self.BCELoss = nn.BCELoss()
+        # # 注意力计算中的线性层
+        # self.attention_liner = nn.Linear(self.embed_dim, t)
+        # # AFM公式中的h
+        # self.h = init.xavier_uniform_(Parameter(torch.empty(t, 1)))
+        # # AFM公式中的p
+        # self.p = init.xavier_uniform_(Parameter(torch.empty(self.embed_dim, 1)))
+
+    def auxiliary(self, outs, history_features, history_labels):
+        '''
+        :param history_features: 历史序列物品的向量 [ batch_size, len_seqs, dim ]
+        :param outs: 兴趣抽取层GRU网络输出的outs [ batch_size, len_seqs, dim ]
+        :param history_labels: 历史序列物品标注 [ batch_size, len_seqs, 1 ]
+        :return: 辅助损失函数
+        '''
+        # [ batch_size * len_seqs, dim ]
+        history_features = history_features.reshape(-1, history_features.shape[2])
+        # [ batch_size * len_seqs, dim ]
+        outs = outs.reshape(-1, outs.shape[2])
+        # [ batch_size * len_seqs ]
+        out = torch.sum(outs * history_features, dim=1)
+        # [ batch_size * len_seqs, 1 ]
+        out = torch.unsqueeze(torch.sigmoid(out), 1)
+        # [ batch_size * len_seqs,1 ]
+        history_labels = history_labels.reshape(-1, 1).float()
+        return self.BCELoss(out, history_labels)
+
+    def forward(self, x):
+        embed_x_features = self.embedding(x, self.features)  # (batch_size, num_features, emb_dim)
+        embed_x_history = self.embedding(
+            x, self.history_features)  # (batch_size, num_history_features, seq_length, emb_dim)
+        embed_x_target = self.embedding(x, self.target_features)  # (batch_size, num_target_features, emb_dim)
+
+        interest_extractor = []
+        auxi_loss = 0
+        for i in range(self.num_history_features):
+            outs, _ = self.interest_extractor_layers[i](embed_x_history[:, i, :, :])
+            # 利用GRU输出的outs得到辅助损失函数
+            auxi_loss += self.auxiliary(outs, embed_x_history[:, i, :, :], self.history_labels)
+            interest_extractor.append(outs.unsqueeze(1))  # (batch_size, 1, seq_length, emb_dim)
+        interest_extractor = torch.cat(interest_extractor,
+                                       dim=1)  # (batch_size, num_history_features, seq_length, emb_dim)
+        interest_evolving = []
+        for i in range(self.num_history_features):
+            _, h = self.interest_evolving_layers[i](interest_extractor[:, i, :, :], embed_x_target[:, i, :])
+            interest_evolving.append(h.unsqueeze(1))  # (batch_size, 1, emb_dim)
+        interest_evolving = torch.cat(interest_evolving, dim=1)  # (batch_size, num_history_features, emb_dim)
+
+        mlp_in = torch.cat([
+            interest_evolving.flatten(start_dim=1),
+            embed_x_target.flatten(start_dim=1),
+            embed_x_features.flatten(start_dim=1)
+        ],
+            dim=1)  # (batch_size, N)
+        y = self.mlp(mlp_in)
+
+        return torch.sigmoid(y.squeeze(1)), self.alpha * auxi_loss

torch_rechub/models/ranking/din.py

@@ -1,81 +1,91 @@
-"""
-Date: create on 23/04/2022, update on 30/04/2022
-References: 
-    paper: (KDD'2018) Deep Interest Network for Click-Through Rate Prediction 
-    url: https://arxiv.org/abs/1706.06978
-    code: https://github.com/huawei-noah/benchmark/blob/main/FuxiCTR/fuxictr/pytorch/models/DIN.py
-Authors: Mincai Lai, laimincai@shanghaitech.edu.cn
-"""
-
-import torch
-import torch.nn as nn
-
-from ...basic.layers import EmbeddingLayer, MLP
-
-
-class DIN(nn.Module):
-    """Deep Interest Network
-    Args:
-        features (list): the list of `Feature Class`. training by MLP. It means the user profile features and context features in origin paper, exclude history and target features.
-        history_features (list): the list of `Feature Class`,training by ActivationUnit. It means the user behaviour sequence features, eg.item id sequence, shop id sequence.
-        target_features (list): the list of `Feature Class`, training by ActivationUnit. It means the target feature which will execute target-attention with history feature.
-        mlp_params (dict): the params of the last MLP module, keys include:`{"dims":list, "activation":str, "dropout":float, "output_layer":bool`}
-        attention_mlp_params (dict): the params of the ActivationUnit module, keys include:`{"dims":list, "activation":str, "dropout":float, "use_softmax":bool`}
-    """
-
-    def __init__(self, features, history_features, target_features, mlp_params={"dims": [200, 80], "activation": "dice"}, attention_mlp_params={"dims": [36], "activation": "dice"}):
-        super().__init__()
-        self.features = features
-        self.history_features = history_features
-        self.target_features = target_features
-        self.num_history_features = len(history_features)
-        self.all_dims = sum([fea.embed_dim for fea in features + history_features + target_features])
-
-        self.embedding = EmbeddingLayer(features + history_features + target_features)
-        self.attention_layers = nn.ModuleList([ActivationUnit(fea.embed_dim, **attention_mlp_params) for fea in self.history_features])
-        self.mlp = MLP(self.all_dims, **mlp_params)
-
-    def forward(self, x):
-        embed_x_features = self.embedding(x, self.features)  #(batch_size, num_features, emb_dim)
-        embed_x_history = self.embedding(x, self.history_features)  #(batch_size, num_history_features, seq_length, emb_dim)
-        embed_x_target = self.embedding(x, self.target_features)  #(batch_size, num_target_features, emb_dim)
-        attention_pooling = []
-        for i in range(self.num_history_features):
-            attention_seq = self.attention_layers[i](embed_x_history[:, i, :, :], embed_x_target[:, i, :])
-            attention_pooling.append(attention_seq.unsqueeze(1))  #(batch_size, 1, emb_dim)
-        attention_pooling = torch.cat(attention_pooling, dim=1)  #(batch_size, num_history_features, emb_dim)
-
-        mlp_in = torch.cat([attention_pooling.flatten(start_dim=1), embed_x_target.flatten(start_dim=1), embed_x_features.flatten(start_dim=1)], dim=1)  #(batch_size, N)
-
-        y = self.mlp(mlp_in)
-        return torch.sigmoid(y.squeeze(1))
-
-
-class ActivationUnit(nn.Module):
-    """Activation Unit Layer mentioned in DIN paper, it is a Target Attention method.
-
-    Args:
-        embed_dim (int): the length of embedding vector.
-        history (tensor):
-    Shape:
-        - Input: `(batch_size, seq_length, emb_dim)`
-        - Output: `(batch_size, emb_dim)`
-    """
-
-    def __init__(self, emb_dim, dims=[36], activation="dice", use_softmax=False):
-        super(ActivationUnit, self).__init__()
-        self.emb_dim = emb_dim
-        self.use_softmax = use_softmax
-        self.attention = MLP(4 * self.emb_dim, dims, activation=activation)
-
-    def forward(self, history, target):
-        seq_length = history.size(1)
-        target = target.unsqueeze(1).expand(-1, seq_length, -1)  #batch_size,seq_length,emb_dim
-        att_input = torch.cat([target, history, target - history, target * history], dim=-1)  # batch_size,seq_length,4*emb_dim
-        att_weight = self.attention(att_input.view(-1, 4 * self.emb_dim))  #  #(batch_size*seq_length,4*emb_dim)
-        att_weight = att_weight.view(-1, seq_length)  #(batch_size*seq_length, 1) -> (batch_size,seq_length)
-        if self.use_softmax:
-            att_weight = att_weight.softmax(dim=-1)
-        # (batch_size, seq_length, 1) * (batch_size, seq_length, emb_dim)
-        output = (att_weight.unsqueeze(-1) * history).sum(dim=1)  #(batch_size,emb_dim)
-        return output
+"""
+Date: create on 23/04/2022, update on 30/04/2022
+References: 
+    paper: (KDD'2018) Deep Interest Network for Click-Through Rate Prediction 
+    url: https://arxiv.org/abs/1706.06978
+    code: https://github.com/huawei-noah/benchmark/blob/main/FuxiCTR/fuxictr/pytorch/models/DIN.py
+Authors: Mincai Lai, laimincai@shanghaitech.edu.cn
+"""
+
+import torch
+import torch.nn as nn
+
+from ...basic.layers import EmbeddingLayer, MLP
+
+
+class DIN(nn.Module):
+    """Deep Interest Network
+    Args:
+        features (list): the list of `Feature Class`. training by MLP. It means the user profile features and context features in origin paper, exclude history and target features.
+        history_features (list): the list of `Feature Class`,training by ActivationUnit. It means the user behaviour sequence features, eg.item id sequence, shop id sequence.
+        target_features (list): the list of `Feature Class`, training by ActivationUnit. It means the target feature which will execute target-attention with history feature.
+        mlp_params (dict): the params of the last MLP module, keys include:`{"dims":list, "activation":str, "dropout":float, "output_layer":bool`}
+        attention_mlp_params (dict): the params of the ActivationUnit module, keys include:`{"dims":list, "activation":str, "dropout":float, "use_softmax":bool`}
+    """
+
+    def __init__(self, features, history_features, target_features, mlp_params, attention_mlp_params):
+        super().__init__()
+        self.features = features
+        self.history_features = history_features
+        self.target_features = target_features
+        self.num_history_features = len(history_features)
+        self.all_dims = sum([fea.embed_dim for fea in features + history_features + target_features])
+
+        self.embedding = EmbeddingLayer(features + history_features + target_features)
+        self.attention_layers = nn.ModuleList(
+            [ActivationUnit(fea.embed_dim, **attention_mlp_params) for fea in self.history_features])
+        self.mlp = MLP(self.all_dims, activation="dice", **mlp_params)
+
+    def forward(self, x):
+        embed_x_features = self.embedding(x, self.features)  #(batch_size, num_features, emb_dim)
+        embed_x_history = self.embedding(
+            x, self.history_features)  #(batch_size, num_history_features, seq_length, emb_dim)
+        embed_x_target = self.embedding(x, self.target_features)  #(batch_size, num_target_features, emb_dim)
+        attention_pooling = []
+        for i in range(self.num_history_features):
+            attention_seq = self.attention_layers[i](embed_x_history[:, i, :, :], embed_x_target[:, i, :])
+            attention_pooling.append(attention_seq.unsqueeze(1))  #(batch_size, 1, emb_dim)
+        attention_pooling = torch.cat(attention_pooling, dim=1)  #(batch_size, num_history_features, emb_dim)
+
+        mlp_in = torch.cat([
+            attention_pooling.flatten(start_dim=1),
+            embed_x_target.flatten(start_dim=1),
+            embed_x_features.flatten(start_dim=1)
+        ],
+                           dim=1)  #(batch_size, N)
+
+        y = self.mlp(mlp_in)
+        return torch.sigmoid(y.squeeze(1))
+
+
+class ActivationUnit(nn.Module):
+    """Activation Unit Layer mentioned in DIN paper, it is a Target Attention method.
+
+    Args:
+        embed_dim (int): the length of embedding vector.
+        history (tensor):
+    Shape:
+        - Input: `(batch_size, seq_length, emb_dim)`
+        - Output: `(batch_size, emb_dim)`
+    """
+
+    def __init__(self, emb_dim, dims=None, activation="dice", use_softmax=False):
+        super(ActivationUnit, self).__init__()
+        if dims is None:
+            dims = [36]
+        self.emb_dim = emb_dim
+        self.use_softmax = use_softmax
+        self.attention = MLP(4 * self.emb_dim, dims=dims, activation=activation)
+
+    def forward(self, history, target):
+        seq_length = history.size(1)
+        target = target.unsqueeze(1).expand(-1, seq_length, -1)  #batch_size,seq_length,emb_dim
+        att_input = torch.cat([target, history, target - history, target * history],
+                              dim=-1)  # batch_size,seq_length,4*emb_dim
+        att_weight = self.attention(att_input.view(-1, 4 * self.emb_dim))  #  #(batch_size*seq_length,4*emb_dim)
+        att_weight = att_weight.view(-1, seq_length)  #(batch_size*seq_length, 1) -> (batch_size,seq_length)
+        if self.use_softmax:
+            att_weight = att_weight.softmax(dim=-1)
+        # (batch_size, seq_length, 1) * (batch_size, seq_length, emb_dim)
+        output = (att_weight.unsqueeze(-1) * history).sum(dim=1)  #(batch_size,emb_dim)
+        return output