easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/model/autoint.py

@@ -0,0 +1,73 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import tensorflow as tf
+
+from easy_rec.python.layers import multihead_attention
+from easy_rec.python.model.rank_model import RankModel
+
+from easy_rec.python.protos.autoint_pb2 import AutoInt as AutoIntConfig  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class AutoInt(RankModel):
+
+  def __init__(self,
+               model_config,
+               feature_configs,
+               features,
+               labels=None,
+               is_training=False):
+    super(AutoInt, self).__init__(model_config, feature_configs, features,
+                                  labels, is_training)
+    assert self._model_config.WhichOneof('model') == 'autoint', \
+        'invalid model config: %s' % self._model_config.WhichOneof('model')
+    self._features, _ = self._input_layer(self._feature_dict, 'all')
+    self._feature_num = len(self._model_config.feature_groups[0].feature_names)
+    self._seq_key_num = 0
+    if len(self._model_config.feature_groups[0].sequence_features) > 0:
+      for seq_fea in self._model_config.feature_groups[0].sequence_features:
+        for seq_att in seq_fea.seq_att_map:
+          self._feature_num += len(seq_att.hist_seq)
+          self._seq_key_num += len(seq_att.key)
+    self._model_config = self._model_config.autoint
+    assert isinstance(self._model_config, AutoIntConfig)
+
+    fea_emb_dim_list = []
+    for feature_config in feature_configs:
+      fea_emb_dim_list.append(feature_config.embedding_dim)
+    assert len(set(fea_emb_dim_list)) == 1 and len(fea_emb_dim_list) == self._feature_num, \
+        'AutoInt requires that all feature dimensions must be consistent.'
+
+    self._d_model = fea_emb_dim_list[0]
+    self._head_num = self._model_config.multi_head_num
+    self._head_size = self._model_config.multi_head_size
+
+  def build_predict_graph(self):
+    logging.info('feature_num: {0}'.format(self._feature_num))
+
+    attention_fea = tf.reshape(
+        self._features,
+        shape=[-1, self._feature_num + self._seq_key_num, self._d_model])
+
+    for i in range(self._model_config.interacting_layer_num):
+      attention_layer = multihead_attention.MultiHeadAttention(
+          head_num=self._head_num,
+          head_size=self._head_size,
+          l2_reg=self._l2_reg,
+          use_res=True,
+          name='multi_head_self_attention_layer_%d' % i)
+      attention_fea = attention_layer(attention_fea)
+
+    attention_fea = tf.reshape(
+        attention_fea,
+        shape=[-1, attention_fea.shape[1] * attention_fea.shape[2]])
+
+    final = tf.layers.dense(attention_fea, self._num_class, name='output')
+
+    self._add_to_prediction_dict(final)
+
+    return self._prediction_dict

easy_rec/python/model/cmbf.py

@@ -0,0 +1,47 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import tensorflow as tf
+
+from easy_rec.python.layers import cmbf
+from easy_rec.python.layers import dnn
+from easy_rec.python.model.rank_model import RankModel
+
+from easy_rec.python.protos.cmbf_pb2 import CMBF as CMBFConfig  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class CMBF(RankModel):
+  """CMBF: Cross-Modal-Based Fusion Recommendation Algorithm.
+
+  This is almost an exact implementation of the original CMBF model.
+  See the original paper:
+  https://www.mdpi.com/1424-8220/21/16/5275
+  """
+
+  def __init__(self,
+               model_config,
+               feature_configs,
+               features,
+               labels=None,
+               is_training=False):
+    super(CMBF, self).__init__(model_config, feature_configs, features, labels,
+                               is_training)
+    assert self._model_config.WhichOneof('model') == 'cmbf', (
+        'invalid model config: %s' % self._model_config.WhichOneof('model'))
+
+    self._cmbf_layer = cmbf.CMBF(model_config, feature_configs, features,
+                                 self._model_config.cmbf.config,
+                                 self._input_layer)
+    self._model_config = self._model_config.cmbf
+
+  def build_predict_graph(self):
+    hidden = self._cmbf_layer(self._is_training, l2_reg=self._l2_reg)
+    final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg,
+                              'final_dnn', self._is_training)
+    all_fea = final_dnn_layer(hidden)
+
+    final = tf.layers.dense(all_fea, self._num_class, name='output')
+    self._add_to_prediction_dict(final)
+    return self._prediction_dict

easy_rec/python/model/collaborative_metric_learning.py

@@ -0,0 +1,182 @@
+import tensorflow as tf
+
+from easy_rec.python.core.metrics import metric_learning_average_precision_at_k
+from easy_rec.python.core.metrics import metric_learning_recall_at_k
+from easy_rec.python.layers import dnn
+from easy_rec.python.layers.common_layers import highway
+from easy_rec.python.loss.circle_loss import circle_loss
+from easy_rec.python.loss.multi_similarity import ms_loss
+from easy_rec.python.model.easy_rec_model import EasyRecModel
+from easy_rec.python.protos.loss_pb2 import LossType
+from easy_rec.python.utils.activation import gelu
+from easy_rec.python.utils.proto_util import copy_obj
+
+from easy_rec.python.protos.collaborative_metric_learning_pb2 import CoMetricLearningI2I as MetricLearningI2IConfig  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class CoMetricLearningI2I(EasyRecModel):
+
+  def __init__(
+      self,
+      model_config,  # pipeline.model_config
+      feature_configs,  # pipeline.feature_configs
+      features,  # same as model_fn input
+      labels=None,
+      is_training=False):
+    super(CoMetricLearningI2I, self).__init__(model_config, feature_configs,
+                                              features, labels, is_training)
+    model = self._model_config.WhichOneof('model')
+    assert model == 'metric_learning', 'invalid model config: %s' % model
+
+    self._loss_type = self._model_config.loss_type
+    loss_type_name = LossType.Name(self._loss_type).lower()
+
+    self._model_config = self._model_config.metric_learning
+    assert isinstance(self._model_config, MetricLearningI2IConfig)
+
+    model_loss = self._model_config.WhichOneof('loss').lower()
+    assert model_loss == loss_type_name, 'invalid loss type: %s' % model_loss
+
+    if self._loss_type == LossType.CIRCLE_LOSS:
+      self.loss = self._model_config.circle_loss
+    elif self._loss_type == LossType.MULTI_SIMILARITY_LOSS:
+      self.loss = self._model_config.multi_similarity_loss
+    else:
+      raise ValueError('unsupported loss type: %s' %
+                       LossType.Name(self._loss_type))
+
+    if not self.has_backbone:
+      self._highway_features = {}
+      self._highway_num = len(self._model_config.highway)
+      for _id in range(self._highway_num):
+        highway_cfg = self._model_config.highway[_id]
+        highway_feature, _ = self._input_layer(self._feature_dict,
+                                               highway_cfg.input)
+        self._highway_features[highway_cfg.input] = highway_feature
+
+      self.input_features = []
+      if self._model_config.HasField('input'):
+        input_feature, _ = self._input_layer(self._feature_dict,
+                                             self._model_config.input)
+        self.input_features.append(input_feature)
+
+      self.dnn = copy_obj(self._model_config.dnn)
+
+    if self._labels is not None:
+      if self._model_config.HasField('session_id'):
+        self.session_ids = self._labels.pop(self._model_config.session_id)
+      else:
+        self.session_ids = None
+
+      assert len(self._labels) > 0
+      self.labels = list(self._labels.values())[0]
+
+    if self._model_config.HasField('sample_id'):
+      self.sample_id = self._model_config.sample_id
+    else:
+      self.sample_id = None
+
+  def build_predict_graph(self):
+    if self.has_backbone:
+      tower_emb = self.backbone
+    else:
+      for _id in range(self._highway_num):
+        highway_cfg = self._model_config.highway[_id]
+        highway_fea = tf.layers.batch_normalization(
+            self._highway_features[highway_cfg.input],
+            training=self._is_training,
+            trainable=True,
+            name='highway_%s_bn' % highway_cfg.input)
+        highway_fea = highway(
+            highway_fea,
+            highway_cfg.emb_size,
+            activation=gelu,
+            scope='highway_%s' % _id)
+        print('highway_fea: ', highway_fea)
+        self.input_features.append(highway_fea)
+
+      feature = tf.concat(self.input_features, axis=1)
+
+      num_dnn_layer = len(self.dnn.hidden_units)
+      last_hidden = self.dnn.hidden_units.pop()
+      dnn_net = dnn.DNN(self.dnn, self._l2_reg, 'dnn', self._is_training)
+      net_output = dnn_net(feature)
+      tower_emb = tf.layers.dense(
+          inputs=net_output,
+          units=last_hidden,
+          kernel_regularizer=self._l2_reg,
+          name='dnn/dnn_%d' % (num_dnn_layer - 1))
+
+    if self._model_config.output_l2_normalized_emb:
+      norm_emb = tf.nn.l2_normalize(tower_emb, axis=-1)
+      self._prediction_dict['norm_emb'] = norm_emb
+      self._prediction_dict['norm_embedding'] = tf.reduce_join(
+          tf.as_string(norm_emb), axis=-1, separator=',')
+
+    self._prediction_dict['float_emb'] = tower_emb
+    self._prediction_dict['embedding'] = tf.reduce_join(
+        tf.as_string(tower_emb), axis=-1, separator=',')
+    if self.sample_id is not None and self.sample_id in self._feature_dict:
+      self._prediction_dict['sample_id'] = tf.identity(
+          self._feature_dict[self.sample_id])
+    return self._prediction_dict
+
+  def build_loss_graph(self):
+    emb = self._prediction_dict['float_emb']
+    emb_normed = self._model_config.output_l2_normalized_emb
+    norm_emb = self._prediction_dict['norm_emb'] if emb_normed else emb
+    if self._loss_type == LossType.CIRCLE_LOSS:
+      self._loss_dict['circle_loss'] = circle_loss(
+          norm_emb,
+          self.labels,
+          self.session_ids,
+          self.loss.margin,
+          self.loss.gamma,
+          embed_normed=emb_normed)
+    elif self._loss_type == LossType.MULTI_SIMILARITY_LOSS:
+      self._loss_dict['ms_loss'] = ms_loss(
+          norm_emb,
+          self.labels,
+          self.session_ids,
+          self.loss.alpha,
+          self.loss.beta,
+          self.loss.lamb,
+          self.loss.eps,
+          embed_normed=emb_normed)
+    else:
+      raise ValueError('invalid loss type: %s' % LossType.Name(self._loss_type))
+
+    return self._loss_dict
+
+  def get_outputs(self):
+    outputs = ['embedding', 'float_emb']
+    if self.sample_id is not None and 'sample_id' in self._prediction_dict:
+      outputs.append('sample_id')
+    if self._model_config.output_l2_normalized_emb:
+      outputs.append('norm_embedding')
+      outputs.append('norm_emb')
+    return outputs
+
+  def build_metric_graph(self, eval_config):
+    metric_dict = {}
+    recall_at_k = []
+    precision_at_k = []
+    for metric in eval_config.metrics_set:
+      if metric.WhichOneof('metric') == 'recall_at_topk':
+        recall_at_k.append(metric.recall_at_topk.topk)
+      elif metric.WhichOneof('metric') == 'precision_at_topk':
+        precision_at_k.append(metric.precision_at_topk.topk)
+
+    emb = self._prediction_dict['float_emb']
+    if len(recall_at_k) > 0:
+      metric_dict.update(
+          metric_learning_recall_at_k(recall_at_k, emb, self.labels,
+                                      self.session_ids))
+    if len(precision_at_k) > 0:
+      metric_dict.update(
+          metric_learning_average_precision_at_k(precision_at_k, emb,
+                                                 self.labels, self.session_ids))
+    return metric_dict

easy_rec/python/model/custom_model.py

@@ -0,0 +1,323 @@
+# easy_rec/python/model/custom_model.py
+import os
+import sys
+
+import six
+import tensorflow as tf
+
+from easy_rec.python.builders import loss_builder
+from easy_rec.python.compat import regularizers
+from easy_rec.python.feature_column.feature_column import FeatureColumnParser
+from easy_rec.python.model.easy_rec_model import EasyRecModel
+from easy_rec.python.protos.deepfm_pb2 import DeepFM as DeepFMConfig
+# from easy_rec.python.protos.easy_rec_model_pb2 import LossType
+from easy_rec.python.protos.loss_pb2 import LossType
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class MultiHeadAttention(tf.compat.v1.keras.layers.Layer):
+
+  def __init__(self, num_heads, d_model):
+    super(MultiHeadAttention, self).__init__()
+    self.num_heads = num_heads
+    self.d_model = d_model
+
+    assert d_model % num_heads == 0
+
+    self.depth = d_model // num_heads
+
+    self.wq = tf.compat.v1.keras.layers.Dense(d_model)
+    self.wk = tf.compat.v1.keras.layers.Dense(d_model)
+    self.wv = tf.compat.v1.keras.layers.Dense(d_model)
+
+    self.dense = tf.compat.v1.keras.layers.Dense(d_model)
+
+  def split_heads(self, x, batch_size):
+    x = tf.reshape(x, (batch_size, 15, self.num_heads, self.depth))
+    return tf.transpose(x, perm=[0, 2, 1, 3])
+
+  def __call__(self, q, k, v, mask):
+    batch_size = tf.shape(q)[0]
+
+    q = self.wq(q)
+    k = self.wk(k)
+    v = self.wv(v)
+
+    q = self.split_heads(q, batch_size)
+    k = self.split_heads(k, batch_size)
+    v = self.split_heads(v, batch_size)
+
+    scaled_attention, attention_weights = self.scaled_dot_product_attention(
+        q, k, v, mask)
+
+    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])
+    concat_attention = tf.reshape(scaled_attention,
+                                  (batch_size, 15, self.d_model))
+
+    output = self.dense(concat_attention)
+
+    return output, attention_weights
+
+  def scaled_dot_product_attention(self, q, k, v, mask):
+    matmul_qk = tf.matmul(q, k, transpose_b=True)
+
+    dk = tf.cast(tf.shape(k)[-1], tf.float32)
+    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
+
+    if mask is not None:
+      scaled_attention_logits += (mask * -1e9)
+
+    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
+
+    output = tf.matmul(attention_weights, v)
+
+    return output, attention_weights
+
+
+class CustomModel(EasyRecModel):
+
+  def __init__(self,
+               model_config,
+               feature_configs,
+               features,
+               labels=None,
+               is_training=False):
+    """
+        Args:
+          model_config: easy_rec.python.protos.easy_rec_model_pb2.EasyRecModel
+               model_config.custom_model is instance of:
+                     easy_rec.python.protos.easy_rec_model_pb2.CustomModel
+          feature_configs: a collection of easy_rec.python.protos.feature_config.FeatureConfig
+          features: dict of feature tensors, which are described by easy_rec.python.protos.DatasetConfig.input_fields
+          labels: dict of labels tensors, which are described by easy_rec.python.protos.DatasetConfig.label_fields
+        """
+    super(CustomModel, self).__init__(model_config, feature_configs, features,
+                                      labels, is_training)
+    self.drop_out_rate = 0.05
+    self._raw_features, self._raw_feature_lst = self._input_layer(
+        self._feature_dict, 'raw_feature')
+    self._seq_features, _, _ = self._input_layer(
+        self._feature_dict, 'seq_feature', is_combine=False)
+    self._multi_head_1_features, _, _ = self._input_layer(
+        self._feature_dict, 'multi_head_feature_1', is_combine=False)
+    self._multi_head_2_features, _, _ = self._input_layer(
+        self._feature_dict, 'multi_head_feature_2', is_combine=False)
+
+    self._seq_features_concat = self._get_features_concat(
+        self._get_seq_features_reduce(
+            self._seq_features, reduce_type='mean', axis=1, keepdims=False),
+        axis=-1)
+
+    self._multi_head_1_features_concat = self._get_features_concat(
+        self._get_seq_features_reduce(
+            self._multi_head_1_features,
+            reduce_type='mean',
+            axis=1,
+            keepdims=True),
+        axis=1)
+
+    self._multi_head_2_features_concat = self._get_features_concat(
+        self._get_seq_features_reduce(
+            self._multi_head_2_features,
+            reduce_type='mean',
+            axis=1,
+            keepdims=True),
+        axis=1)
+
+    self._multi_head_1_layer = MultiHeadAttention(4, 12)
+    self._multi_head_2_layer = MultiHeadAttention(4, 12)
+
+    self._multi_head_1_output, _ = self._multi_head_1_layer(
+        self._multi_head_1_features_concat, self._multi_head_1_features_concat,
+        self._multi_head_1_features_concat, None)
+
+    self._multi_head_2_output, _ = self._multi_head_2_layer(
+        self._multi_head_2_features_concat, self._multi_head_2_features_concat,
+        self._multi_head_2_features_concat, None)
+
+    self._multi_head_1_output_end = self._get_seq_feature_reduce(
+        self._multi_head_1_output, reduce_type='mean', axis=1, keepdims=False)
+    self._multi_head_2_output_end = self._get_seq_feature_reduce(
+        self._multi_head_2_output, reduce_type='mean', axis=1, keepdims=False)
+
+    self.deep_input = self._get_features_concat([
+        self._raw_features, self._seq_features_concat,
+        self._multi_head_1_output_end, self._multi_head_2_output_end
+    ],
+                                                axis=-1)
+
+  def _get_seq_features_reduce(self, seq_features, reduce_type, axis: int,
+                               keepdims: bool):
+    assert reduce_type in ['mean', 'sum',
+                           'max'], 'reduce_type  must in mean | sum | max'
+    assert axis in [-1, 1, 2], 'axis  must in -1 | 1 | 2'
+    seq_features_reduce = []
+    for feature in seq_features:
+      if reduce_type == 'mean':
+        seq_features_reduce.append(
+            tf.reduce_mean(feature[0], axis=axis, keepdims=keepdims))
+      elif reduce_type == 'sum':
+        seq_features_reduce.append(
+            tf.reduce_sum(feature[0], axis=axis, keepdims=keepdims))
+      elif reduce_type == 'max':
+        seq_features_reduce.append(
+            tf.reduce_max(feature[0], axis=axis, keepdims=keepdims))
+      else:
+        pass
+    return seq_features_reduce
+
+  def _get_seq_feature_reduce(self, seq_feature, reduce_type, axis: int,
+                              keepdims: bool):
+    assert reduce_type in ['mean', 'sum',
+                           'max'], 'reduce_type  must in mean | sum | max'
+    assert axis in [-1, 1, 2], 'axis  must in -1 | 1 | 2'
+    if reduce_type == 'mean':
+      return tf.reduce_mean(seq_feature, axis=axis, keepdims=keepdims)
+    elif reduce_type == 'sum':
+      return tf.reduce_sum(seq_feature, axis=axis, keepdims=keepdims)
+    elif reduce_type == 'max':
+      return tf.reduce_max(seq_feature, axis=axis, keepdims=keepdims)
+    else:
+      pass
+
+  def _get_features_concat(self, features, axis):
+    assert axis in [-1, 1, 2], 'axis  must in -1 | 1 | 2'
+    return tf.concat(features, axis=axis)
+
+  def build_predict_graph(self):
+    # build forward graph
+    dnn_1_list = self.get_layer_1(
+        self.deep_input, '1:64', prefix='dnn_1_1', n=1)
+
+    dnn_1_2_list = self.get_layer_n(
+        dnn_1_list, '1:32', prefix='dnn_1_2', branch_num=2)
+
+    dnn_1_3_list = self.get_layer_n(
+        dnn_1_2_list, '1:16', prefix='dnn_1_3', branch_num=2)
+
+    dnn_1_4_list = self.get_layer_n(
+        dnn_1_3_list, '1:8', prefix='dnn_1_4', branch_num=2)
+    dnn_1_5_list = self.get_layer_n(
+        dnn_1_4_list, '1:4', prefix='dnn_1_5', branch_num=2)
+    dnn_1_concat = tf.concat(dnn_1_5_list, axis=-1, name='dnn_1_concat')
+    dnn_2_1 = tf.keras.layers.Dense(
+        units=32, activation='relu', name=f'dnn_layer_2_1')(
+            dnn_1_concat)
+    if self.drop_out_rate == 0:
+      dnn_2_1_dropout = dnn_2_1
+    else:
+      dnn_2_1_dropout = tf.keras.layers.Dropout(
+          self.drop_out_rate, noise_shape=None, seed=None)(
+              dnn_2_1)
+
+    dnn_2_2 = tf.keras.layers.Dense(
+        units=16, activation='relu', name=f'dnn_layer_2_2')(
+            dnn_2_1_dropout)
+    if self.drop_out_rate == 0:
+      dnn_2_2_dropout = dnn_2_2
+    else:
+      dnn_2_2_dropout = tf.keras.layers.Dropout(
+          self.drop_out_rate, noise_shape=None, seed=None)(
+              dnn_2_2)
+
+    dnn_2_3 = tf.keras.layers.Dense(
+        units=8, activation='relu', name=f'dnn_layer_2_3')(
+            dnn_2_2_dropout)
+    if self.drop_out_rate == 0:
+      dnn_2_3_dropout = dnn_2_3
+    else:
+      dnn_2_3_dropout = tf.keras.layers.Dropout(
+          self.drop_out_rate, noise_shape=None, seed=None)(
+              dnn_2_3)
+
+    dnn_1_sig = tf.keras.layers.Dense(
+        units=1, activation='sigmoid', name='dnn_1_sig')(
+            dnn_2_3_dropout)
+
+    self._prediction_dict['label'] = dnn_1_sig
+    return self._prediction_dict
+
+  def build_loss_graph(self):
+    # assert self._model_config.loss_type == LossType.CLASSIFICATION
+    loss = tf.keras.losses.BinaryFocalCrossentropy(gamma=2, from_logits=False)
+    label = list(self._labels.values())[0]
+
+    self._loss_dict['custom_loss'] = loss(label, self._prediction_dict['label'])
+
+    return self._loss_dict
+
+  def build_metric_graph(self, eval_config):
+    metric_dict = {}
+    num_thresholds = eval_config.metrics_set[0].auc.num_thresholds
+    metric_dict['auc'] = tf.metrics.auc(
+        list(self._labels.values())[0],
+        self._prediction_dict['label'],
+        num_thresholds=num_thresholds)
+    return metric_dict
+
+  def get_outputs(self):
+
+    return ['label']
+
+  def get_layer_1(self, input, dnn_layers, prefix, n=2):
+    output_list = []
+    dnn_layers_list = dnn_layers.split(',')
+    for i in range(n):
+      for j in range(len(dnn_layers_list)):
+        dnn_info_list = dnn_layers_list[j].split(':')
+        if j == 0:
+          deep_layer = tf.keras.layers.Dense(
+              units=int(dnn_info_list[1]),
+              activation='relu'
+              # , kernel_regularizer=tf.keras.regularizers.l1_l2(l1=0.01, l2=0.01)
+              ,
+              name=f'dnn_layer_{prefix}_{i}_{j}')(
+                  input)
+
+        else:
+          deep_layer = tf.keras.layers.Dense(
+              units=int(dnn_info_list[1]),
+              activation='relu'
+              # , kernel_regularizer=tf.keras.regularizers.l1_l2(l1=0.01, l2=0.01)
+              ,
+              name=f'dnn_layer_{prefix}_{i}_{j}')(
+                  deep_layer)
+      output_list.append(deep_layer)
+    return output_list
+
+  def get_layer_n(self, layer_output_list, dnn_layers, prefix, branch_num=2):
+    output_list = []
+    dnn_layers_list = dnn_layers.split(',')
+    for branch in range(branch_num):
+      for i in range(len(layer_output_list)):
+        for j in range(len(dnn_layers_list)):
+          dnn_info_list = dnn_layers_list[j].split(':')
+          if j == 0:
+            deep_layer = tf.keras.layers.Dense(
+                units=int(dnn_info_list[1]),
+                activation='relu'
+                # , kernel_regularizer=tf.keras.regularizers.l1_l2(l1=0.01, l2=0.01)
+                ,
+                name=f'dnn_layer_{prefix}_{branch}_{i}_{j}')(
+                    layer_output_list[i])
+
+          else:
+            deep_layer = tf.keras.layers.Dense(
+                units=int(dnn_info_list[1]),
+                activation='relu'
+                # , kernel_regularizer=tf.keras.regularizers.l1_l2(l1=0.01, l2=0.01)
+                ,
+                name=f'dnn_layer_{prefix}_{branch}_{i}_{j}')(
+                    deep_layer)
+        # deep_layer_end = tf.concat([deep_layer,bundle_info_sum],axis=-1)
+        if self.drop_out_rate == 0:
+          deep_layer_dropout = deep_layer
+        else:
+          deep_layer_dropout = tf.keras.layers.Dropout(
+              self.drop_out_rate, noise_shape=None, seed=None)(
+                  deep_layer)
+        output_list.append(deep_layer_dropout)
+
+    return output_list