easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/model/multi_task_model.py

@@ -0,0 +1,303 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+from collections import OrderedDict
+
+import tensorflow as tf
+
+from easy_rec.python.builders import loss_builder
+from easy_rec.python.layers.dnn import DNN
+from easy_rec.python.model.rank_model import RankModel
+from easy_rec.python.protos import tower_pb2
+from easy_rec.python.protos.easy_rec_model_pb2 import EasyRecModel
+from easy_rec.python.protos.loss_pb2 import LossType
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class MultiTaskModel(RankModel):
+
+  def __init__(self,
+               model_config,
+               feature_configs,
+               features,
+               labels=None,
+               is_training=False):
+    super(MultiTaskModel, self).__init__(model_config, feature_configs,
+                                         features, labels, is_training)
+    self._task_towers = []
+    self._task_num = None
+    self._label_name_dict = {}
+
+  def build_predict_graph(self):
+    if not self.has_backbone:
+      raise NotImplementedError(
+          'method `build_predict_graph` must be implemented when backbone network do not exists'
+      )
+    model = self._model_config.WhichOneof('model')
+    assert model == 'model_params', '`model_params` must be configured'
+    config = self._model_config.model_params
+    for out in config.outputs:
+      self._outputs.append(out)
+
+    self._init_towers(config.task_towers)
+
+    backbone = self.backbone
+    if type(backbone) in (list, tuple):
+      if len(backbone) != len(config.task_towers):
+        raise ValueError(
+            'The number of backbone outputs and task towers must be equal')
+      task_input_list = backbone
+    else:
+      task_input_list = [backbone] * len(config.task_towers)
+
+    tower_features = {}
+    for i, task_tower_cfg in enumerate(config.task_towers):
+      tower_name = task_tower_cfg.tower_name
+      with tf.name_scope(tower_name):
+        if task_tower_cfg.HasField('dnn'):
+          tower_dnn = DNN(
+              task_tower_cfg.dnn,
+              self._l2_reg,
+              name=tower_name,
+              is_training=self._is_training)
+          tower_output = tower_dnn(task_input_list[i])
+        else:
+          tower_output = task_input_list[i]
+        tower_features[tower_name] = tower_output
+
+    tower_outputs = {}
+    relation_features = {}
+    # bayes network
+    for task_tower_cfg in config.task_towers:
+      tower_name = task_tower_cfg.tower_name
+      with tf.name_scope(tower_name):
+        if task_tower_cfg.HasField('relation_dnn'):
+          relation_dnn = DNN(
+              task_tower_cfg.relation_dnn,
+              self._l2_reg,
+              name=tower_name + '/relation_dnn',
+              is_training=self._is_training)
+          tower_inputs = [tower_features[tower_name]]
+          for relation_tower_name in task_tower_cfg.relation_tower_names:
+            tower_inputs.append(relation_features[relation_tower_name])
+          relation_input = tf.concat(
+              tower_inputs, axis=-1, name=tower_name + '/relation_input')
+          relation_fea = relation_dnn(relation_input)
+          relation_features[tower_name] = relation_fea
+        else:
+          relation_fea = tower_features[tower_name]
+
+        output_logits = tf.layers.dense(
+            relation_fea,
+            task_tower_cfg.num_class,
+            kernel_regularizer=self._l2_reg,
+            name=tower_name + '/output')
+      tower_outputs[tower_name] = output_logits
+
+    self._add_to_prediction_dict(tower_outputs)
+    return self._prediction_dict
+
+  def _init_towers(self, task_tower_configs):
+    """Init task towers."""
+    self._task_towers = task_tower_configs
+    self._task_num = len(task_tower_configs)
+    for i, task_tower_config in enumerate(task_tower_configs):
+      assert isinstance(task_tower_config, tower_pb2.TaskTower) or \
+          isinstance(task_tower_config, tower_pb2.BayesTaskTower), \
+          'task_tower_config must be a instance of tower_pb2.TaskTower or tower_pb2.BayesTaskTower'
+      tower_name = task_tower_config.tower_name
+
+      # For label backward compatibility with list
+      if self._labels is not None:
+        if task_tower_config.HasField('label_name'):
+          label_name = task_tower_config.label_name
+        else:
+          # If label name is not specified, task_tower and label will be matched by order
+          label_name = list(self._labels.keys())[i]
+          logging.info('Task Tower [%s] use label [%s]' %
+                       (tower_name, label_name))
+        assert label_name in self._labels, 'label [%s] must exists in labels' % label_name
+        self._label_name_dict[tower_name] = label_name
+
+  def _add_to_prediction_dict(self, output):
+    for task_tower_cfg in self._task_towers:
+      tower_name = task_tower_cfg.tower_name
+      if len(task_tower_cfg.losses) == 0:
+        self._prediction_dict.update(
+            self._output_to_prediction_impl(
+                output[tower_name],
+                loss_type=task_tower_cfg.loss_type,
+                num_class=task_tower_cfg.num_class,
+                suffix='_%s' % tower_name))
+      else:
+        for loss in task_tower_cfg.losses:
+          self._prediction_dict.update(
+              self._output_to_prediction_impl(
+                  output[tower_name],
+                  loss_type=loss.loss_type,
+                  num_class=task_tower_cfg.num_class,
+                  suffix='_%s' % tower_name))
+
+  def build_metric_graph(self, eval_config):
+    """Build metric graph for multi task model."""
+    for task_tower_cfg in self._task_towers:
+      tower_name = task_tower_cfg.tower_name
+      for metric in task_tower_cfg.metrics_set:
+        loss_types = {task_tower_cfg.loss_type}
+        if len(task_tower_cfg.losses) > 0:
+          loss_types = {loss.loss_type for loss in task_tower_cfg.losses}
+        self._metric_dict.update(
+            self._build_metric_impl(
+                metric,
+                loss_type=loss_types,
+                label_name=self._label_name_dict[tower_name],
+                num_class=task_tower_cfg.num_class,
+                suffix='_%s' % tower_name))
+    return self._metric_dict
+
+  def build_loss_weight(self):
+    loss_weights = OrderedDict()
+    num_loss = 0
+    for task_tower_cfg in self._task_towers:
+      tower_name = task_tower_cfg.tower_name
+      losses = task_tower_cfg.losses
+      n = len(losses)
+      if n > 0:
+        loss_weights[tower_name] = [
+            loss.weight * task_tower_cfg.weight for loss in losses
+        ]
+        num_loss += n
+      else:
+        loss_weights[tower_name] = [task_tower_cfg.weight]
+        num_loss += 1
+
+    strategy = self._base_model_config.loss_weight_strategy
+    if strategy == self._base_model_config.Random:
+      weights = tf.random_normal([num_loss])
+      weights = tf.nn.softmax(weights)
+      i = 0
+      for k, v in loss_weights.items():
+        n = len(v)
+        loss_weights[k] = weights[i:i + n]
+        i += n
+    return loss_weights
+
+  def get_learnt_loss(self, loss_type, name, value):
+    strategy = self._base_model_config.loss_weight_strategy
+    if strategy == self._base_model_config.Uncertainty:
+      uncertainty = tf.Variable(
+          0, name='%s_loss_weight' % name, dtype=tf.float32)
+      tf.summary.scalar('loss/%s_uncertainty' % name, uncertainty)
+      if loss_type in {LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS}:
+        return 0.5 * tf.exp(-uncertainty) * value + 0.5 * uncertainty
+      else:
+        return tf.exp(-uncertainty) * value + 0.5 * uncertainty
+    else:
+      strategy_name = EasyRecModel.LossWeightStrategy.Name(strategy)
+      raise ValueError('Unsupported loss weight strategy: ' + strategy_name)
+
+  def build_loss_graph(self):
+    """Build loss graph for multi task model."""
+    task_loss_weights = self.build_loss_weight()
+    for task_tower_cfg in self._task_towers:
+      tower_name = task_tower_cfg.tower_name
+      loss_weight = 1.0
+      if task_tower_cfg.use_sample_weight:
+        loss_weight *= self._sample_weight
+
+      if hasattr(task_tower_cfg, 'task_space_indicator_label') and \
+          task_tower_cfg.HasField('task_space_indicator_label'):
+        in_task_space = tf.to_float(
+            self._labels[task_tower_cfg.task_space_indicator_label] > 0)
+        loss_weight = loss_weight * (
+            task_tower_cfg.in_task_space_weight * in_task_space +
+            task_tower_cfg.out_task_space_weight * (1 - in_task_space))
+
+      if task_tower_cfg.HasField('task_space_indicator_name') and \
+          task_tower_cfg.HasField('task_space_indicator_value'):
+        in_task_space = tf.to_float(
+            tf.equal(
+                self._feature_dict[task_tower_cfg.task_space_indicator_name],
+                task_tower_cfg.task_space_indicator_value))
+        loss_weight = loss_weight * (
+            task_tower_cfg.in_task_space_weight * in_task_space +
+            task_tower_cfg.out_task_space_weight * (1 - in_task_space))
+
+      task_loss_weight = task_loss_weights[tower_name]
+      loss_dict = {}
+      losses = task_tower_cfg.losses
+      if len(losses) == 0:
+        loss_dict = self._build_loss_impl(
+            task_tower_cfg.loss_type,
+            label_name=self._label_name_dict[tower_name],
+            loss_weight=loss_weight,
+            num_class=task_tower_cfg.num_class,
+            suffix='_%s' % tower_name)
+        for loss_name in loss_dict.keys():
+          loss_dict[loss_name] = loss_dict[loss_name] * task_loss_weight[0]
+      else:
+        calibrate_loss = []
+        for loss in losses:
+          if loss.loss_type == LossType.ORDER_CALIBRATE_LOSS:
+            y_t = self._prediction_dict['probs_%s' % tower_name]
+            for relation_tower_name in task_tower_cfg.relation_tower_names:
+              y_rt = self._prediction_dict['probs_%s' % relation_tower_name]
+              cali_loss = tf.reduce_mean(tf.nn.relu(y_t - y_rt))
+              calibrate_loss.append(cali_loss * loss.weight)
+              logging.info('calibrate loss: %s -> %s' %
+                           (relation_tower_name, tower_name))
+            continue
+          loss_param = loss.WhichOneof('loss_param')
+          if loss_param is not None:
+            loss_param = getattr(loss, loss_param)
+          loss_ops = self._build_loss_impl(
+              loss.loss_type,
+              label_name=self._label_name_dict[tower_name],
+              loss_weight=loss_weight,
+              num_class=task_tower_cfg.num_class,
+              suffix='_%s' % tower_name,
+              loss_name=loss.loss_name,
+              loss_param=loss_param)
+          for i, loss_name in enumerate(loss_ops):
+            loss_value = loss_ops[loss_name]
+            if loss.learn_loss_weight:
+              loss_dict[loss_name] = self.get_learnt_loss(
+                  loss.loss_type, loss_name, loss_value)
+            else:
+              loss_dict[loss_name] = loss_value * task_loss_weight[i]
+        if calibrate_loss:
+          cali_loss = tf.add_n(calibrate_loss)
+          loss_dict['order_calibrate_loss'] = cali_loss
+          tf.summary.scalar('loss/order_calibrate_loss', cali_loss)
+      self._loss_dict.update(loss_dict)
+
+    kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict,
+                                              self._labels, self._feature_dict)
+    self._loss_dict.update(kd_loss_dict)
+
+    return self._loss_dict
+
+  def get_outputs(self):
+    outputs = []
+    if self._outputs:
+      outputs.extend(self._outputs)
+    for task_tower_cfg in self._task_towers:
+      tower_name = task_tower_cfg.tower_name
+      if len(task_tower_cfg.losses) == 0:
+        outputs.extend(
+            self._get_outputs_impl(
+                task_tower_cfg.loss_type,
+                task_tower_cfg.num_class,
+                suffix='_%s' % tower_name))
+      else:
+        for loss in task_tower_cfg.losses:
+          if loss.loss_type == LossType.ORDER_CALIBRATE_LOSS:
+            continue
+          outputs.extend(
+              self._get_outputs_impl(
+                  loss.loss_type,
+                  task_tower_cfg.num_class,
+                  suffix='_%s' % tower_name))
+    return list(set(outputs))

easy_rec/python/model/multi_tower.py

@@ -0,0 +1,62 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import tensorflow as tf
+
+from easy_rec.python.layers import dnn
+from easy_rec.python.model.rank_model import RankModel
+
+from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class MultiTower(RankModel):
+
+  def __init__(self,
+               model_config,
+               feature_configs,
+               features,
+               labels=None,
+               is_training=False):
+    super(MultiTower, self).__init__(model_config, feature_configs, features,
+                                     labels, is_training)
+    assert self._model_config.WhichOneof('model') == 'multi_tower', (
+        'invalid model config: %s' % self._model_config.WhichOneof('model'))
+    self._model_config = self._model_config.multi_tower
+    assert isinstance(self._model_config, MultiTowerConfig)
+
+    self._tower_features = []
+    self._tower_num = len(self._model_config.towers)
+    for tower_id in range(self._tower_num):
+      tower = self._model_config.towers[tower_id]
+      tower_feature, _ = self._input_layer(self._feature_dict, tower.input)
+      self._tower_features.append(tower_feature)
+
+  def build_predict_graph(self):
+    tower_fea_arr = []
+    for tower_id in range(self._tower_num):
+      tower_fea = self._tower_features[tower_id]
+      tower = self._model_config.towers[tower_id]
+      tower_name = tower.input
+      tower_fea = tf.layers.batch_normalization(
+          tower_fea,
+          training=self._is_training,
+          trainable=True,
+          name='%s_fea_bn' % tower_name)
+
+      tower_dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name,
+                                self._is_training)
+      tower_fea = tower_dnn_layer(tower_fea)
+      tower_fea_arr.append(tower_fea)
+
+    all_fea = tf.concat(tower_fea_arr, axis=1)
+    final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg,
+                              'final_dnn', self._is_training)
+    all_fea = final_dnn_layer(all_fea)
+    output = tf.layers.dense(all_fea, self._num_class, name='output')
+
+    self._add_to_prediction_dict(output)
+
+    return self._prediction_dict

easy_rec/python/model/multi_tower_bst.py

@@ -0,0 +1,190 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+import math
+
+import tensorflow as tf
+
+from easy_rec.python.compat import regularizers
+from easy_rec.python.layers import dnn
+from easy_rec.python.layers import layer_norm
+from easy_rec.python.layers import seq_input_layer
+from easy_rec.python.model.rank_model import RankModel
+
+from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class MultiTowerBST(RankModel):
+
+  def __init__(self,
+               model_config,
+               feature_configs,
+               features,
+               labels=None,
+               is_training=False):
+    super(MultiTowerBST, self).__init__(model_config, feature_configs, features,
+                                        labels, is_training)
+    self._seq_input_layer = seq_input_layer.SeqInputLayer(
+        feature_configs,
+        model_config.seq_att_groups,
+        embedding_regularizer=self._emb_reg,
+        ev_params=self._global_ev_params)
+    assert self._model_config.WhichOneof('model') == 'multi_tower', \
+        'invalid model config: %s' % self._model_config.WhichOneof('model')
+    self._model_config = self._model_config.multi_tower
+    assert isinstance(self._model_config, MultiTowerConfig)
+
+    self._tower_features = []
+    self._tower_num = len(self._model_config.towers)
+    for tower_id in range(self._tower_num):
+      tower = self._model_config.towers[tower_id]
+      tower_feature, _ = self._input_layer(self._feature_dict, tower.input)
+      self._tower_features.append(tower_feature)
+
+    self._bst_tower_features = []
+    self._bst_tower_num = len(self._model_config.bst_towers)
+
+    logging.info('all tower num: {0}'.format(self._tower_num +
+                                             self._bst_tower_num))
+    logging.info('bst tower num: {0}'.format(self._bst_tower_num))
+
+    for tower_id in range(self._bst_tower_num):
+      tower = self._model_config.bst_towers[tower_id]
+      tower_feature = self._seq_input_layer(self._feature_dict, tower.input)
+      regularizers.apply_regularization(
+          self._emb_reg, weights_list=[tower_feature['key']])
+      regularizers.apply_regularization(
+          self._emb_reg, weights_list=[tower_feature['hist_seq_emb']])
+      self._bst_tower_features.append(tower_feature)
+
+  def dnn_net(self, net, dnn_units, name):
+    dnn_units_len = len(dnn_units)
+    with tf.variable_scope(name_or_scope=name, reuse=tf.AUTO_REUSE):
+      for idx, units in enumerate(dnn_units):
+        if idx + 1 < dnn_units_len:
+          net = tf.layers.dense(
+              net,
+              units=units,
+              activation=tf.nn.relu,
+              name='%s_%d' % (name, idx))
+        else:
+          net = tf.layers.dense(
+              net, units=units, activation=None, name='%s_%d' % (name, idx))
+    return net
+
+  def attention_net(self, net, dim, cur_seq_len, seq_size, name):
+    query_net = self.dnn_net(net, [dim], name + '_query')  # B, seq_len，dim
+    key_net = self.dnn_net(net, [dim], name + '_key')
+    value_net = self.dnn_net(net, [dim], name + '_value')
+    scores = tf.matmul(
+        query_net, key_net, transpose_b=True)  # [B, seq_size, seq_size]
+
+    hist_mask = tf.sequence_mask(
+        cur_seq_len, maxlen=seq_size - 1)  # [B, seq_size-1]
+    cur_id_mask = tf.ones(
+        tf.stack([tf.shape(hist_mask)[0], 1]), dtype=tf.bool)  # [B, 1]
+    mask = tf.concat([hist_mask, cur_id_mask], axis=1)  # [B, seq_size]
+    masks = tf.reshape(tf.tile(mask, [1, seq_size]),
+                       (-1, seq_size, seq_size))  # [B, seq_size, seq_size]
+    padding = tf.ones_like(scores) * (-2**32 + 1)
+    scores = tf.where(masks, scores, padding)  # [B, seq_size, seq_size]
+
+    # Scale
+    scores = tf.nn.softmax(scores)  # (B, seq_size, seq_size)
+    att_res_net = tf.matmul(scores, value_net)  # [B, seq_size, emb_dim]
+    return att_res_net
+
+  def multi_head_att_net(self, id_cols, head_count, emb_dim, seq_len, seq_size):
+    multi_head_attention_res = []
+    part_cols_emd_dim = int(math.ceil(emb_dim / head_count))
+    for start_idx in range(0, emb_dim, part_cols_emd_dim):
+      if start_idx + part_cols_emd_dim > emb_dim:
+        part_cols_emd_dim = emb_dim - start_idx
+      part_id_col = tf.slice(id_cols, [0, 0, start_idx],
+                             [-1, -1, part_cols_emd_dim])
+      part_attention_net = self.attention_net(
+          part_id_col,
+          part_cols_emd_dim,
+          seq_len,
+          seq_size,
+          name='multi_head_%d' % start_idx)
+      multi_head_attention_res.append(part_attention_net)
+    multi_head_attention_res_net = tf.concat(multi_head_attention_res, axis=2)
+    multi_head_attention_res_net = self.dnn_net(
+        multi_head_attention_res_net, [emb_dim], name='multi_head_attention')
+    return multi_head_attention_res_net
+
+  def add_and_norm(self, net_1, net_2, emb_dim, name):
+    net = tf.add(net_1, net_2)
+    # layer = tf.keras.layers.LayerNormalization(axis=2)
+    layer = layer_norm.LayerNormalization(emb_dim)
+    net = layer(net)
+    return net
+
+  def bst(self, bst_fea, seq_size, head_count, name):
+    cur_id, hist_id_col, seq_len = bst_fea['key'], bst_fea[
+        'hist_seq_emb'], bst_fea['hist_seq_len']
+
+    cur_batch_max_seq_len = tf.shape(hist_id_col)[1]
+
+    hist_id_col = tf.cond(
+        tf.constant(seq_size) > cur_batch_max_seq_len, lambda: tf.pad(
+            hist_id_col, [[0, 0], [0, seq_size - cur_batch_max_seq_len - 1],
+                          [0, 0]], 'CONSTANT'),
+        lambda: tf.slice(hist_id_col, [0, 0, 0], [-1, seq_size - 1, -1]))
+    all_ids = tf.concat([hist_id_col, tf.expand_dims(cur_id, 1)],
+                        axis=1)  # b, seq_size, emb_dim
+
+    emb_dim = int(all_ids.shape[2])
+    attention_net = self.multi_head_att_net(all_ids, head_count, emb_dim,
+                                            seq_len, seq_size)
+
+    tmp_net = self.add_and_norm(
+        all_ids, attention_net, emb_dim, name='add_and_norm_1')
+    feed_forward_net = self.dnn_net(tmp_net, [emb_dim], 'feed_forward_net')
+    net = self.add_and_norm(
+        tmp_net, feed_forward_net, emb_dim, name='add_and_norm_2')
+    bst_output = tf.reshape(net, [-1, seq_size * emb_dim])
+    return bst_output
+
+  def build_predict_graph(self):
+    tower_fea_arr = []
+    for tower_id in range(self._tower_num):
+      tower_fea = self._tower_features[tower_id]
+      tower = self._model_config.towers[tower_id]
+      tower_name = tower.input
+      tower_fea = tf.layers.batch_normalization(
+          tower_fea,
+          training=self._is_training,
+          trainable=True,
+          name='%s_fea_bn' % tower_name)
+      tower_dnn = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name,
+                          self._is_training)
+      tower_fea = tower_dnn(tower_fea)
+      tower_fea_arr.append(tower_fea)
+
+    for tower_id in range(self._bst_tower_num):
+      tower_fea = self._bst_tower_features[tower_id]
+      tower = self._model_config.bst_towers[tower_id]
+      tower_name = tower.input
+      tower_seq_len = tower.seq_len
+      tower_multi_head_size = tower.multi_head_size
+      tower_fea = self.bst(
+          tower_fea,
+          seq_size=tower_seq_len,
+          head_count=tower_multi_head_size,
+          name=tower_name)
+      tower_fea_arr.append(tower_fea)
+
+    all_fea = tf.concat(tower_fea_arr, axis=1)
+    final_dnn = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn',
+                        self._is_training)
+    all_fea = final_dnn(all_fea)
+    output = tf.layers.dense(all_fea, self._num_class, name='output')
+
+    self._add_to_prediction_dict(output)
+
+    return self._prediction_dict

easy_rec/python/model/multi_tower_din.py

@@ -0,0 +1,130 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import tensorflow as tf
+
+from easy_rec.python.compat import regularizers
+from easy_rec.python.layers import dnn
+from easy_rec.python.layers import seq_input_layer
+from easy_rec.python.model.rank_model import RankModel
+
+from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class MultiTowerDIN(RankModel):
+
+  def __init__(self,
+               model_config,
+               feature_configs,
+               features,
+               labels=None,
+               is_training=False):
+    super(MultiTowerDIN, self).__init__(model_config, feature_configs, features,
+                                        labels, is_training)
+    self._seq_input_layer = seq_input_layer.SeqInputLayer(
+        feature_configs,
+        model_config.seq_att_groups,
+        embedding_regularizer=self._emb_reg,
+        ev_params=self._global_ev_params)
+    assert self._model_config.WhichOneof('model') == 'multi_tower', \
+        'invalid model config: %s' % self._model_config.WhichOneof('model')
+    self._model_config = self._model_config.multi_tower
+    assert isinstance(self._model_config, MultiTowerConfig)
+
+    self._tower_features = []
+    self._tower_num = len(self._model_config.towers)
+    for tower_id in range(self._tower_num):
+      tower = self._model_config.towers[tower_id]
+      tower_feature, _ = self._input_layer(self._feature_dict, tower.input)
+      self._tower_features.append(tower_feature)
+
+    self._din_tower_features = []
+    self._din_tower_num = len(self._model_config.din_towers)
+
+    logging.info('all tower num: {0}'.format(self._tower_num +
+                                             self._din_tower_num))
+    logging.info('din tower num: {0}'.format(self._din_tower_num))
+
+    for tower_id in range(self._din_tower_num):
+      tower = self._model_config.din_towers[tower_id]
+      tower_feature = self._seq_input_layer(self._feature_dict, tower.input)
+
+      # apply regularization for sequence feature key in seq_input_layer.
+
+      regularizers.apply_regularization(
+          self._emb_reg, weights_list=[tower_feature['hist_seq_emb']])
+      self._din_tower_features.append(tower_feature)
+
+  def din(self, dnn_config, deep_fea, name):
+    cur_id, hist_id_col, seq_len = deep_fea['key'], deep_fea[
+        'hist_seq_emb'], deep_fea['hist_seq_len']
+
+    seq_max_len = tf.shape(hist_id_col)[1]
+    emb_dim = hist_id_col.shape[2]
+
+    cur_ids = tf.tile(cur_id, [1, seq_max_len])
+    cur_ids = tf.reshape(cur_ids,
+                         tf.shape(hist_id_col))  # (B, seq_max_len, emb_dim)
+
+    din_net = tf.concat(
+        [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col],
+        axis=-1)  # (B, seq_max_len, emb_dim*4)
+
+    din_layer = dnn.DNN(
+        dnn_config,
+        self._l2_reg,
+        name,
+        self._is_training,
+        last_layer_no_activation=True,
+        last_layer_no_batch_norm=True)
+    din_net = din_layer(din_net)
+    scores = tf.reshape(din_net, [-1, 1, seq_max_len])  # (B, 1, ?)
+
+    seq_len = tf.expand_dims(seq_len, 1)
+    mask = tf.sequence_mask(seq_len)
+    padding = tf.ones_like(scores) * (-2**32 + 1)
+    scores = tf.where(mask, scores, padding)  # [B, 1, seq_max_len]
+
+    # Scale
+    scores = tf.nn.softmax(scores)  # (B, 1, seq_max_len)
+    hist_din_emb = tf.matmul(scores, hist_id_col)  # [B, 1, emb_dim]
+    hist_din_emb = tf.reshape(hist_din_emb, [-1, emb_dim])  # [B, emb_dim]
+    din_output = tf.concat([hist_din_emb, cur_id], axis=1)
+    return din_output
+
+  def build_predict_graph(self):
+    tower_fea_arr = []
+    for tower_id in range(self._tower_num):
+      tower_fea = self._tower_features[tower_id]
+      tower = self._model_config.towers[tower_id]
+      tower_name = tower.input
+      tower_fea = tf.layers.batch_normalization(
+          tower_fea,
+          training=self._is_training,
+          trainable=True,
+          name='%s_fea_bn' % tower_name)
+      dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name,
+                          self._is_training)
+      tower_fea = dnn_layer(tower_fea)
+      tower_fea_arr.append(tower_fea)
+
+    for tower_id in range(self._din_tower_num):
+      tower_fea = self._din_tower_features[tower_id]
+      tower = self._model_config.din_towers[tower_id]
+      tower_name = tower.input
+      tower_fea = self.din(tower.dnn, tower_fea, name='%s_dnn' % tower_name)
+      tower_fea_arr.append(tower_fea)
+
+    all_fea = tf.concat(tower_fea_arr, axis=1)
+    final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg,
+                              'final_dnn', self._is_training)
+    all_fea = final_dnn_layer(all_fea)
+    output = tf.layers.dense(all_fea, self._num_class, name='output')
+
+    self._add_to_prediction_dict(output)
+
+    return self._prediction_dict