easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/layers/senet.py

@@ -0,0 +1,73 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import tensorflow as tf
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class SENet:
+  """Squeeze and Excite Network.
+
+  Input shape
+  - A list of 2D tensor with shape: ``(batch_size,embedding_size)``.
+    The ``embedding_size`` of each field can have different value.
+
+  Args:
+    num_fields: int, number of fields.
+    num_squeeze_group: int, number of groups for squeeze.
+    reduction_ratio: int, reduction ratio for squeeze.
+    l2_reg: float, l2 regularizer for embedding.
+    name: str, name of the layer.
+  """
+
+  def __init__(self,
+               num_fields,
+               num_squeeze_group,
+               reduction_ratio,
+               l2_reg,
+               name='SENet'):
+    self.num_fields = num_fields
+    self.num_squeeze_group = num_squeeze_group
+    self.reduction_ratio = reduction_ratio
+    self._l2_reg = l2_reg
+    self._name = name
+
+  def __call__(self, inputs):
+    g = self.num_squeeze_group
+    f = self.num_fields
+    r = self.reduction_ratio
+    reduction_size = max(1, f * g * 2 // r)
+
+    emb_size = 0
+    for input in inputs:
+      emb_size += int(input.shape[-1])
+
+    group_embs = [
+        tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs
+    ]
+
+    squeezed = []
+    for emb in group_embs:
+      squeezed.append(tf.reduce_max(emb, axis=-1))  # [B, g]
+      squeezed.append(tf.reduce_mean(emb, axis=-1))  # [B, g]
+    z = tf.concat(squeezed, axis=1)  # [bs, field_size * num_groups * 2]
+
+    reduced = tf.layers.dense(
+        inputs=z,
+        units=reduction_size,
+        kernel_regularizer=self._l2_reg,
+        activation='relu',
+        name='%s/reduce' % self._name)
+
+    excited_weights = tf.layers.dense(
+        inputs=reduced,
+        units=emb_size,
+        kernel_initializer='glorot_normal',
+        name='%s/excite' % self._name)
+
+    # Re-weight
+    inputs = tf.concat(inputs, axis=-1)
+    output = inputs * excited_weights
+
+    return output

easy_rec/python/layers/seq_input_layer.py

@@ -0,0 +1,134 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import logging
+
+import tensorflow as tf
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import variable_scope
+
+from easy_rec.python.compat import regularizers
+from easy_rec.python.compat.feature_column import feature_column
+from easy_rec.python.feature_column.feature_column import FeatureColumnParser
+from easy_rec.python.protos.feature_config_pb2 import WideOrDeep
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class SeqInputLayer(object):
+
+  def __init__(self,
+               feature_configs,
+               feature_groups_config,
+               embedding_regularizer=None,
+               ev_params=None):
+    self._feature_groups_config = {
+        x.group_name: x for x in feature_groups_config
+    }
+    wide_and_deep_dict = self.get_wide_deep_dict()
+    self._fc_parser = FeatureColumnParser(
+        feature_configs, wide_and_deep_dict, ev_params=ev_params)
+    self._embedding_regularizer = embedding_regularizer
+
+  def __call__(self,
+               features,
+               group_name,
+               feature_name_to_output_tensors={},
+               allow_key_search=True,
+               scope_name=None):
+    feature_column_dict = self._fc_parser.deep_columns
+    feature_column_dict.update(self._fc_parser.sequence_columns)
+
+    builder = feature_column._LazyBuilder(features)
+
+    feature_dict = self._feature_groups_config[group_name]
+    tf_summary = feature_dict.tf_summary
+    if tf_summary:
+      logging.info('Write sequence feature to tensorflow summary.')
+
+    def _seq_embed_summary_name(input_name):
+      input_name = input_name.split(':')[0]
+      input_name = input_name.split('/')[:2]
+      return 'sequence_feature/' + '/'.join(input_name)
+
+    if scope_name is None:
+      scope_name = group_name
+    # name_scope is specified to avoid adding _1 _2 after scope_name
+    with variable_scope.variable_scope(
+        scope_name,
+        reuse=variable_scope.AUTO_REUSE), ops.name_scope(scope_name + '/'):
+      key_tensors = []
+      hist_tensors = []
+      check_op_list = []
+      for x in feature_dict.seq_att_map:
+        for key in x.key:
+          if key not in feature_name_to_output_tensors or (
+              feature_name_to_output_tensors[key] is None and allow_key_search):
+            qfc = feature_column_dict[key]
+            with variable_scope.variable_scope(qfc._var_scope_name):
+              tmp_key_tensor = feature_column_dict[key]._get_dense_tensor(
+                  builder)
+              regularizers.apply_regularization(
+                  self._embedding_regularizer, weights_list=[tmp_key_tensor])
+              key_tensors.append(tmp_key_tensor)
+          elif feature_name_to_output_tensors[key] is None:
+            assert feature_name_to_output_tensors[
+                key] is not None, 'When allow_key_search is False, key: %s should defined in same feature group.' % key
+          else:
+            key_tensors.append(feature_name_to_output_tensors[key])
+
+        if tf_summary:
+          for key_tensor in key_tensors:
+            tf.summary.histogram(
+                _seq_embed_summary_name(key_tensor.name), key_tensor)
+        cur_hist_seqs = []
+        for hist_seq in x.hist_seq:
+          seq_fc = feature_column_dict[hist_seq]
+          with variable_scope.variable_scope(seq_fc._var_scope_name):
+            cur_hist_seqs.append(
+                feature_column_dict[hist_seq]._get_sequence_dense_tensor(
+                    builder))
+        hist_tensors.extend(cur_hist_seqs)
+
+        aux_hist_emb_list = []
+        for aux_hist_seq in x.aux_hist_seq:
+          seq_fc = feature_column_dict[aux_hist_seq]
+          with variable_scope.variable_scope(seq_fc._var_scope_name):
+            aux_hist_embedding, _ = feature_column_dict[
+                aux_hist_seq]._get_sequence_dense_tensor(builder)
+          aux_hist_emb_list.append(aux_hist_embedding)
+
+        if tf_summary:
+          for hist_embed, hist_seq_len in hist_tensors:
+            tf.summary.histogram(
+                _seq_embed_summary_name(hist_embed.name), hist_embed)
+            tf.summary.histogram(
+                _seq_embed_summary_name(hist_seq_len.name), hist_seq_len)
+
+        for idx in range(1, len(cur_hist_seqs)):
+          check_op = tf.assert_equal(
+              cur_hist_seqs[0][1],
+              cur_hist_seqs[idx][1],
+              message='SequenceFeature Error: The size of %s not equal to the size of %s.'
+              % (x.hist_seq[idx], x.hist_seq[0]))
+          check_op_list.append(check_op)
+
+    with tf.control_dependencies(check_op_list):
+      features = {
+          'key': tf.concat(key_tensors, axis=-1),
+          'hist_seq_emb': tf.concat([x[0] for x in hist_tensors], axis=-1),
+          'hist_seq_len': hist_tensors[0][1],
+          'aux_hist_seq_emb_list': aux_hist_emb_list
+      }
+    return features
+
+  def get_wide_deep_dict(self):
+    wide_and_deep_dict = {}
+    for group_name_config in self._feature_groups_config.values():
+      for x in group_name_config.seq_att_map:
+        for key in x.key:
+          wide_and_deep_dict[key] = WideOrDeep.DEEP
+        for hist_seq in x.hist_seq:
+          wide_and_deep_dict[hist_seq] = WideOrDeep.DEEP
+    return wide_and_deep_dict

easy_rec/python/layers/sequence_feature_layer.py

@@ -0,0 +1,249 @@
+import logging
+import os
+
+import tensorflow as tf
+from tensorflow.python.framework import ops
+
+from easy_rec.python.compat import regularizers
+from easy_rec.python.layers import dnn
+from easy_rec.python.layers import seq_input_layer
+from easy_rec.python.utils import conditional
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class SequenceFeatureLayer(object):
+
+  def __init__(self,
+               feature_configs,
+               feature_groups_config,
+               ev_params=None,
+               embedding_regularizer=None,
+               kernel_regularizer=None,
+               is_training=False,
+               is_predicting=False):
+    self._seq_feature_groups_config = []
+    for x in feature_groups_config:
+      for y in x.sequence_features:
+        self._seq_feature_groups_config.append(y)
+    self._seq_input_layer = None
+    if len(self._seq_feature_groups_config) > 0:
+      self._seq_input_layer = seq_input_layer.SeqInputLayer(
+          feature_configs,
+          self._seq_feature_groups_config,
+          embedding_regularizer=embedding_regularizer,
+          ev_params=ev_params)
+    self._embedding_regularizer = embedding_regularizer
+    self._kernel_regularizer = kernel_regularizer
+    self._is_training = is_training
+    self._is_predicting = is_predicting
+
+  def negative_sampler_target_attention(self,
+                                        dnn_config,
+                                        deep_fea,
+                                        concat_features,
+                                        name,
+                                        need_key_feature=True,
+                                        allow_key_transform=False):
+    cur_id, hist_id_col, seq_len, aux_hist_emb_list = deep_fea['key'], deep_fea[
+        'hist_seq_emb'], deep_fea['hist_seq_len'], deep_fea[
+            'aux_hist_seq_emb_list']
+
+    seq_max_len = tf.shape(hist_id_col)[1]
+    seq_emb_dim = hist_id_col.shape[2]
+    cur_id_dim = tf.shape(cur_id)[-1]
+    batch_size = tf.shape(hist_id_col)[0]
+
+    pos_feature = cur_id[:batch_size]
+    neg_feature = cur_id[batch_size:]
+    cur_id = tf.concat([
+        pos_feature[:, tf.newaxis, :],
+        tf.tile(neg_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1])
+    ],
+                       axis=1)  # noqa: E126
+    neg_num_add_1 = tf.shape(cur_id)[1]
+    hist_id_col_tmp = tf.tile(
+        hist_id_col[:, :, :], multiples=[1, neg_num_add_1, 1])
+    hist_id_col = tf.reshape(
+        hist_id_col_tmp, [batch_size * neg_num_add_1, seq_max_len, seq_emb_dim])
+
+    concat_features = tf.tile(
+        concat_features[:, tf.newaxis, :], multiples=[1, neg_num_add_1, 1])
+    seq_len = tf.tile(seq_len, multiples=[neg_num_add_1])
+
+    if allow_key_transform and (cur_id_dim != seq_emb_dim):
+      cur_id = tf.layers.dense(
+          cur_id, seq_emb_dim, name='sequence_key_transform_layer')
+
+    cur_ids = tf.tile(cur_id, [1, 1, seq_max_len])
+    cur_ids = tf.reshape(
+        cur_ids,
+        tf.shape(hist_id_col))  # (B * neg_num_add_1, seq_max_len, seq_emb_dim)
+
+    din_net = tf.concat(
+        [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col],
+        axis=-1)  # (B * neg_num_add_1, seq_max_len, seq_emb_dim*4)
+
+    din_layer = dnn.DNN(
+        dnn_config,
+        self._kernel_regularizer,
+        name,
+        self._is_training,
+        last_layer_no_activation=True,
+        last_layer_no_batch_norm=True)
+    din_net = din_layer(din_net)
+    scores = tf.reshape(din_net, [-1, 1, seq_max_len])  # (B, 1, ?)
+
+    seq_len = tf.expand_dims(seq_len, 1)
+    mask = tf.sequence_mask(seq_len)
+    padding = tf.ones_like(scores) * (-2**32 + 1)
+    scores = tf.where(mask, scores,
+                      padding)  # [B*neg_num_add_1, 1, seq_max_len]
+
+    # Scale
+    scores = tf.nn.softmax(scores)  # (B * neg_num_add_1, 1, seq_max_len)
+    hist_din_emb = tf.matmul(scores,
+                             hist_id_col)  # [B * neg_num_add_1, 1, seq_emb_dim]
+    hist_din_emb = tf.reshape(hist_din_emb,
+                              [batch_size, neg_num_add_1, seq_emb_dim
+                               ])  # [B * neg_num_add_1, seq_emb_dim]
+    if len(aux_hist_emb_list) > 0:
+      all_hist_dim_emb = [hist_din_emb]
+      for hist_col in aux_hist_emb_list:
+        cur_aux_hist = tf.matmul(scores, hist_col)
+        outputs = tf.reshape(cur_aux_hist, [-1, seq_emb_dim])
+        all_hist_dim_emb.append(outputs)
+      hist_din_emb = tf.concat(all_hist_dim_emb, axis=1)
+    if not need_key_feature:
+      return hist_din_emb, concat_features
+    din_output = tf.concat([hist_din_emb, cur_id], axis=2)
+    return din_output, concat_features
+
+  def target_attention(self,
+                       dnn_config,
+                       deep_fea,
+                       name,
+                       need_key_feature=True,
+                       allow_key_transform=False,
+                       transform_dnn=False):
+    cur_id, hist_id_col, seq_len, aux_hist_emb_list = deep_fea['key'], deep_fea[
+        'hist_seq_emb'], deep_fea['hist_seq_len'], deep_fea[
+            'aux_hist_seq_emb_list']
+
+    seq_max_len = tf.shape(hist_id_col)[1]
+    seq_emb_dim = hist_id_col.shape[2]
+    cur_id_dim = cur_id.shape[-1]
+
+    if allow_key_transform and (cur_id_dim != seq_emb_dim):
+      if seq_emb_dim > cur_id_dim and not transform_dnn:
+        cur_id = tf.pad(cur_id, [[0, 0], [0, seq_emb_dim - cur_id_dim]])
+      else:
+        cur_key_layer_name = 'sequence_key_transform_layer_' + name
+        cur_id = tf.layers.dense(cur_id, seq_emb_dim, name=cur_key_layer_name)
+        cur_fea_layer_name = 'sequence_fea_transform_layer_' + name
+        hist_id_col = tf.layers.dense(
+            hist_id_col, seq_emb_dim, name=cur_fea_layer_name)
+    else:
+      cur_id = cur_id[:tf.shape(hist_id_col)[0], ...]  # for negative sampler
+
+    cur_ids = tf.tile(cur_id, [1, seq_max_len])
+    cur_ids = tf.reshape(cur_ids,
+                         tf.shape(hist_id_col))  # (B, seq_max_len, seq_emb_dim)
+
+    din_net = tf.concat(
+        [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col],
+        axis=-1)  # (B, seq_max_len, seq_emb_dim*4)
+
+    din_layer = dnn.DNN(
+        dnn_config,
+        self._kernel_regularizer,
+        name,
+        self._is_training,
+        last_layer_no_activation=True,
+        last_layer_no_batch_norm=True)
+    din_net = din_layer(din_net)
+    scores = tf.reshape(din_net, [-1, 1, seq_max_len])  # (B, 1, ?)
+
+    seq_len = tf.expand_dims(seq_len, 1)
+    mask = tf.sequence_mask(seq_len)
+    padding = tf.ones_like(scores) * (-2**32 + 1)
+    scores = tf.where(mask, scores, padding)  # [B, 1, seq_max_len]
+
+    # Scale
+    scores = tf.nn.softmax(scores)  # (B, 1, seq_max_len)
+    hist_din_emb = tf.matmul(scores, hist_id_col)  # [B, 1, seq_emb_dim]
+    hist_din_emb = tf.reshape(hist_din_emb,
+                              [-1, seq_emb_dim])  # [B, seq_emb_dim]
+    if len(aux_hist_emb_list) > 0:
+      all_hist_dim_emb = [hist_din_emb]
+      for hist_col in aux_hist_emb_list:
+        aux_hist_dim = hist_col.shape[-1]
+        cur_aux_hist = tf.matmul(scores, hist_col)
+        outputs = tf.reshape(cur_aux_hist, [-1, aux_hist_dim])
+        all_hist_dim_emb.append(outputs)
+      hist_din_emb = tf.concat(all_hist_dim_emb, axis=1)
+    if not need_key_feature:
+      return hist_din_emb
+    din_output = tf.concat([hist_din_emb, cur_id], axis=1)
+    return din_output
+
+  def __call__(self,
+               features,
+               concat_features,
+               all_seq_att_map_config,
+               feature_name_to_output_tensors=None,
+               negative_sampler=False,
+               scope_name=None):
+    logging.info('use sequence feature layer.')
+    all_seq_fea = []
+    # process all sequence features
+    for seq_att_map_config in all_seq_att_map_config:
+      group_name = seq_att_map_config.group_name
+      allow_key_search = seq_att_map_config.allow_key_search
+      need_key_feature = seq_att_map_config.need_key_feature
+      allow_key_transform = seq_att_map_config.allow_key_transform
+      transform_dnn = seq_att_map_config.transform_dnn
+
+      place_on_cpu = os.getenv('place_embedding_on_cpu')
+      place_on_cpu = eval(place_on_cpu) if place_on_cpu else False
+      with conditional(self._is_predicting and place_on_cpu,
+                       ops.device('/CPU:0')):
+        seq_features = self._seq_input_layer(features, group_name,
+                                             feature_name_to_output_tensors,
+                                             allow_key_search, scope_name)
+
+      # apply regularization for sequence feature key in seq_input_layer.
+
+      regularizers.apply_regularization(
+          self._embedding_regularizer,
+          weights_list=[seq_features['hist_seq_emb']])
+      seq_dnn_config = None
+      if seq_att_map_config.HasField('seq_dnn'):
+        seq_dnn_config = seq_att_map_config.seq_dnn
+      else:
+        logging.info(
+            'seq_dnn not set in seq_att_groups, will use default settings')
+        # If not set seq_dnn, will use default settings
+        from easy_rec.python.protos.dnn_pb2 import DNN
+        seq_dnn_config = DNN()
+        seq_dnn_config.hidden_units.extend([128, 64, 32, 1])
+      cur_target_attention_name = 'seq_dnn' + group_name
+      if negative_sampler:
+        seq_fea, concat_features = self.negative_sampler_target_attention(
+            seq_dnn_config,
+            seq_features,
+            concat_features,
+            name=cur_target_attention_name,
+            need_key_feature=need_key_feature,
+            allow_key_transform=allow_key_transform)
+      else:
+        seq_fea = self.target_attention(
+            seq_dnn_config,
+            seq_features,
+            name=cur_target_attention_name,
+            need_key_feature=need_key_feature,
+            allow_key_transform=allow_key_transform,
+            transform_dnn=transform_dnn)
+      all_seq_fea.append(seq_fea)
+    return concat_features, all_seq_fea