easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/test/util_test.py

@@ -0,0 +1,85 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import tensorflow as tf
+
+from easy_rec.python.utils import estimator_utils
+from easy_rec.python.utils.dag import DAG
+from easy_rec.python.utils.expr_util import get_expression
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+gfile = tf.gfile
+
+
+class UtilTest(tf.test.TestCase):
+
+  def test_get_ckpt_version(self):
+    ver = estimator_utils.get_ckpt_version(
+        'oss://easyrec/ckpts/model.ckpt-6500.meta')
+    assert ver == 6500, 'invalid version: %s' % str(ver)
+    ver = estimator_utils.get_ckpt_version(
+        'oss://easyrec/ckpts/model.ckpt-6500')
+    assert ver == 6500, 'invalid version: %s' % str(ver)
+
+  def test_get_expression_greater(self):
+    result = get_expression('age_level>item_age_level',
+                            ['age_level', 'item_age_level'])
+    assert result == "tf.greater(parsed_dict['age_level'], parsed_dict['item_age_level'])"
+
+  def test_get_expression_greater_equal(self):
+    result = get_expression('age_level>=item_age_level',
+                            ['age_level', 'item_age_level'])
+    assert result == "tf.greater_equal(parsed_dict['age_level'], parsed_dict['item_age_level'])"
+
+  def test_get_expression_less(self):
+    result = get_expression('age_level<item_age_level',
+                            ['age_level', 'item_age_level'])
+    assert result == "tf.less(parsed_dict['age_level'], parsed_dict['item_age_level'])"
+
+  def test_get_expression_less_equal(self):
+    result = get_expression('age_level<=item_age_level',
+                            ['age_level', 'item_age_level'])
+    assert result == "tf.less_equal(parsed_dict['age_level'], parsed_dict['item_age_level'])"
+
+  def test_get_expression_and(self):
+    result = get_expression('(age_level>3)&(item_age_level<1)',
+                            ['age_level', 'item_age_level'])
+    assert result == "tf.greater(parsed_dict['age_level'], 3) & tf.less(parsed_dict['item_age_level'], 1)"
+
+    result = get_expression(
+        '(age_level>item_age_level) & (age_level<item_age_level*3)',
+        ['age_level', 'item_age_level'])
+    assert result == "tf.greater(parsed_dict['age_level'], parsed_dict['item_age_level']) &" \
+                     " tf.less(parsed_dict['age_level'], parsed_dict['item_age_level']*3)"
+
+  def test_get_expression_or(self):
+    result = get_expression('(age_level>3)|(item_age_level<1)',
+                            ['age_level', 'item_age_level'])
+    assert result == "tf.greater(parsed_dict['age_level'], 3) | tf.less(parsed_dict['item_age_level'], 1)"
+
+  def test_dag(self):
+    dag = DAG()
+    dag.add_node('a')
+    dag.add_node('b')
+    dag.add_node('c')
+    dag.add_node('d')
+    dag.add_edge('a', 'b')
+    dag.add_edge('a', 'd')
+    dag.add_edge('b', 'c')
+    order = dag.topological_sort()
+    idx_a = order.index('a')
+    idx_b = order.index('b')
+    idx_c = order.index('c')
+    idx_d = order.index('d')
+    assert idx_a < idx_b
+    assert idx_a < idx_d
+    assert idx_b < idx_c
+    c = dag.all_downstreams('b')
+    assert c == ['c']
+    leaf = dag.all_leaves()
+    assert leaf == ['c', 'd']
+
+
+if __name__ == '__main__':
+  tf.test.main()

easy_rec/python/test/zero_inflated_lognormal_test.py

@@ -0,0 +1,53 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import numpy as np
+import tensorflow as tf
+from scipy import stats
+
+from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_loss  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+# Absolute error tolerance in asserting array near.
+_ERR_TOL = 1e-6
+
+
+# softplus function that calculates log(1+exp(x))
+def _softplus(x):
+  return np.log(1.0 + np.exp(x))
+
+
+# sigmoid function that calculates 1/(1+exp(-x))
+def _sigmoid(x):
+  return 1 / (1 + np.exp(-x))
+
+
+class ZeroInflatedLognormalLossTest(tf.test.TestCase):
+
+  def setUp(self):
+    super(ZeroInflatedLognormalLossTest, self).setUp()
+    self.logits = np.array([[.1, .2, .3], [.4, .5, .6]])
+    self.labels = np.array([[0.], [1.5]])
+
+  def zero_inflated_lognormal(self, labels, logits):
+    positive_logits = logits[..., :1]
+    loss_zero = _softplus(positive_logits)
+    loc = logits[..., 1:2]
+    scale = np.maximum(
+        _softplus(logits[..., 2:]), np.sqrt(tf.keras.backend.epsilon()))
+    log_prob_non_zero = stats.lognorm.logpdf(
+        x=labels, s=scale, loc=0, scale=np.exp(loc))
+    loss_non_zero = _softplus(-positive_logits) - log_prob_non_zero
+    return np.mean(np.where(labels == 0., loss_zero, loss_non_zero), axis=-1)
+
+  def test_loss_value(self):
+    expected_loss = self.zero_inflated_lognormal(self.labels, self.logits)
+    expected_loss = np.average(expected_loss)
+    loss = zero_inflated_lognormal_loss(self.labels, self.logits)
+    self.assertNear(self.evaluate(loss), expected_loss, _ERR_TOL)
+
+
+if __name__ == '__main__':
+  tf.enable_eager_execution()
+  tf.test.main()

easy_rec/python/tools/add_boundaries_to_config.py

@@ -0,0 +1,67 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import json
+import logging
+import os
+import sys
+
+import common_io
+import tensorflow as tf
+
+from easy_rec.python.utils import config_util
+from easy_rec.python.utils import io_util
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+logging.basicConfig(
+    format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s',
+    level=logging.INFO)
+tf.app.flags.DEFINE_string('template_config_path', None,
+                           'Path to template pipeline config '
+                           'file.')
+tf.app.flags.DEFINE_string('output_config_path', None,
+                           'Path to output pipeline config '
+                           'file.')
+tf.app.flags.DEFINE_string('tables', '', 'quantile binning table')
+
+FLAGS = tf.app.flags.FLAGS
+
+
+def main(argv):
+  pipeline_config = config_util.get_configs_from_pipeline_file(
+      FLAGS.template_config_path)
+
+  feature_boundaries_info = {}
+  reader = common_io.table.TableReader(
+      FLAGS.tables, selected_cols='feature,json')
+  while True:
+    try:
+      record = reader.read()
+      raw_info = json.loads(record[0][1])
+      bin_info = []
+      for info in raw_info['bin']['norm'][:-1]:
+        split_point = float(info['value'].split(',')[1][:-1])
+        bin_info.append(split_point)
+      feature_boundaries_info[record[0][0]] = bin_info
+    except common_io.exception.OutOfRangeException:
+      reader.close()
+      break
+
+  logging.info('feature boundaries: %s' % feature_boundaries_info)
+
+  for feature_config in pipeline_config.feature_configs:
+    feature_name = feature_config.input_names[0]
+    if feature_name in feature_boundaries_info:
+      feature_config.feature_type = feature_config.RawFeature
+      feature_config.hash_bucket_size = 0
+      feature_config.boundaries.extend(feature_boundaries_info[feature_name])
+      logging.info('edited %s' % feature_name)
+
+  config_dir, config_name = os.path.split(FLAGS.output_config_path)
+  config_util.save_pipeline_config(pipeline_config, config_dir, config_name)
+
+
+if __name__ == '__main__':
+  sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv)
+  tf.app.run()

easy_rec/python/tools/add_feature_info_to_config.py

@@ -0,0 +1,145 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import json
+import logging
+import os
+import sys
+
+import tensorflow as tf
+
+from easy_rec.python.utils import config_util
+from easy_rec.python.utils import io_util
+from easy_rec.python.utils.hive_utils import HiveUtils
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+logging.basicConfig(
+    format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s',
+    level=logging.INFO)
+tf.app.flags.DEFINE_string('template_config_path', None,
+                           'Path to template pipeline config '
+                           'file.')
+tf.app.flags.DEFINE_string('output_config_path', None,
+                           'Path to output pipeline config '
+                           'file.')
+tf.app.flags.DEFINE_string('config_table', '', 'config table')
+
+FLAGS = tf.app.flags.FLAGS
+
+
+def main(argv):
+  pipeline_config = config_util.get_configs_from_pipeline_file(
+      FLAGS.template_config_path)
+  sels = 'feature,feature_info,message'
+  feature_info_map = {}
+  drop_feature_names = []
+
+  if pipeline_config.WhichOneof('train_path') == 'hive_train_input':
+    hive_util = HiveUtils(
+        data_config=pipeline_config.data_config,
+        hive_config=pipeline_config.hive_train_input,
+        selected_cols=sels,
+        record_defaults=['', '', ''])
+    reader = hive_util.hive_read_line(FLAGS.config_table)
+    for record in reader:
+      feature_name = record[0][0]
+      feature_info_map[feature_name] = json.loads(record[0][1])
+      if 'DROP IT' in record[0][2]:
+        drop_feature_names.append(feature_name)
+
+  else:
+    import common_io
+    reader = common_io.table.TableReader(FLAGS.config_table, selected_cols=sels)
+    while True:
+      try:
+        record = reader.read()
+        feature_name = record[0][0]
+        feature_info_map[feature_name] = json.loads(record[0][1])
+        if 'DROP IT' in record[0][2]:
+          drop_feature_names.append(feature_name)
+      except common_io.exception.OutOfRangeException:
+        reader.close()
+        break
+
+  feature_configs = config_util.get_compatible_feature_configs(pipeline_config)
+  if drop_feature_names:
+    tmp_feature_configs = feature_configs[:]
+    for fea_cfg in tmp_feature_configs:
+      fea_name = fea_cfg.input_names[0]
+      if fea_name in drop_feature_names:
+        feature_configs.remove(fea_cfg)
+  for feature_config in feature_configs:
+    feature_name = feature_config.input_names[0]
+    if feature_name in feature_info_map:
+      logging.info('edited %s' % feature_name)
+      feature_config.embedding_dim = int(
+          feature_info_map[feature_name]['embedding_dim'])
+      logging.info('modify embedding_dim to %s' % feature_config.embedding_dim)
+      if 'boundary' in feature_info_map[feature_name]:
+        feature_config.ClearField('boundaries')
+        feature_config.boundaries.extend(
+            [float(i) for i in feature_info_map[feature_name]['boundary']])
+        logging.info('modify boundaries to %s' % feature_config.boundaries)
+      elif 'hash_bucket_size' in feature_info_map[feature_name]:
+        feature_config.hash_bucket_size = int(
+            feature_info_map[feature_name]['hash_bucket_size'])
+        logging.info('modify hash_bucket_size to %s' %
+                     feature_config.hash_bucket_size)
+  # modify num_steps
+  pipeline_config.train_config.num_steps = feature_info_map['__NUM_STEPS__'][
+      'num_steps']
+  logging.info('modify num_steps to %s' %
+               pipeline_config.train_config.num_steps)
+  # modify decay_steps
+  optimizer_configs = pipeline_config.train_config.optimizer_config
+  for optimizer_config in optimizer_configs:
+    optimizer = optimizer_config.WhichOneof('optimizer')
+    optimizer = getattr(optimizer_config, optimizer)
+    learning_rate = optimizer.learning_rate.WhichOneof('learning_rate')
+    learning_rate = getattr(optimizer.learning_rate, learning_rate)
+    if hasattr(learning_rate, 'decay_steps'):
+      learning_rate.decay_steps = feature_info_map['__DECAY_STEPS__'][
+          'decay_steps']
+    logging.info('modify decay_steps to %s' % learning_rate.decay_steps)
+
+  for feature_group in pipeline_config.model_config.feature_groups:
+    feature_names = feature_group.feature_names
+    reserved_features = []
+    for feature_name in feature_names:
+      if feature_name not in drop_feature_names:
+        reserved_features.append(feature_name)
+      else:
+        logging.info('drop feature: %s' % feature_name)
+    feature_group.ClearField('feature_names')
+    feature_group.feature_names.extend(reserved_features)
+    for sequence_feature in feature_group.sequence_features:
+      seq_att_maps = sequence_feature.seq_att_map
+      for seq_att in seq_att_maps:
+        keys = seq_att.key
+        reserved_keys = []
+        for key in keys:
+          if key not in drop_feature_names:
+            reserved_keys.append(key)
+          else:
+            logging.info('drop sequence feature key: %s' % key)
+        seq_att.ClearField('key')
+        seq_att.key.extend(reserved_keys)
+
+        hist_seqs = seq_att.hist_seq
+        reserved_hist_seqs = []
+        for hist_seq in hist_seqs:
+          if hist_seq not in drop_feature_names:
+            reserved_hist_seqs.append(hist_seq)
+          else:
+            logging.info('drop sequence feature hist_seq: %s' % hist_seq)
+        seq_att.ClearField('hist_seq')
+        seq_att.hist_seq.extend(reserved_hist_seqs)
+
+  config_dir, config_name = os.path.split(FLAGS.output_config_path)
+  config_util.save_pipeline_config(pipeline_config, config_dir, config_name)
+
+
+if __name__ == '__main__':
+  sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv)
+  tf.app.run()

easy_rec/python/tools/convert_config_format.py

@@ -0,0 +1,48 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+
+from google.protobuf import json_format
+from google.protobuf import text_format
+
+from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig
+
+
+def load_config(input_config):
+  pipeline_config = EasyRecConfig()
+  with open(input_config, 'r') as fin:
+    tmp_str = fin.read()
+    if input_config.endswith('.config'):
+      text_format.Merge(tmp_str, pipeline_config)
+    elif input_config.endswith('.json'):
+      json_format.Parse(tmp_str, pipeline_config)
+    else:
+      assert False, 'only .config/.json are supported(%s)' % input_config
+  return pipeline_config
+
+
+def save_config(pipeline_config, save_path):
+  with open(save_path, 'w') as fout:
+    if save_path.endswith('.config'):
+      fout.write(text_format.MessageToString(pipeline_config, as_utf8=True))
+    elif save_path.endswith('.json'):
+      fout.write(
+          json_format.MessageToJson(
+              pipeline_config, preserving_proto_field_name=True))
+    else:
+      assert False, 'only .config/.json are supported(%s)' % save_path
+
+
+if __name__ == '__main__':
+  import argparse
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--input_config', type=str, help='input_config path', default=None)
+  parser.add_argument(
+      '--output_config', type=str, help='output_config path', default=None)
+  args = parser.parse_args()
+
+  assert os.path.exists(args.input_config)
+  pipeline_config = load_config(args.input_config)
+  save_config(pipeline_config, args.output_config)

easy_rec/python/tools/convert_rtp_data.py

@@ -0,0 +1,79 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""Convert the original rtp data format to csv format.
+
+The original data format is not suggested to use with EasyRec.
+In the original format: features are in kv format, if a feature has
+more than one value, there will be multiple kvs, such as:
+  ...tagbeautytagsmart...
+In our new format:
+  ...beautysmart...
+"""
+import argparse
+import csv
+import json
+import logging
+import sys
+
+import tensorflow as tf
+
+logging.basicConfig(
+    format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s',
+    level=logging.INFO)
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--rtp_fg', type=str, default='', help='rtp fg path(.json)')
+  parser.add_argument('--input_path', type=str, default='', help='input path')
+  parser.add_argument('--output_path', type=str, default='', help='output path')
+  parser.add_argument('--label', type=str, default='', help='label for train')
+  args = parser.parse_args()
+
+  if not args.rtp_fg:
+    logging.error('rtp_fg is not set')
+    sys.exit(1)
+
+  if not args.input_path:
+    logging.error('input_path is not set')
+    sys.exit(1)
+
+  if not args.output_path:
+    logging.error('output_path is not set')
+    sys.exit(1)
+
+  if not args.label:
+    logging.error('label is not set')
+    sys.exit(1)
+
+  with open(args.rtp_fg, 'r') as fin:
+    rtp_fg = json.load(fin)
+
+  feature_names = [args.label]
+  for feature in rtp_fg['features']:
+    feature_name = feature['feature_name']
+    feature_names.append(feature_name)
+
+  with open(args.input_path, 'r') as fin:
+    with open(args.output_path, 'w') as fout:
+      writer = csv.writer(fout)
+      for line_str in fin:
+        line_str = line_str.strip()
+        line_toks = line_str.split('\002')
+        temp_dict = {}
+        for line_tok in line_toks:
+          k, v = line_tok.split('\003')
+          if k not in temp_dict:
+            temp_dict[k] = [v]
+          else:
+            temp_dict[k].append(v)
+        temp_vs = []
+        for feature_name in feature_names:
+          if feature_name in temp_dict:
+            temp_vs.append('|'.join(temp_dict[feature_name]))
+          else:
+            temp_vs.append('')
+        writer.writerow(temp_vs)

easy_rec/python/tools/convert_rtp_fg.py

@@ -0,0 +1,106 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""Convert rtp fg feature config to easy_rec data_config and feature_config."""
+import argparse
+import logging
+import sys
+
+import tensorflow as tf
+
+from easy_rec.python.utils.config_util import save_message
+from easy_rec.python.utils.convert_rtp_fg import convert_rtp_fg
+
+logging.basicConfig(
+    format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s',
+    level=logging.INFO)
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+model_types = ['deepfm', 'multi_tower', 'wide_and_deep', 'esmm', 'dbmtl', '']
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--model_type',
+      type=str,
+      choices=model_types,
+      default='',
+      help='model type, currently support: %s' % ','.join(model_types))
+  parser.add_argument('--rtp_fg', type=str, help='rtp fg path')
+  parser.add_argument(
+      '--embedding_dim', type=int, default=16, help='embedding_dimension')
+  parser.add_argument(
+      '--batch_size', type=int, default=1024, help='batch_size for train')
+  parser.add_argument(
+      '--label',
+      type=str,
+      default='',
+      nargs='+',
+      required=True,
+      help='label fields')
+  parser.add_argument(
+      '--num_steps',
+      type=int,
+      default=1000,
+      help='number of train steps = num_samples * num_epochs / batch_size / num_workers'
+  )
+  parser.add_argument('--output_path', type=str, help='generated config path')
+  parser.add_argument(
+      '--incol_separator',
+      type=str,
+      default='\003',
+      help='separator for multi_value features')
+  parser.add_argument(
+      '--separator',
+      type=str,
+      default='\002',
+      help='separator between different features')
+  parser.add_argument(
+      '--train_input_path', type=str, default=None, help='train data path')
+  parser.add_argument(
+      '--eval_input_path', type=str, default=None, help='eval data path')
+  parser.add_argument(
+      '--selected_cols',
+      type=str,
+      default=None,
+      help='selected cols, for csv input, it is in the format of: label_col_id0,...,lable_cold_idn,feature_col_id '
+      'for odps table input, it is in the format of: label_col_name0,...,label_col_namen,feature_col_name '
+  )
+  parser.add_argument(
+      '--rtp_separator', type=str, default=';', help='separator')
+  parser.add_argument(
+      '--input_type',
+      type=str,
+      default='OdpsRTPInput',
+      help='default to OdpsRTPInput, if test local, change it to RTPInput')
+  parser.add_argument(
+      '--is_async', action='store_true', help='async mode, debug to false')
+
+  args = parser.parse_args()
+
+  if not args.rtp_fg:
+    logging.error('rtp_fg is not set')
+    sys.exit(1)
+
+  if not args.output_path:
+    logging.error('output_path is not set')
+    sys.exit(1)
+
+  pipeline_config = convert_rtp_fg(args.rtp_fg, args.embedding_dim,
+                                   args.batch_size, args.label, args.num_steps,
+                                   args.model_type, args.separator,
+                                   args.incol_separator, args.train_input_path,
+                                   args.eval_input_path, args.selected_cols,
+                                   args.input_type, args.is_async)
+  save_message(pipeline_config, args.output_path)
+  logging.info('Conversion done.')
+  logging.info('Tips:')
+  logging.info(
+      'if run on local, please change data_config.input_type to RTPInput, '
+      'and model_dir/train_input_path/eval_input_path must also be set, ')
+  logging.info(
+      'if run local, please set data_config.selected_cols in the format '
+      'label_col_id0,label_col_id1,...,label_col_idn,feature_col_id')
+  logging.info(
+      'if run on odps, selected_cols must be set, which are label0_col,'
+      'label1_col, ..., feature_col_name')