easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/tools/write_kafka.py

@@ -0,0 +1,65 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import argparse
+import logging
+import sys
+
+# from kafka import KafkaConsumer
+from kafka import KafkaAdminClient
+from kafka import KafkaProducer
+from kafka.admin import NewTopic
+
+# from kafka.structs import TopicPartition
+
+logging.basicConfig(
+    level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s')
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--servers', type=str, default='localhost:9092')
+  parser.add_argument('--topic', type=str, default=None)
+  parser.add_argument('--group', type=str, default='consumer')
+  parser.add_argument('--partitions', type=str, default=None)
+  parser.add_argument('--timeout', type=float, default=float('inf'))
+  # file to send
+  parser.add_argument('--input_path', type=str, default=None)
+  args = parser.parse_args()
+
+  if args.input_path is None:
+    logging.error('input_path is not set')
+    sys.exit(1)
+
+  if args.topic is None:
+    logging.error('topic is not set')
+    sys.exit(1)
+
+  servers = args.servers.split(',')
+
+  admin_clt = KafkaAdminClient(bootstrap_servers=servers)
+  if args.topic not in admin_clt.list_topics():
+    admin_clt.create_topics(
+        new_topics=[
+            NewTopic(
+                name=args.topic,
+                num_partitions=1,
+                replication_factor=1,
+                topic_configs={'max.message.bytes': 1024 * 1024 * 1024})
+        ],
+        validate_only=False)
+    logging.info('create increment save topic: %s' % args.topic)
+  admin_clt.close()
+
+  producer = KafkaProducer(
+      bootstrap_servers=servers,
+      request_timeout_ms=args.timeout * 1000,
+      api_version=(0, 10, 1))
+
+  i = 1
+  with open(args.input_path, 'r') as fin:
+    for line_str in fin:
+      producer.send(args.topic, line_str.encode('utf-8'))
+      i += 1
+      break
+      if i % 100 == 0:
+        logging.info('progress: %d' % i)
+  producer.close()

easy_rec/python/train_eval.py

@@ -0,0 +1,325 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import argparse
+import json
+import logging
+import os
+py_root_dir_path = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+print(f"py_root_dir_path:{py_root_dir_path}")
+import sys
+sys.path.append(py_root_dir_path)
+import warnings
+
+import tensorflow as tf
+
+
+from easy_rec.python.main import _train_and_evaluate_impl
+from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig
+from easy_rec.python.protos.train_pb2 import DistributionStrategy
+from easy_rec.python.utils import config_util
+from easy_rec.python.utils import ds_util
+from easy_rec.python.utils import estimator_utils
+from easy_rec.python.utils import fg_util
+from easy_rec.python.utils import hpo_util
+from easy_rec.python.utils.config_util import process_neg_sampler_data_path
+from easy_rec.python.utils.config_util import set_eval_input_path
+from easy_rec.python.utils.config_util import set_train_input_path
+
+logging.basicConfig(level=logging.INFO)
+warnings.filterwarnings('ignore')
+
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
+if tf.__version__.startswith('1.'):
+  from tensorflow.python.platform import gfile
+else:
+  import tensorflow.io.gfile as gfile
+
+from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_train_worker_num_on_ds  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
+
+logging.basicConfig(
+    format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s',
+    level=logging.INFO)
+
+
+def _get_file_path(root_path, file_list):
+  # 获取该目录下所有的文件名称和目录名称
+  dir_or_files = os.listdir(root_path)
+  for dir_file in dir_or_files:
+    # 获取目录或者文件的路径
+    dir_file_path = os.path.join(root_path, dir_file)
+    # 判断该路径为文件还是路径
+    if os.path.isdir(dir_file_path):
+      # 递归获取所有文件和目录的路径
+      _get_file_path(dir_file_path, file_list)
+    else:
+      if not str(dir_file_path).__contains__('_SUCCESS'):
+        file_list.append(dir_file_path)
+
+
+def get_vocab_list(vocab_path):
+  with gfile.GFile(vocab_path, 'r') as fin:
+    vocabulary_list = [str(line).strip() for line in fin]
+    return vocabulary_list
+
+
+def get_file_path_list(root_path):
+  file_list = []
+  _get_file_path(root_path, file_list)
+  return file_list
+
+
+def change_pipeline_config(pipeline_config: EasyRecConfig):
+  for data in pipeline_config.feature_config.features:
+    # print("****"*10)
+    vocab_file = data.vocab_list
+    if vocab_file:
+      vocab_file_new = get_file_path_list(f"{data_root_path}/{vocab_file.pop()}")[0]
+      # print(vocab_file_new)
+      vocab_list = get_vocab_list(vocab_file_new)
+      for vocab in vocab_list:
+        data.vocab_list.append(vocab)
+
+  model_dir = pipeline_config.model_dir
+  pipeline_config.model_dir = f"{data_root_path}/{model_dir}"
+
+  train_input_path = f"{data_root_path}/{pipeline_config.train_input_path}"
+  train_input_path_new = get_file_path_list(train_input_path)
+  pipeline_config.train_input_path = ','.join(train_input_path_new)
+
+  eval_input_path = f"{data_root_path}/{pipeline_config.eval_input_path}"
+  eval_input_path_new = get_file_path_list(eval_input_path)
+  pipeline_config.eval_input_path = ','.join(eval_input_path_new)
+
+  pipeline_config.data_config.batch_size = batch_size
+  pipeline_config.data_config.num_epochs = num_epochs
+
+  pipeline_config.train_config.log_step_count_steps = int(train_sample_cnt /
+                                                          batch_size)
+  pipeline_config.train_config.save_checkpoints_steps = int(train_sample_cnt /
+                                                            batch_size)
+
+  pipeline_config.train_config.optimizer_config[
+      0].adam_optimizer.learning_rate.exponential_decay_learning_rate.initial_learning_rate = initial_learning_rate
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--pipeline_config_path',
+      type=str,
+      # default="/Users/chensheng/PycharmProjects/EasyRec/samples/model_config/deepfm_on_criteo_tfrecord.config",
+      default='/Users/chensheng/PycharmProjects/EasyRec/samples/model_config/custom_model.config',
+      help='Path to pipeline config file.')
+
+  parser.add_argument(
+      '--data_root_path',
+      type = str,
+      default= '/Users/chensheng/PycharmProjects/EasyRec/data/test/cs_data'
+  )
+
+  parser.add_argument(
+      '--train_sample_cnt',
+      type=int,
+      default=27000,
+      help='训练集合的样本数,该数与save_checkpoints_steps 数值相等')
+
+  parser.add_argument(
+      '--batch_size',
+      type=int,
+      default=3000,
+  )
+
+  parser.add_argument(
+      '--num_epochs',
+      type=int,
+      default=10,
+  )
+  parser.add_argument(
+      '--initial_learning_rate',
+      type=float,
+      default=0.001,
+  )
+
+  parser.add_argument(
+      '--continue_train',
+      action='store_true',
+      default=False,
+      help='continue train using existing model_dir')
+  parser.add_argument(
+      '--hpo_param_path',
+      type=str,
+      default=None,
+      help='hyperparam tuning param path')
+  parser.add_argument(
+      '--hpo_metric_save_path',
+      type=str,
+      default=None,
+      help='hyperparameter save metric path')
+  parser.add_argument(
+      '--model_dir',
+      type=str,
+      default=None,
+      help='will update the model_dir in pipeline_config')
+  parser.add_argument(
+      '--train_input_path',
+      type=str,
+      nargs='*',
+      default=None,
+      help='train data input path')
+  parser.add_argument(
+      '--eval_input_path',
+      type=str,
+      nargs='*',
+      default=None,
+      help='eval data input path')
+  parser.add_argument(
+      '--fit_on_eval',
+      action='store_true',
+      default=False,
+      help='Fit evaluation data after fitting and evaluating train data')
+  parser.add_argument(
+      '--fit_on_eval_steps',
+      type=int,
+      default=None,
+      help='Fit evaluation data steps')
+  parser.add_argument(
+      '--fine_tune_checkpoint',
+      type=str,
+      default=None,
+      help='will update the train_config.fine_tune_checkpoint in pipeline_config'
+  )
+  parser.add_argument(
+      '--edit_config_json',
+      type=str,
+      default=None,
+      help='edit pipeline config str, example: {"model_dir":"experiments/",'
+      '"feature_config.feature[0].boundaries":[4,5,6,7]}')
+  parser.add_argument(
+      '--ignore_finetune_ckpt_error',
+      action='store_true',
+      default=False,
+      help='During incremental training, ignore the problem of missing fine_tune_checkpoint files'
+  )
+  parser.add_argument(
+      '--odps_config', type=str, default=None, help='odps config path')
+  parser.add_argument(
+      '--is_on_ds', action='store_true', default=False, help='is on ds')
+  parser.add_argument(
+      '--check_mode',
+      action='store_true',
+      default=False,
+      help='is use check mode')
+  parser.add_argument(
+      '--selected_cols', type=str, default=None, help='select input columns')
+  parser.add_argument('--gpu', type=str, default=None, help='gpu id')
+  args, extra_args = parser.parse_known_args()
+
+  data_root_path = args.data_root_path
+  train_sample_cnt = args.train_sample_cnt
+  batch_size = args.batch_size
+  num_epochs = args.num_epochs
+  initial_learning_rate = args.initial_learning_rate
+
+  if args.gpu is not None:
+    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
+
+  edit_config_json = {}
+  if args.edit_config_json:
+    edit_config_json = json.loads(args.edit_config_json)
+
+  if extra_args is not None and len(extra_args) > 0:
+    config_util.parse_extra_config_param(extra_args, edit_config_json)
+
+  if args.pipeline_config_path is not None:
+    pipeline_config = config_util.get_configs_from_pipeline_file(
+        args.pipeline_config_path, False)
+    if args.selected_cols:
+      pipeline_config.data_config.selected_cols = args.selected_cols
+    if args.model_dir:
+      pipeline_config.model_dir = args.model_dir
+      logging.info('update model_dir to %s' % pipeline_config.model_dir)
+    if args.train_input_path:
+      set_train_input_path(pipeline_config, args.train_input_path)
+    if args.eval_input_path:
+      set_eval_input_path(pipeline_config, args.eval_input_path)
+
+    if args.fine_tune_checkpoint:
+      ckpt_path = estimator_utils.get_latest_checkpoint_from_checkpoint_path(
+          args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error)
+
+      if ckpt_path:
+        pipeline_config.train_config.fine_tune_checkpoint = ckpt_path
+
+    if pipeline_config.fg_json_path:
+      fg_util.load_fg_json_to_config(pipeline_config)
+
+    if args.odps_config:
+      os.environ['ODPS_CONFIG_FILE_PATH'] = args.odps_config
+
+    if len(edit_config_json) > 0:
+      fine_tune_checkpoint = edit_config_json.get('train_config', {}).get(
+          'fine_tune_checkpoint', None)
+      if fine_tune_checkpoint:
+        ckpt_path = estimator_utils.get_latest_checkpoint_from_checkpoint_path(
+            args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error)
+        edit_config_json['train_config']['fine_tune_checkpoint'] = ckpt_path
+      config_util.edit_config(pipeline_config, edit_config_json)
+
+    process_neg_sampler_data_path(pipeline_config)
+
+    if args.is_on_ds:
+      ds_util.set_on_ds()
+      set_tf_config_and_get_train_worker_num_on_ds()
+      if pipeline_config.train_config.fine_tune_checkpoint:
+        ds_util.cache_ckpt(pipeline_config)
+
+    if pipeline_config.train_config.train_distribute in [
+        DistributionStrategy.HorovodStrategy,
+    ]:
+      estimator_utils.init_hvd()
+    elif pipeline_config.train_config.train_distribute in [
+        DistributionStrategy.EmbeddingParallelStrategy,
+        DistributionStrategy.SokStrategy
+    ]:
+      estimator_utils.init_hvd()
+      estimator_utils.init_sok()
+
+    if args.hpo_param_path:
+      with gfile.GFile(args.hpo_param_path, 'r') as fin:
+        hpo_config = json.load(fin)
+        hpo_params = hpo_config['param']
+        config_util.edit_config(pipeline_config, hpo_params)
+      config_util.auto_expand_share_feature_configs(pipeline_config)
+      _train_and_evaluate_impl(pipeline_config, args.continue_train,
+                               args.check_mode)
+      hpo_util.save_eval_metrics(
+          pipeline_config.model_dir,
+          metric_save_path=args.hpo_metric_save_path,
+          has_evaluator=False)
+    else:
+
+      change_pipeline_config(pipeline_config)
+
+      if args.continue_train:
+        pass
+      else:
+        model_dir = pipeline_config.model_dir
+        print(f'model_dir:{model_dir}')
+        os.system(f'rm -rf {model_dir}')
+
+
+      config_util.auto_expand_share_feature_configs(pipeline_config)
+      _train_and_evaluate_impl(
+          pipeline_config,
+          args.continue_train,
+          args.check_mode,
+          fit_on_eval=args.fit_on_eval,
+          fit_on_eval_steps=args.fit_on_eval_steps)
+  else:
+    raise ValueError('pipeline_config_path should not be empty when training!')

easy_rec/python/utils/__init__.py

@@ -0,0 +1,15 @@
+class conditional(object):
+  """Wrap another context manager and enter it only if condition is true."""
+
+  def __init__(self, condition, contextmanager):
+    self.condition = condition
+    self.contextmanager = contextmanager
+
+  def __enter__(self):
+    """Conditionally enter a context manager."""
+    if self.condition:
+      return self.contextmanager.__enter__()
+
+  def __exit__(self, *args):
+    if self.condition:
+      return self.contextmanager.__exit__(*args)

easy_rec/python/utils/activation.py

@@ -0,0 +1,120 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import numpy as np
+import six
+import tensorflow as tf
+
+from easy_rec.python.utils.load_class import load_by_path
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def dice(_x, axis=-1, epsilon=1e-9, name='dice', training=True):
+  """The Data Adaptive Activation Function in DIN.
+
+  Which can be viewed as a generalization of PReLu,
+  and can adaptively adjust the rectified point according to distribution of input data.
+
+  Arguments
+    - **axis** : Integer, the axis that should be used to compute data distribution (typically the features axis).
+    - **epsilon** : Small float added to variance to avoid dividing by zero.
+
+  References
+    - [Zhou G, Zhu X, Song C, et al. Deep interest network for click-through rate prediction[C]
+     Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining.
+     ACM, 2018: 1059-1068.] (https://arxiv.org/pdf/1706.06978.pdf)
+  """
+  alphas = tf.get_variable(
+      'alpha_' + name,
+      _x.get_shape()[-1],
+      initializer=tf.constant_initializer(0.0),
+      dtype=tf.float32)
+  inputs_normed = tf.layers.batch_normalization(
+      inputs=_x,
+      axis=axis,
+      epsilon=epsilon,
+      center=False,
+      scale=False,
+      training=training)
+  x_p = tf.sigmoid(inputs_normed)
+  return alphas * (1.0 - x_p) * _x + x_p * _x
+
+
+def gelu(x, name='gelu'):
+  """Gaussian Error Linear Unit.
+
+  This is a smoother version of the RELU.
+  Original paper: https://arxiv.org/abs/1606.08415
+
+  Args:
+    x: float Tensor to perform activation.
+    name: name for this activation
+
+  Returns:
+    `x` with the GELU activation applied.
+  """
+  with tf.name_scope(name):
+    cdf = 0.5 * (1.0 + tf.tanh(
+        (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3)))))
+    return x * cdf
+
+
+def swish(x, name='swish'):
+  with tf.name_scope(name):
+    return x * tf.sigmoid(x)
+
+
+def get_activation(activation_string, **kwargs):
+  """Maps a string to a Python function, e.g., "relu" => `tf.nn.relu`.
+
+  Args:
+    activation_string: String name of the activation function.
+
+  Returns:
+    A Python function corresponding to the activation function. If
+    `activation_string` is None, empty, or "linear", this will return None.
+    If `activation_string` is not a string, it will return `activation_string`.
+
+  Raises:
+    ValueError: The `activation_string` does not correspond to a known
+      activation.
+  """
+  # We assume that anything that's not a string is already an activation
+  # function, so we just return it.
+  if not isinstance(activation_string, six.string_types):
+    return activation_string
+
+  if not activation_string:
+    return None
+
+  act = activation_string.lower()
+  if act == 'linear':
+    return None
+  elif act == 'relu':
+    return tf.nn.relu
+  elif act == 'gelu':
+    return gelu
+  elif act == 'leaky_relu':
+    return tf.nn.leaky_relu
+  elif act == 'prelu':
+    if len(kwargs) == 0:
+      return tf.nn.leaky_relu
+    return tf.keras.layers.PReLU(**kwargs)
+  elif act == 'dice':
+    return lambda x, name='dice': dice(x, name=name, **kwargs)
+  elif act == 'elu':
+    return tf.nn.elu
+  elif act == 'selu':
+    return tf.nn.selu
+  elif act == 'tanh':
+    return tf.tanh
+  elif act == 'swish':
+    if tf.__version__ < '1.13.0':
+      return swish
+    return tf.nn.swish
+  elif act == 'sigmoid':
+    return tf.nn.sigmoid
+  else:
+    return load_by_path(activation_string)

easy_rec/python/utils/check_utils.py

@@ -0,0 +1,87 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import tensorflow as tf
+
+from easy_rec.python.protos.dataset_pb2 import DatasetConfig
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def check_split(line, sep, requried_field_num, field_name=''):
+  assert sep, 'must have separator.' + (' field: %s.' %
+                                        field_name) if field_name else ''
+
+  for one_line in line:
+    field_num = len(one_line.split(sep))
+    if field_name:
+      assert_info = 'sep[%s] maybe invalid. field_num=%d, required_num=%d, field: %s, value: %s, ' \
+                    'please check separator and data.' % \
+                    (sep, field_num, requried_field_num, field_name, one_line)
+    else:
+      assert_info = 'sep[%s] maybe invalid. field_num=%d, required_num=%d, current line is: %s, ' \
+                    'please check separator and data.' % \
+                    (sep, field_num, requried_field_num, one_line)
+    assert field_num == requried_field_num, assert_info
+  return True
+
+
+def check_string_to_number(field_vals, field_name):
+  for val in field_vals:
+    try:
+      float(val)
+    except:  # noqa: E722
+      assert False, 'StringToNumber ERROR: cannot convert string_to_number, field: %s, value: %s. ' \
+                    'please check data.' % (field_name, val)
+  return True
+
+
+def check_sequence(pipeline_config_path, features):
+  seq_att_groups = pipeline_config_path.model_config.seq_att_groups
+  if not seq_att_groups:
+    return
+  for seq_att_group in seq_att_groups:
+    seq_att_maps = seq_att_group.seq_att_map
+    if not seq_att_maps:
+      return
+    for seq_att_map in seq_att_maps:
+      assert len(seq_att_map.key) == len(seq_att_map.hist_seq), \
+          'The size of hist_seq must equal to the size of key in one seq_att_map.'
+      size_list = []
+      for hist_seq in seq_att_map.hist_seq:
+        cur_seq_size = len(features[hist_seq].values)
+        size_list.append(cur_seq_size)
+      hist_seqs = ' '.join(seq_att_map.hist_seq)
+      assert len(set(size_list)) == 1, \
+          'SequenceFeature Error: The size in [%s] should be consistent. Please check input: [%s].' % \
+          (hist_seqs, hist_seqs)
+
+
+def check_env_and_input_path(pipeline_config, input_path):
+  input_type = pipeline_config.data_config.input_type
+  input_type_name = DatasetConfig.InputType.Name(input_type)
+  ignore_input_list = [
+      DatasetConfig.InputType.TFRecordInput,
+      DatasetConfig.InputType.BatchTFRecordInput,
+      DatasetConfig.InputType.KafkaInput,
+      DatasetConfig.InputType.DataHubInput,
+      DatasetConfig.InputType.HiveInput,
+      DatasetConfig.InputType.DummyInput,
+  ]
+  if input_type in ignore_input_list:
+    return True
+  assert_info = 'Current InputType is %s, InputPath is %s. Please check InputType and InputPath.' % \
+                (input_type_name, input_path)
+  if input_type_name.startswith('Odps'):
+    # is on pai
+    for path in input_path.split(','):
+      if not path.startswith('odps://'):
+        assert False, assert_info
+    return True
+  else:
+    # local or ds
+    for path in input_path.split(','):
+      if path.startswith('odps://'):
+        assert False, assert_info
+  return True

easy_rec/python/utils/compat.py

@@ -0,0 +1,14 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Date: 2019-10-12
+# util to hanlde python2 python3 compatibility
+
+import sys
+
+
+def in_python2():
+  return sys.version_info[0] == 2
+
+
+def in_python3():
+  return sys.version_info[0] == 3