easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/test/csv_input_test.py

@@ -0,0 +1,340 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""Define cv_input, the base class for cv tasks."""
+
+import os
+import unittest
+
+import tensorflow as tf
+from google.protobuf import text_format
+
+from easy_rec.python.input.csv_input import CSVInput
+from easy_rec.python.input.csv_input_ex import CSVInputEx
+from easy_rec.python.protos.dataset_pb2 import DatasetConfig
+from easy_rec.python.protos.feature_config_pb2 import FeatureConfig
+from easy_rec.python.utils import config_util
+from easy_rec.python.utils import constant
+from easy_rec.python.utils.test_utils import RunAsSubprocess
+
+if tf.__version__ >= '2.0':
+  from tensorflow.python.framework.ops import disable_eager_execution
+
+  disable_eager_execution()
+  tf = tf.compat.v1
+
+
+class CSVInputTest(tf.test.TestCase):
+
+  def __init__(self, methodName='CSVInputTest'):
+    super(CSVInputTest, self).__init__(methodName=methodName)
+    self._input_path = 'data/test/test.csv'
+    self._input_path_with_quote = 'data/test/test_with_quote.csv'
+
+  @RunAsSubprocess
+  def test_csv_data(self):
+    data_config_str = """
+      input_fields {
+        input_name: 'label'
+        input_type: FLOAT
+      }
+      input_fields {
+        input_name: 'field[1-3]'
+        input_type: STRING
+      }
+      label_fields: 'label'
+      batch_size: 1024
+      num_epochs: 10000
+      prefetch_size: 32
+      auto_expand_input_fields: true
+    """
+    feature_config_str = """
+      input_names: 'field1'
+      shared_names: 'field[2-3]'
+      feature_type: IdFeature
+      embedding_dim: 32
+      hash_bucket_size: 2000
+    """
+    dataset_config = DatasetConfig()
+    text_format.Merge(data_config_str, dataset_config)
+    feature_config = FeatureConfig()
+    text_format.Merge(feature_config_str, feature_config)
+    feature_configs = [feature_config]
+    empty_config = FeatureConfig()
+    empty_config.CopyFrom(feature_config)
+    while len(empty_config.input_names) > 0:
+      empty_config.input_names.pop()
+    while len(empty_config.shared_names) > 0:
+      empty_config.shared_names.pop()
+    for input_name in feature_config.shared_names:
+      input_names = config_util.auto_expand_names(input_name)
+      for tmp_name in input_names:
+        tmp_config = FeatureConfig()
+        tmp_config.CopyFrom(empty_config)
+        tmp_config.input_names.append(tmp_name)
+        feature_configs.append(tmp_config)
+    train_input_fn = CSVInput(dataset_config, feature_configs,
+                              self._input_path).create_input()
+    dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN)
+    iterator = dataset.make_initializable_iterator()
+    tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer)
+    features, labels = iterator.get_next()
+    init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS)
+    gpu_options = tf.GPUOptions(allow_growth=True)
+    session_config = tf.ConfigProto(
+        gpu_options=gpu_options,
+        allow_soft_placement=True,
+        log_device_placement=False)
+    with self.test_session(config=session_config) as sess:
+      sess.run(init_op)
+      feature_dict, label_dict = sess.run([features, labels])
+
+  @RunAsSubprocess
+  def test_csv_data_flt_to_str_exception(self):
+    data_config_str = """
+      input_fields {
+        input_name: 'label'
+        input_type: FLOAT
+      }
+      input_fields {
+        input_name: 'field1'
+        input_type: STRING
+      }
+      input_fields {
+        input_name: 'field[2-3]'
+        input_type: FLOAT
+      }
+      label_fields: 'label'
+      batch_size: 1024
+      num_epochs: 10000
+      prefetch_size: 32
+      auto_expand_input_fields: true
+    """
+    feature_config_str = """
+      input_names: 'field1'
+      shared_names: 'field[2-3]'
+      feature_type: IdFeature
+      embedding_dim: 32
+      hash_bucket_size: 2000
+    """
+
+    dataset_config = DatasetConfig()
+    text_format.Merge(data_config_str, dataset_config)
+    feature_config = FeatureConfig()
+    text_format.Merge(feature_config_str, feature_config)
+    feature_configs = [feature_config]
+    empty_config = FeatureConfig()
+    empty_config.CopyFrom(feature_config)
+    while len(empty_config.input_names) > 0:
+      empty_config.input_names.pop()
+    while len(empty_config.shared_names) > 0:
+      empty_config.shared_names.pop()
+    for input_name in feature_config.shared_names:
+      input_names = config_util.auto_expand_names(input_name)
+      for tmp_name in input_names:
+        tmp_config = FeatureConfig()
+        tmp_config.CopyFrom(empty_config)
+        tmp_config.input_names.append(tmp_name)
+        feature_configs.append(tmp_config)
+    train_input_fn = CSVInput(dataset_config, feature_configs,
+                              self._input_path).create_input()
+    try:
+      dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN)  # noqa: F841
+      passed = True
+    except Exception:
+      passed = False
+    assert not passed, 'if precision is not set, exception should be reported in convert float to string'
+
+  @RunAsSubprocess
+  def test_csv_data_flt_to_str(self):
+    data_config_str = """
+      input_fields {
+        input_name: 'label'
+        input_type: FLOAT
+      }
+      input_fields {
+        input_name: 'field1'
+        input_type: STRING
+      }
+      input_fields {
+        input_name: 'field[2-3]'
+        input_type: FLOAT
+      }
+      label_fields: 'label'
+      batch_size: 1024
+      num_epochs: 10000
+      prefetch_size: 32
+      auto_expand_input_fields: true
+    """
+    feature_config_str = """
+      input_names: 'field1'
+      shared_names: 'field[2-3]'
+      feature_type: IdFeature
+      embedding_dim: 32
+      hash_bucket_size: 2000
+      precision: 3
+    """
+
+    dataset_config = DatasetConfig()
+    text_format.Merge(data_config_str, dataset_config)
+    feature_config = FeatureConfig()
+    text_format.Merge(feature_config_str, feature_config)
+    feature_configs = [feature_config]
+    empty_config = FeatureConfig()
+    empty_config.CopyFrom(feature_config)
+    while len(empty_config.input_names) > 0:
+      empty_config.input_names.pop()
+    while len(empty_config.shared_names) > 0:
+      empty_config.shared_names.pop()
+    for input_name in feature_config.shared_names:
+      input_names = config_util.auto_expand_names(input_name)
+      for tmp_name in input_names:
+        tmp_config = FeatureConfig()
+        tmp_config.CopyFrom(empty_config)
+        tmp_config.input_names.append(tmp_name)
+        feature_configs.append(tmp_config)
+    train_input_fn = CSVInput(dataset_config, feature_configs,
+                              self._input_path).create_input()
+
+    dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN)
+
+    iterator = dataset.make_initializable_iterator()
+    tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer)
+    features, labels = iterator.get_next()
+    init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS)
+    gpu_options = tf.GPUOptions(allow_growth=True)
+    session_config = tf.ConfigProto(
+        gpu_options=gpu_options,
+        allow_soft_placement=True,
+        log_device_placement=False)
+    with self.test_session(config=session_config) as sess:
+      sess.run(init_op)
+      feature_dict, label_dict = sess.run([features, labels])
+
+  @RunAsSubprocess
+  def test_csv_input_ex(self):
+    data_config_str = """
+      input_fields {
+        input_name: 'label'
+        input_type: FLOAT
+      }
+      input_fields {
+        input_name: 'field[1-3]'
+        input_type: STRING
+      }
+      label_fields: 'label'
+      batch_size: 1024
+      num_epochs: 10000
+      prefetch_size: 32
+      auto_expand_input_fields: true
+    """
+    feature_config_str = """
+      input_names: 'field1'
+      shared_names: 'field[2-3]'
+      feature_type: IdFeature
+      embedding_dim: 32
+      hash_bucket_size: 2000
+    """
+    dataset_config = DatasetConfig()
+    text_format.Merge(data_config_str, dataset_config)
+    feature_config = FeatureConfig()
+    text_format.Merge(feature_config_str, feature_config)
+    feature_configs = [feature_config]
+    empty_config = FeatureConfig()
+    empty_config.CopyFrom(feature_config)
+    while len(empty_config.input_names) > 0:
+      empty_config.input_names.pop()
+    while len(empty_config.shared_names) > 0:
+      empty_config.shared_names.pop()
+    for input_name in feature_config.shared_names:
+      input_names = config_util.auto_expand_names(input_name)
+      for tmp_name in input_names:
+        tmp_config = FeatureConfig()
+        tmp_config.CopyFrom(empty_config)
+        tmp_config.input_names.append(tmp_name)
+        feature_configs.append(tmp_config)
+    train_input_fn = CSVInputEx(dataset_config, feature_configs,
+                                self._input_path_with_quote).create_input()
+    dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN)
+    iterator = dataset.make_initializable_iterator()
+    tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer)
+    features, labels = iterator.get_next()
+    init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS)
+    gpu_options = tf.GPUOptions(allow_growth=True)
+    session_config = tf.ConfigProto(
+        gpu_options=gpu_options,
+        allow_soft_placement=True,
+        log_device_placement=False)
+    with self.test_session(config=session_config) as sess:
+      sess.run(init_op)
+      feature_dict, label_dict = sess.run([features, labels])
+
+  @unittest.skipIf('AVX_TEST' not in os.environ,
+                   'Only execute when avx512 instructions are supported')
+  @RunAsSubprocess
+  def test_csv_input_ex_avx(self):
+    constant.enable_avx_str_split()
+    self.test_csv_input_ex()
+    constant.disable_avx_str_split()
+
+  @RunAsSubprocess
+  def test_csv_data_ignore_error(self):
+    data_config_str = """
+      input_fields {
+        input_name: 'label'
+        input_type: FLOAT
+      }
+      input_fields {
+        input_name: 'field[1-3]'
+        input_type: STRING
+      }
+      label_fields: 'label'
+      batch_size: 32
+      num_epochs: 10000
+      prefetch_size: 32
+      auto_expand_input_fields: true
+      ignore_error: true
+    """
+    feature_config_str = """
+      input_names: 'field1'
+      shared_names: 'field[2-3]'
+      feature_type: IdFeature
+      embedding_dim: 32
+      hash_bucket_size: 2000
+    """
+    dataset_config = DatasetConfig()
+    text_format.Merge(data_config_str, dataset_config)
+    feature_config = FeatureConfig()
+    text_format.Merge(feature_config_str, feature_config)
+    feature_configs = [feature_config]
+    empty_config = FeatureConfig()
+    empty_config.CopyFrom(feature_config)
+    while len(empty_config.input_names) > 0:
+      empty_config.input_names.pop()
+    while len(empty_config.shared_names) > 0:
+      empty_config.shared_names.pop()
+    for input_name in feature_config.shared_names:
+      input_names = config_util.auto_expand_names(input_name)
+      for tmp_name in input_names:
+        tmp_config = FeatureConfig()
+        tmp_config.CopyFrom(empty_config)
+        tmp_config.input_names.append(tmp_name)
+        feature_configs.append(tmp_config)
+    train_input_fn = CSVInput(dataset_config, feature_configs,
+                              self._input_path_with_quote).create_input()
+    dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN)
+    iterator = dataset.make_initializable_iterator()
+    tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer)
+    features, labels = iterator.get_next()
+    init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS)
+    gpu_options = tf.GPUOptions(allow_growth=True)
+    session_config = tf.ConfigProto(
+        gpu_options=gpu_options,
+        allow_soft_placement=True,
+        log_device_placement=False)
+    with self.test_session(config=session_config) as sess:
+      sess.run(init_op)
+      feature_dict, label_dict = sess.run([features, labels])
+
+
+if __name__ == '__main__':
+  tf.test.main()

easy_rec/python/test/custom_early_stop_func.py

@@ -0,0 +1,19 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import json
+import logging
+
+
+def custom_early_stop_func(eval_results, func_param):
+  params = json.loads(func_param)
+  tmp_thre = params['thre']
+  metric_name = params['metric_name']
+  for step, metrics in eval_results.items():
+    val = metrics[metric_name]
+    if val > tmp_thre:
+      logging.info(
+          'At step %s, metric "%s" has value %s, which is larger than %s, causing early stop.'
+          % (step, metric_name, val, tmp_thre))
+      return True
+  return False

easy_rec/python/test/dh_local_run.py

@@ -0,0 +1,104 @@
+import argparse
+import logging
+import os
+import shutil
+import sys
+
+import tensorflow as tf
+
+from easy_rec.python.test.odps_command import OdpsCommand
+from easy_rec.python.test.odps_test_prepare import prepare
+from easy_rec.python.test.odps_test_util import OdpsOSSConfig
+from easy_rec.python.test.odps_test_util import delete_oss_path
+from easy_rec.python.test.odps_test_util import get_oss_bucket
+from easy_rec.python.utils import test_utils
+
+logging.basicConfig(
+    level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s')
+
+odps_oss_config = OdpsOSSConfig(script_path='./samples/dh_script')
+
+
+class TestPipelineOnEmr(tf.test.TestCase):
+  """Train eval test on emr."""
+
+  def setUp(self):
+    logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName))
+    self._success = True
+    self._test_dir = test_utils.get_tmp_dir()
+    logging.info('test datahub local dir: %s' % self._test_dir)
+
+  def tearDown(self):
+    if self._success:
+      shutil.rmtree(self._test_dir)
+
+  def test_datahub_train_eval(self):
+    end = ['deep_fm/drop_table.sql']
+    odps_cmd = OdpsCommand(odps_oss_config)
+
+    self._success = test_utils.test_datahub_train_eval(
+        '%s/configs/deepfm.config' % odps_oss_config.temp_dir, odps_oss_config,
+        self._test_dir)
+    odps_cmd.run_list(end)
+    self.assertTrue(self._success)
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--odps_config', type=str, default=None, help='odps config path')
+  parser.add_argument(
+      '--oss_config', type=str, default=None, help='ossutilconfig path')
+  parser.add_argument(
+      '--bucket_name', type=str, default=None, help='test oss bucket name')
+  parser.add_argument('--arn', type=str, default=None, help='oss rolearn')
+  parser.add_argument(
+      '--odpscmd', type=str, default='odpscmd', help='odpscmd path')
+  parser.add_argument(
+      '--algo_project', type=str, default=None, help='algo project name')
+  parser.add_argument(
+      '--algo_res_project',
+      type=str,
+      default=None,
+      help='algo resource project name')
+  parser.add_argument(
+      '--algo_version', type=str, default=None, help='algo version')
+  args, unknown_args = parser.parse_known_args()
+
+  sys.argv = [sys.argv[0]]
+  for unk_arg in unknown_args:
+    sys.argv.append(unk_arg)
+
+  if args.odps_config:
+    odps_oss_config.load_odps_config(args.odps_config)
+    os.environ['ODPS_CONFIG_FILE_PATH'] = args.odps_config
+  if args.oss_config:
+    odps_oss_config.load_oss_config(args.oss_config)
+  if args.odpscmd:
+    odps_oss_config.odpscmd_path = args.odpscmd
+  if args.algo_project:
+    odps_oss_config.algo_project = args.algo_project
+  if args.algo_res_project:
+    odps_oss_config.algo_res_project = args.algo_res_project
+  if args.algo_version:
+    odps_oss_config.algo_version = args.algo_version
+  if args.arn:
+    odps_oss_config.arn = args.arn
+  if args.bucket_name:
+    odps_oss_config.bucket_name = args.bucket_name
+  prepare(odps_oss_config)
+  start = [
+      'deep_fm/create_external_deepfm_table.sql',
+      'deep_fm/create_inner_deepfm_table.sql'
+  ]
+  end = ['deep_fm/drop_table.sql']
+  odps_cmd = OdpsCommand(odps_oss_config)
+  odps_cmd.run_list(start)
+  odps_oss_config.init_dh_and_odps()
+  tf.test.main()
+  # delete oss path
+  bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret,
+                          odps_oss_config.endpoint, odps_oss_config.bucket_name)
+  delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name)
+  # delete tmp
+  shutil.rmtree(odps_oss_config.temp_dir)

easy_rec/python/test/embed_test.py

@@ -0,0 +1,155 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import logging
+
+import numpy as np
+import tensorflow as tf
+from google.protobuf import text_format
+
+from easy_rec.python.compat.feature_column import feature_column
+from easy_rec.python.feature_column.feature_column import FeatureColumnParser
+from easy_rec.python.input.dummy_input import DummyInput
+from easy_rec.python.protos.dataset_pb2 import DatasetConfig
+from easy_rec.python.protos.feature_config_pb2 import FeatureConfig
+from easy_rec.python.protos.feature_config_pb2 import WideOrDeep
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class EmbedTest(tf.test.TestCase):
+
+  def test_raw_embed(self):
+    # embedding variable is:
+    #    [[1, 2 ],
+    #     [3, 4 ],
+    #     [5, 6 ],
+    #     [7, 8 ],
+    #     [9, 10]
+    #    ]
+    feature_config_str = '''
+      input_names: 'field1'
+      feature_type: RawFeature
+      initializer {
+         constant_initializer {
+            consts: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+         }
+      }
+      separator: ',',
+      raw_input_dim: 5
+      embedding_dim: 2
+      combiner: 'sum'
+    '''
+    feature_config = FeatureConfig()
+    text_format.Merge(feature_config_str, feature_config)
+
+    data_config_str = '''
+        input_fields {
+           input_name: 'clk'
+           input_type: INT32
+           default_val: '0'
+        }
+        input_fields {
+           input_name: 'field1'
+           input_type: STRING
+           default_val: '0'
+        }
+        label_fields: 'clk'
+        batch_size: 1
+    '''
+    data_config = DatasetConfig()
+    text_format.Merge(data_config_str, data_config)
+
+    feature_configs = [feature_config]
+    features = {'field1': tf.constant(['0.1,0.2,0.3,0.4,0.5'])}
+    dummy_input = DummyInput(
+        data_config, feature_configs, '', input_vals=features)
+    field_dict, _ = dummy_input._build(tf.estimator.ModeKeys.TRAIN, {})
+
+    wide_and_deep_dict = {'field1': WideOrDeep.WIDE_AND_DEEP}
+    fc_parser = FeatureColumnParser(feature_configs, wide_and_deep_dict, 2)
+    wide_cols = list(fc_parser._wide_columns.values())
+    wide_features = feature_column.input_layer(field_dict, wide_cols)
+    deep_cols = list(fc_parser._deep_columns.values())
+    deep_features = feature_column.input_layer(field_dict, deep_cols)
+    init = tf.initialize_all_variables()
+    with tf.Session() as sess:
+      sess.run(init)
+      fea_val = sess.run(wide_features)
+      logging.info('wide fea_val = %s' % str(fea_val[0]))
+      assert np.abs(fea_val[0][0] - 9.5) < 1e-6
+      assert np.abs(fea_val[0][1] - 11.0) < 1e-6
+      fea_val = sess.run(deep_features)
+      logging.info('deep fea_val = %s' % str(fea_val[0]))
+      assert np.abs(fea_val[0][0] - 9.5) < 1e-6
+      assert np.abs(fea_val[0][1] - 11.0) < 1e-6
+
+  def test_seq_multi_embed(self):
+    # embedding variable is:
+    #    [[1, 2 ],
+    #     [3, 4 ],
+    #     [5, 6 ],
+    #     [7, 8 ],
+    #     [9, 10]
+    #    ]
+    feature_config_str = '''
+      input_names: 'field1'
+      feature_type: SequenceFeature
+      initializer {
+         constant_initializer {
+            consts: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+         }
+      }
+      separator: '',
+      seq_multi_sep: '',
+      embedding_dim: 2
+      num_buckets: 5
+      combiner: 'mean'
+    '''
+    feature_config = FeatureConfig()
+    text_format.Merge(feature_config_str, feature_config)
+
+    data_config_str = '''
+        input_fields {
+           input_name: 'clk'
+           input_type: INT32
+           default_val: '0'
+        }
+        input_fields {
+           input_name: 'field1'
+           input_type: STRING
+           default_val: '0'
+        }
+        label_fields: 'clk'
+        batch_size: 1
+    '''
+    data_config = DatasetConfig()
+    text_format.Merge(data_config_str, data_config)
+
+    feature_configs = [feature_config]
+    features = {'field1': tf.constant(['0112', '132430'])}
+    dummy_input = DummyInput(
+        data_config, feature_configs, '', input_vals=features)
+    field_dict, _ = dummy_input._build(tf.estimator.ModeKeys.TRAIN, {})
+
+    wide_and_deep_dict = {'field1': WideOrDeep.DEEP}
+    fc_parser = FeatureColumnParser(feature_configs, wide_and_deep_dict)
+    builder = feature_column._LazyBuilder(field_dict)
+    hist_embedding, hist_seq_len = \
+        fc_parser.sequence_columns['field1']._get_sequence_dense_tensor(builder)
+
+    init = tf.initialize_all_variables()
+    with tf.Session() as sess:
+      sess.run(init)
+      fea_val, len_val = sess.run([hist_embedding, hist_seq_len])
+      logging.info('length_val = %s' % str(len_val))
+      logging.info('deep fea_val = %s' % str(fea_val))
+      assert np.abs(fea_val[0][0][0] - 2) < 1e-6
+      assert np.abs(fea_val[0][0][1] - 3) < 1e-6
+      assert np.abs(fea_val[0][1][0] - 4) < 1e-6
+      assert np.abs(fea_val[0][1][1] - 5) < 1e-6
+
+
+if __name__ == '__main__':
+  tf.test.main()