easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/test/odps_local_run.py

@@ -0,0 +1,86 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import argparse
+import logging
+import os
+import shutil
+import sys
+
+import tensorflow as tf
+
+from easy_rec.python.test.odps_command import OdpsCommand
+from easy_rec.python.test.odps_test_prepare import prepare
+from easy_rec.python.test.odps_test_util import OdpsOSSConfig
+from easy_rec.python.test.odps_test_util import delete_oss_path
+from easy_rec.python.test.odps_test_util import get_oss_bucket
+from easy_rec.python.utils import test_utils
+
+logging.basicConfig(
+    level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s')
+
+odps_oss_config = OdpsOSSConfig(script_path='./samples/emr_script')
+
+
+class TestPipelineLocal(tf.test.TestCase):
+  """Train eval test on emr."""
+
+  def setUp(self):
+    logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName))
+    self._success = True
+    self._test_dir = test_utils.get_tmp_dir()
+
+  def tearDown(self):
+    if self._success:
+      shutil.rmtree(self._test_dir)
+
+  def test_deepfm_local_with_common_io(self):
+    start = [
+        'deep_fm/create_external_deepfm_table.sql',
+        'deep_fm/create_inner_deepfm_table.sql'
+    ]
+    end = ['deep_fm/drop_table.sql']
+    odps_cmd = OdpsCommand(odps_oss_config)
+    odps_cmd.run_list(start)
+    self._success = test_utils.test_single_train_eval(
+        '%s/configs/deepfm.config' % odps_oss_config.temp_dir, self._test_dir)
+    odps_cmd.run_list(end)
+    self.assertTrue(self._success)
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--odps_config', type=str, default=None, help='odps config path')
+  parser.add_argument(
+      '--oss_config', type=str, default=None, help='ossutilconfig path')
+  parser.add_argument(
+      '--bucket_name', type=str, default=None, help='test oss bucket name')
+  parser.add_argument('--arn', type=str, default=None, help='oss rolearn')
+  parser.add_argument(
+      '--odpscmd', type=str, default='odpscmd', help='odpscmd path')
+  args, unknown_args = parser.parse_known_args()
+  sys.argv = [sys.argv[0]]
+  for unk_arg in unknown_args:
+    sys.argv.append(unk_arg)
+
+  if args.odps_config:
+    odps_oss_config.load_odps_config(args.odps_config)
+    os.environ['ODPS_CONFIG_FILE_PATH'] = args.odps_config
+  if args.oss_config:
+    odps_oss_config.load_oss_config(args.oss_config)
+  if args.odpscmd:
+    odps_oss_config.odpscmd_path = args.odpscmd
+  if args.arn:
+    odps_oss_config.arn = args.arn
+  if args.bucket_name:
+    odps_oss_config.bucket_name = args.bucket_name
+
+  prepare(odps_oss_config)
+  tf.test.main()
+  # delete oss path
+  bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret,
+                          odps_oss_config.endpoint, odps_oss_config.bucket_name)
+  delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name)
+  # delete tmp
+  shutil.rmtree(odps_oss_config.temp_dir)

easy_rec/python/test/odps_run.py

@@ -0,0 +1,254 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import argparse
+import logging
+import os
+import shutil
+import sys
+
+import oss2
+import tensorflow as tf
+
+from easy_rec.python.test.odps_test_cls import OdpsTest
+from easy_rec.python.test.odps_test_prepare import prepare
+from easy_rec.python.test.odps_test_util import OdpsOSSConfig
+from easy_rec.python.test.odps_test_util import delete_oss_path
+from easy_rec.python.test.odps_test_util import get_oss_bucket
+from easy_rec.python.utils import config_util
+
+logging.basicConfig(
+    level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s')
+
+odps_oss_config = OdpsOSSConfig()
+
+
+class TestPipelineOnOdps(tf.test.TestCase):
+  """train eval export test on odps."""
+
+  def test_deepfm(self):
+    start_files = ['deep_fm/create_inner_deepfm_table.sql']
+    test_files = [
+        'deep_fm/train_deepfm_model.sql', 'deep_fm/eval_deepfm.sql',
+        'deep_fm/export_deepfm.sql', 'deep_fm/predict_deepfm.sql',
+        'deep_fm/export_rtp_ckpt.sql'
+    ]
+    end_file = ['deep_fm/drop_table.sql']
+
+    tot = OdpsTest(start_files, test_files, end_file, odps_oss_config)
+    tot.start_test()
+    tot.drop_table()
+
+  def test_mmoe(self):
+    start_files = ['mmoe/create_inner_mmoe_table.sql']
+    test_files = [
+        'mmoe/train_mmoe_model.sql',
+        'mmoe/eval_mmoe.sql',
+        'mmoe/export_mmoe.sql',
+        'mmoe/predict_mmoe.sql',
+    ]
+    end_file = ['mmoe/drop_mmoe_table.sql']
+    tot = OdpsTest(start_files, test_files, end_file, odps_oss_config)
+    tot.start_test()
+    tot.drop_table()
+
+  def test_dssm(self):
+    start_files = [
+        'dssm/create_inner_dssm_table.sql',
+    ]
+    test_files = [
+        'dssm/train_dssm_model.sql',
+        'dssm/eval_dssm.sql',
+        'dssm/export_dssm.sql',
+        'dssm/predict_dssm.sql',
+    ]
+    end_file = [
+        'dssm/drop_dssm_table.sql',
+    ]
+    tot = OdpsTest(start_files, test_files, end_file, odps_oss_config)
+    tot.start_test()
+    tot.drop_table()
+
+  def test_multi_tower(self):
+    start_files = ['multi_tower/create_inner_multi_tower_table.sql']
+    test_files = [
+        'multi_tower/train_multil_tower_din_model.sql',
+        'multi_tower/train_multil_tower_bst_model.sql',
+        'multi_tower/eval_multil_tower.sql',
+        'multi_tower/export_multil_tower.sql',
+        'multi_tower/export_again_multi_tower.sql',
+        'multi_tower/predict_multil_tower.sql',
+    ]
+    end_file = ['multi_tower/drop_multil_tower_table.sql']
+    tot = OdpsTest(start_files, test_files, end_file, odps_oss_config)
+    tot.start_test()
+    tot.drop_table()
+
+  def test_other(self):
+    start_files = ['deep_fm/create_inner_deepfm_table.sql']
+    test_files = [
+        # 'other_test/test_train_gpuRequired_mirrored', # 线上报错，
+        # 'other_test/test_train_distribute_strategy_collective',  # 线上报错，
+        'other_test/test_train_hpo_with_evaluator.sql',
+        # 'other_test/test_train_version.sql',
+        # 'other_test/test_train_distribute_strategy_ess.sql',
+        'other_test/test_train_before_export.sql',
+        'other_test/test_eval_checkpoint_path.sql',
+        'other_test/test_export_checkpoint_path.sql',
+        'other_test/test_export_update_model_dir.sql',
+        'other_test/test_predict_selected_cols.sql',
+    ]
+    end_file = ['other_test/drop_table.sql']
+    tot = OdpsTest(start_files, test_files, end_file, odps_oss_config)
+    tot.start_test()
+    tot.drop_table()
+
+  def test_best_exporter(self):
+    start_files = ['deep_fm/create_inner_deepfm_table.sql']
+    test_files = [
+        'other_test/test_train_best_export.sql',
+    ]
+    end_file = ['other_test/drop_table.sql']
+    tot = OdpsTest(start_files, test_files, end_file, odps_oss_config)
+    tot.start_test()
+    config_path = os.path.join(
+        odps_oss_config.temp_dir,
+        'configs/dwd_avazu_ctr_deepmodel_ext_best_export.config')
+    config = config_util.get_configs_from_pipeline_file(config_path)
+    model_dir = config.model_dir
+    logging.info('raw model_dir = %s' % model_dir)
+    if model_dir.startswith('oss://'):
+      spos = model_dir.index('/', len('oss://') + 1) + 1
+      model_dir = model_dir[spos:]
+    logging.info('stripped model_dir = %s' % model_dir)
+
+    bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret,
+                            odps_oss_config.endpoint,
+                            odps_oss_config.bucket_name)
+    best_ckpt_prefix = os.path.join(model_dir, 'best_ckpt/model.ckpt')
+    best_ckpts = [
+        x.key
+        for x in oss2.ObjectIterator(bucket, prefix=best_ckpt_prefix)
+        if x.key.endswith('.meta')
+    ]
+    logging.info('best ckpts: %s' % str(best_ckpts))
+    assert len(best_ckpts) <= 2, 'too many best ckpts: %s' % str(best_ckpts)
+    best_export_prefix = os.path.join(model_dir, 'export/best/')
+    best_exports = [
+        x.key
+        for x in oss2.ObjectIterator(bucket, prefix=best_export_prefix)
+        if x.key.endswith('/saved_model.pb')
+    ]
+    logging.info('best exports: %s' % str(best_exports))
+    assert len(
+        best_exports) <= 2, 'too many best exports: %s' % str(best_exports)
+    return True
+
+  def test_embedding_variable(self):
+    start_files = [
+        'embedding_variable/create_table.sql',
+    ]
+    test_files = [
+        'embedding_variable/train.sql', 'embedding_variable/train_work_que.sql',
+        'embedding_variable/export.sql'
+    ]
+    end_file = ['embedding_variable/drop_table.sql']
+    tot = OdpsTest(start_files, test_files, end_file, odps_oss_config)
+    tot.start_test()
+    tot.drop_table()
+
+  def test_multi_value_export(self):
+    start_files = ['multi_value/create_inner_multi_value_table.sql']
+    test_files = ['multi_value/train_multi_tower_model.sql']
+    end_file = ['multi_value/drop_table.sql']
+    tot = OdpsTest(start_files, test_files, end_file, odps_oss_config)
+    tot.start_test()
+    tot.drop_table()
+
+  def test_boundary_test(self):
+    start_files = [
+        'boundary/create_inner_boundary_table.sql',
+    ]
+    test_files = [
+        'boundary/train_multi_tower_model.sql',
+        'boundary/finetune_multi_tower_model.sql',
+        'boundary/finetune_multi_tower_conti.sql', 'boundary/train_compat.sql'
+    ]
+    end_file = ['boundary/drop_table.sql']
+    tot = OdpsTest(start_files, test_files, end_file, odps_oss_config)
+    tot.start_test()
+    tot.drop_table()
+
+  def test_vector_retrieve(self):
+    start_files = ['vector_retrieve/create_inner_vector_table.sql']
+    test_files = ['vector_retrieve/run_vector_retrieve.sql']
+    end_file = ['vector_retrieve/drop_table.sql']
+    tot = OdpsTest(start_files, test_files, end_file, odps_oss_config)
+    tot.start_test()
+    tot.drop_table()
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--odps_config', type=str, default=None, help='odps config path')
+  parser.add_argument(
+      '--oss_config', type=str, default=None, help='ossutilconfig path')
+  parser.add_argument(
+      '--bucket_name', type=str, default=None, help='test oss bucket name')
+  parser.add_argument('--arn', type=str, default=None, help='oss rolearn')
+  parser.add_argument(
+      '--odpscmd', type=str, default='odpscmd', help='odpscmd path')
+  parser.add_argument(
+      '--algo_name',
+      type=str,
+      default='easy_rec_ext',
+      help='whether use pai-tf 1.15')
+  parser.add_argument(
+      '--algo_project', type=str, default=None, help='algo project name')
+  parser.add_argument(
+      '--algo_res_project',
+      type=str,
+      default=None,
+      help='algo resource project name')
+  parser.add_argument(
+      '--algo_version', type=str, default=None, help='algo version')
+  parser.add_argument(
+      '--is_outer',
+      type=int,
+      default=1,
+      help='is outer pai or inner pai, the arguments are differed slightly due to history reasons'
+  )
+  args, unknown_args = parser.parse_known_args()
+  sys.argv = [sys.argv[0]]
+  for unk_arg in unknown_args:
+    sys.argv.append(unk_arg)
+
+  if args.odps_config:
+    odps_oss_config.load_odps_config(args.odps_config)
+  if args.oss_config:
+    odps_oss_config.load_oss_config(args.oss_config)
+  if args.odpscmd:
+    odps_oss_config.odpscmd_path = args.odpscmd
+  if args.algo_project:
+    odps_oss_config.algo_project = args.algo_project
+  if args.algo_res_project:
+    odps_oss_config.algo_res_project = args.algo_res_project
+  if args.algo_version:
+    odps_oss_config.algo_version = args.algo_version
+  algo_names = ['easy_rec_ext15', 'easy_rec_ext']
+  assert args.algo_name in algo_names, 'algo_name must be oneof: %s' % (
+      ','.join(algo_names))
+  odps_oss_config.algo_name = args.algo_name
+  if args.arn:
+    odps_oss_config.arn = args.arn
+  if args.bucket_name:
+    odps_oss_config.bucket_name = args.bucket_name
+  odps_oss_config.is_outer = args.is_outer
+
+  prepare(odps_oss_config)
+  tf.test.main()
+  bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret,
+                          odps_oss_config.endpoint, odps_oss_config.bucket_name)
+  delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name)
+  shutil.rmtree(odps_oss_config.temp_dir)

easy_rec/python/test/odps_test_cls.py

@@ -0,0 +1,39 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from multiprocessing import Process
+
+from easy_rec.python.test.odps_command import OdpsCommand
+
+
+class OdpsTest(Process):
+  """Test easyrec on odps."""
+
+  def __init__(self, start_files, test_files, end_file, odps_oss_config):
+    """Init odps test instance.
+
+    Args:
+      start_files: prepare sql files, to create tables
+      test_files: actual test sql files, such as train/eval/export
+      end_file: drop table sql files
+      odps_oss_config: easy_rec.python.test.odps_test_util.OdpsOSSConfig
+    """
+    super(OdpsTest, self).__init__()
+    self.start_files = start_files
+    self.test_files = test_files
+    self.end_file = end_file
+
+    self.odps = OdpsCommand(odps_oss_config)
+    self.init_table()
+
+  def init_table(self):
+    """Init table for test."""
+    self.odps.run_list(self.start_files)
+
+  def start_test(self):
+    """Exec every file in self.test_files."""
+    self.odps.run_list(self.test_files)
+
+  def drop_table(self):
+    """Drop temporary tables."""
+    self.odps.run_list(self.end_file)

easy_rec/python/test/odps_test_prepare.py

@@ -0,0 +1,198 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import glob
+import logging
+import os
+import shutil
+import sys
+
+import oss2
+
+from easy_rec.python.test.odps_test_util import OdpsOSSConfig
+from easy_rec.python.test.odps_test_util import get_oss_bucket
+
+
+def download_data(ali_bucket, script_path):
+  """Download data from alibaba bucket(readonly) to script_path.
+
+  Args:
+    ali_bucket: alibaba bucket
+    script_path: down path
+  """
+  if not os.path.exists(script_path):
+    raise '%s path is not exists' % script_path
+  if os.path.exists(os.path.join(script_path, 'test')):
+    shutil.rmtree(os.path.join(script_path, 'test'))
+
+  # download data from oss://${ali_bucket}/data/odps_test/
+  # to script_path/test_data
+  for obj in oss2.ObjectIterator(ali_bucket, prefix='data/odps_test/'):
+    obj_key = obj.key
+    tmp_oss_dir = os.path.split(obj_key)[0]
+    obj_path = os.path.join(script_path, tmp_oss_dir)
+    try:
+      os.makedirs(obj_path)
+    except Exception:
+      pass
+
+    if obj_key.endswith('/'):
+      continue
+
+    dst_name = obj_key.replace('data/odps_test/', 'test_data/')
+    dst_path = os.path.join(script_path, dst_name)
+    dst_dir, _ = os.path.split(dst_path)
+    if not os.path.exists(dst_dir):
+      os.makedirs(dst_dir)
+    ali_bucket.get_object_to_file(obj_key, dst_path)
+    logging.info('down file oss://%s/%s to %s completed' %
+                 (ali_bucket.bucket_name, obj_key, dst_path))
+
+
+def merge_files(merge_dir, merge_out):
+  """Merge files in merge_dir into merge_out.
+
+  Args:
+    merge_dir: files of directory to merge.
+    merge_out: merged output file.
+  """
+  input_files = list(glob.glob(merge_dir + '/*'))
+  logging.info('merge %s into %s' % (','.join(input_files), merge_out))
+  with open(merge_out, 'w') as fout:
+    for input_path in glob.glob(merge_dir + '/*'):
+      with open(input_path, 'r') as fin:
+        for line_str in fin:
+          fout.write(line_str)
+  return merge_out
+
+
+def change_files(odps_oss_config, file_path):
+  """Update params in config files.
+
+  Args:
+    odps_oss_config: odps oss test config
+    file_path: config file path
+  """
+  # skip temporary file
+  if len(file_path) > 3 and file_path[-4:-1] == '.sw':
+    return
+
+  endpoint = odps_oss_config.endpoint.replace('http://', '')
+  # endpoint_internal = endpoint.replace('.aliyuncs.com',
+  #                                      '-internal.aliyuncs.com')
+
+  test_data_dir = os.path.join(odps_oss_config.temp_dir, 'test_data')
+
+  with open(file_path, 'r') as fin:
+    lines = fin.readlines()
+
+  with open(file_path, 'w') as fw:
+    for line in lines:
+      if 'pai' in line.lower() and 'easy_rec_ext' in line.lower():
+        line = 'pai -name ' + odps_oss_config.algo_name + '\n'
+        if odps_oss_config.algo_project:
+          line += '-project=%s\n' % odps_oss_config.algo_project
+        if odps_oss_config.algo_res_project:
+          line += '-Dres_project=%s\n' % odps_oss_config.algo_res_project
+        if odps_oss_config.algo_version:
+          line += '-Dversion=%s\n' % odps_oss_config.algo_version
+
+      if odps_oss_config.is_outer:
+        line = line.replace('{OSS_BUCKET_NAME}', odps_oss_config.bucket_name)
+        line = line.replace('{ROLEARN}', odps_oss_config.arn)
+        line = line.replace('{OSS_ENDPOINT}', endpoint)
+      else:
+        tmp_e = odps_oss_config.endpoint
+        # tmp_e = tmp_e.replace('oss-cn-', 'cn-')
+        # tmp_e = tmp_e.replace('.aliyuncs.com', '.oss-internal.aliyun-inc.com')
+        if '-Dbuckets=' in line:
+          line = '-Dbuckets=oss://%s/?role_arn=%s&host=%s\n' % (
+              odps_oss_config.bucket_name, odps_oss_config.arn, tmp_e)
+        elif '-Darn=' in line or '-DossHost' in line:
+          continue
+        line = line.replace('{OSS_BUCKET_NAME}', odps_oss_config.bucket_name)
+
+      line = line.replace('{TIME_STAMP}', str(odps_oss_config.time_stamp))
+
+      if 'tunnel upload' in line:
+        line = line.replace('{TEST_DATA_DIR}', test_data_dir)
+        # merge files
+        toks = [x for x in line.split(' ') if x != '']
+        merge_path = toks[2]
+        merge_dir = '_'.join(merge_path.split('_')[:-1])
+        if not os.path.exists(merge_path):
+          merge_files(merge_dir, merge_path)
+
+      # for emr odps test only
+      line = line.replace('{TEMP_DIR}', str(odps_oss_config.temp_dir))
+      line = line.replace('{ODPS_PROJ_NAME}', odps_oss_config.project_name)
+      line = line.replace('{EXP_NAME}', odps_oss_config.exp_dir)
+      fw.write(line)
+
+
+def put_data_to_bucket(odps_oss_config):
+  """Upload test data and configs to my_bucket.
+
+  Args:
+    odps_oss_config: odps oss config obj
+  """
+  test_bucket = get_oss_bucket(odps_oss_config.oss_key,
+                               odps_oss_config.oss_secret,
+                               odps_oss_config.endpoint,
+                               odps_oss_config.bucket_name)
+  for sub_dir in ['configs']:
+    for root, dirs, files in os.walk(
+        os.path.join(odps_oss_config.temp_dir, sub_dir)):
+      for one_file in files:
+        file_path = os.path.join(root, one_file)
+        obj_path = file_path.split(sub_dir + '/')[1]
+        dst_path = os.path.join(odps_oss_config.exp_dir, sub_dir, obj_path)
+        test_bucket.put_object_from_file(dst_path, file_path)
+        logging.info('put %s to oss://%s/%s' %
+                     (file_path, odps_oss_config.bucket_name, dst_path))
+
+
+def prepare(odps_oss_config):
+  logging.info('temp_dir = %s' % odps_oss_config.temp_dir)
+  ali_bucket = get_oss_bucket(odps_oss_config.ali_oss_key,
+                              odps_oss_config.ali_oss_secret,
+                              odps_oss_config.ali_bucket_endpoint,
+                              odps_oss_config.ali_bucket_name)
+  shutil.copytree(odps_oss_config.script_path, odps_oss_config.temp_dir)
+  logging.info('start down data')
+  download_data(ali_bucket, odps_oss_config.temp_dir)
+  logging.info('down data completed')
+
+  # update params in config files
+  for root, dirs, files in os.walk(odps_oss_config.temp_dir):
+    for file in files:
+      file_path = os.path.join(root, file)
+      # drop .template
+      if file_path.endswith('.template'):
+        tmp_path = file_path[:-len('.template')]
+        os.rename(file_path, tmp_path)
+        file_path = tmp_path
+      if 'data' not in file_path:
+        logging.info('modify %s' % file_path)
+        change_files(odps_oss_config, file_path)
+
+  # put data to test bucket
+  put_data_to_bucket(odps_oss_config)
+
+  # mkdir log dir
+  if not os.path.exists(odps_oss_config.log_dir):
+    os.makedirs(odps_oss_config.log_dir)
+
+
+if __name__ == '__main__':
+  if len(sys.argv) < 5:
+    print('usage: %s ossutilconfig bucket_name rolearn odpsconfig' %
+          sys.argv[0])
+    sys.exit(1)
+
+  odps_oss_config = OdpsOSSConfig()
+  odps_oss_config.load_oss_config(sys.argv[1])
+  odps_oss_config.bucket_name = sys.argv[2]
+  odps_oss_config.arn = sys.argv[3]
+  odps_oss_config.load_odps_config(sys.argv[4])
+  prepare(odps_oss_config)