easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/feature_column/feature_group.py

@@ -0,0 +1,89 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import re
+
+from easy_rec.python.protos.feature_config_pb2 import FeatureGroupConfig
+from easy_rec.python.protos.feature_config_pb2 import WideOrDeep
+
+
+class FeatureGroup(object):
+
+  def __init__(self, feature_group_config):
+    self._config = feature_group_config
+    assert isinstance(self._config, FeatureGroupConfig)
+    assert self._config.wide_deep in [WideOrDeep.WIDE, WideOrDeep.DEEP]
+    self._auto_expand_feature_name()
+
+  @property
+  def group_name(self):
+    return self._config.group_name
+
+  @property
+  def wide_and_deep_dict(self):
+    wide_and_deep_dict = {}
+    for feature_name in self._config.feature_names:
+      wide_and_deep_dict[feature_name] = self._config.wide_deep
+    return wide_and_deep_dict
+
+  @property
+  def feature_names(self):
+    return self._config.feature_names
+
+  def select_columns(self, fc):
+    if self._config.wide_deep == WideOrDeep.WIDE:
+      wide_columns = [fc.wide_columns[x] for x in self._config.feature_names]
+      return wide_columns, []
+    else:
+      sequence_columns = []
+      deep_columns = []
+      for x in self._config.feature_names:
+        if x in fc.sequence_columns:
+          sequence_columns.append(fc.sequence_columns[x])
+        else:
+          deep_columns.append(fc.deep_columns[x])
+      return deep_columns, sequence_columns
+
+  # def _auto_expand_feature_name(self):
+  #   features = [x for x in self._config.feature_names]
+  #   while len(self._config.feature_names) > 0:
+  #     self._config.feature_names.pop()
+  #   for feature in features:
+  #     match_obj = re.match(r'([a-zA-Z_]+)\[([0-9]+)-([0-9]+)\]', feature)
+  #     if match_obj:
+  #       prefix = match_obj.group(1)
+  #       sid = int(match_obj.group(2))
+  #       eid = int(match_obj.group(3)) + 1
+  #       for tid in range(sid, eid):
+  #         tmp_f = '%s%d' % (prefix, tid)
+  #         self._config.feature_names.append(tmp_f)
+  #     else:
+  #       self._config.feature_names.append(feature)
+
+  def _auto_expand_feature_name(self):
+    features = [x for x in self._config.feature_names]
+    while len(self._config.feature_names) > 0:
+      self._config.feature_names.pop()
+    for feature in features:
+      flag = 1
+      if feature.endswith(']'):
+        match_obj = re.match(r'([a-zA-Z_]+)\[([0-9]+)-([0-9]+)\]', feature)
+      else:
+        flag = 2
+        match_obj = re.match(r'([a-zA-Z_]+)\[([0-9]+)-([0-9]+)\]([a-zA-Z_]+)',
+                             feature)
+      if match_obj:
+        prefix = match_obj.group(1)
+        sid = int(match_obj.group(2))
+        eid = int(match_obj.group(3)) + 1
+        if flag == 2:
+          endfix = match_obj.group(4)
+        else:
+          endfix = ''
+        for tid in range(sid, eid):
+          if flag == 2:
+            tmp_f = '%s%d%s' % (prefix, tid, endfix)
+          else:
+            tmp_f = '%s%d' % (prefix, tid)
+          self._config.feature_names.append(tmp_f)
+      else:
+        self._config.feature_names.append(feature)

easy_rec/python/hpo/emr_hpo.py

@@ -0,0 +1,140 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""Hyperparameter search for easy_rec on emr."""
+import argparse
+import json
+import logging
+import os
+import shutil
+import time
+
+from pai.automl.hpo.autotuner import AutoTuner
+
+from easy_rec.python.utils import hpo_util
+
+file_dir, _ = os.path.split(os.path.abspath(__file__))
+logging.basicConfig(
+    level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s')
+
+
+def hpo_config(config_path, hyperparams, exp_dir, metric_name,
+               el_submit_params):
+  earlystop = {
+      'type': 'large_is_better',
+      'threshold': 0.99,
+      'max_runtime': 2400
+  }
+  algorithm = {
+      'type': 'gp',
+      'initial_trials_num': 4,
+      'stop_when_exception': True
+  }
+
+  tmp_dir = '/tmp/emr_easy_rec_hpo_%d' % time.time()
+  os.makedirs(tmp_dir)
+  logging.info('local temporary path: %s' % tmp_dir)
+
+  param_path = tmp_dir + '/rewrite_{{ trial.id }}.json'
+  param_path_file = 'rewrite_{{ trial.id }}.json'
+  model_path = '%s/trail_{{ trial.id }}' % exp_dir
+  metric_path = os.path.join(model_path, 'res.metric')
+
+  pre_task = {
+      'type': 'BashTask',
+      'cmd': ['hadoop', 'fs', '-mkdir', '-p', model_path]
+  }
+  adapter_task = {
+      'type': 'localadaptertask',
+      # hpo_param_path for easy_rec
+      'param_file': param_path,
+  }
+  el_params = [
+      x.strip() for x in el_submit_params.split(' ') if x.strip() != ''
+  ]
+  assert len(
+      el_params) % 2 == 0, 'invalid number of el_submit params: %d[%s]' % (
+          len(el_params), str(el_params))
+  for i in range(0, len(el_params), 2):
+    assert el_params[i] in [
+        '-t', '-m', '-pn', '-pc', '-pg', '-pm', '-wn', '-wc', '-wm', '-wg'
+    ]
+  cmd = ['el_submit'] + el_params + [
+      '-a', 'easy_rec_hpo', '-m', 'local', '-f', '{},train_eval.py,{}'.format(
+          config_path, param_path), '--interact', 'INTERACT', '-c',
+      'python -m easy_rec.python.train_eval --hpo_metric_save_path {} '
+      '--hpo_param_path {} --pipeline_config_path {} --model_dir {}'.format(
+          metric_path, param_path_file, config_path, model_path)
+  ]
+
+  train_task = {
+      'type': 'BashTask',
+      'cmd': cmd,
+      'metric_reader': {
+          'type': 'hdfs_reader',
+          'location': metric_path,
+          'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name
+      }
+  }
+
+  tasks = [pre_task, adapter_task, train_task]
+  data = {
+      'earlystop': earlystop,
+      'algorithm': algorithm,
+      'hyperparams': hyperparams,
+      'tasks': tasks
+  }
+  return data, tmp_dir
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--hyperparams', type=str, help='hyper parameters', default=None)
+  parser.add_argument(
+      '--config_path', type=str, help='pipeline config', default=None)
+  parser.add_argument(
+      '--exp_dir', type=str, help='hpo experiment directory', default=None)
+  parser.add_argument(
+      '--el_submit_params',
+      type=str,
+      help='el_submit parameters(-t x -m x [-pn x -pc x -pm x] -wn x -wc x -wm x -wg x)',
+      default='-t standalone -m local -wn 1 -wc 6 -wm 20000 -wg 1')
+  parser.add_argument(
+      '--metric_name', type=str, help='metric_name', default='auc')
+  parser.add_argument(
+      '--max_parallel',
+      type=int,
+      help='max number of trials run at the same time',
+      default=4)
+  parser.add_argument(
+      '--total_trial_num',
+      type=int,
+      help='total number of trials will run',
+      default=6)
+  parser.add_argument(
+      '--debug',
+      action='store_true',
+      help='debug mode, will keep the temporary folder')
+
+  args = parser.parse_args()
+
+  assert args.hyperparams is not None
+  assert args.config_path is not None
+  assert args.exp_dir is not None
+
+  with open(args.hyperparams, 'r') as fin:
+    hyperparams = json.load(fin)
+
+  data, tmp_dir = hpo_config(args.config_path, hyperparams, args.exp_dir,
+                             args.metric_name, args.el_submit_params)
+
+  hpo_util.kill_old_proc(tmp_dir, platform='emr')
+
+  tuner = AutoTuner.create_tuner(
+      data, max_parallel=args.max_parallel, max_trial_num=args.total_trial_num)
+  tuner.fit(synchronize=True)
+
+  if not args.debug:
+    shutil.rmtree(tmp_dir)
+  else:
+    logging.info('temporary directory is: %s' % tmp_dir)

easy_rec/python/hpo/generate_hpo_sql.py

@@ -0,0 +1,71 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""Called by pai_hpo.py."""
+
+if __name__ == '__main__':
+  import argparse
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--sql_path', type=str, help='output sql path', default=None)
+  parser.add_argument(
+      '--config_path', type=str, help='config path', default=None)
+  parser.add_argument(
+      '--tables', type=str, help='train_table and test_table', default=None)
+  parser.add_argument(
+      '--train_tables', type=str, help='train_tables', default=None)
+  parser.add_argument(
+      '--eval_tables', type=str, help='eval_tables', default=None)
+  parser.add_argument(
+      '--cluster',
+      type=str,
+      help='specify tensorflow train jobs cluster parameter',
+      default=None)
+  parser.add_argument('--bucket', type=str, help='oss bucket', default=None)
+  parser.add_argument(
+      '--hpo_param_path', type=str, help='hpo param path', default=None)
+  parser.add_argument(
+      '--hpo_metric_save_path',
+      type=str,
+      help='hpo metric save path',
+      default=None)
+  parser.add_argument('--model_dir', type=str, help='model_dir', default=None)
+  parser.add_argument('--oss_host', type=str, help='oss endpoint', default=None)
+  parser.add_argument('--role_arn', type=str, help='role arn', default=None)
+  parser.add_argument(
+      '--algo_proj_name',
+      type=str,
+      help='algorithm project name',
+      default='algo_public')
+  parser.add_argument(
+      '--algo_res_proj', type=str, help='algo resource project', default=None)
+  parser.add_argument(
+      '--algo_version', type=str, help='algo version', default=None)
+
+  args = parser.parse_args()
+
+  with open(args.sql_path, 'w') as fout:
+    fout.write('pai -name easy_rec_ext -project %s\n' % args.algo_proj_name)
+    if args.algo_res_proj:
+      fout.write('  -Dres_project=%s\n' % args.algo_res_proj)
+    else:
+      fout.write('  -Dres_project=%s\n' % args.algo_proj_name)
+    if args.algo_version:
+      fout.write('  -Dversion=%s\n' % args.algo_version)
+    fout.write('  -Dconfig=%s\n' % args.config_path)
+    fout.write('  -Dcmd=train\n')
+    if args.tables:
+      fout.write('  -Dtables=%s\n' % args.tables)
+    else:
+      fout.write('  -Dtrain_tables=%s\n' % args.train_tables)
+      fout.write('  -Deval_tables=%s\n' % args.eval_tables)
+    fout.write('  -Dcluster=\'%s\'\n' % args.cluster)
+    fout.write('  -Darn=%s\n' % args.role_arn)
+    fout.write('  -Dbuckets=%s\n' % args.bucket)
+    fout.write('  -Dhpo_param_path=%s\n' % args.hpo_param_path)
+    fout.write('  -Dhpo_metric_save_path=%s\n' % args.hpo_metric_save_path)
+    fout.write('  -Dmodel_dir=%s\n' % args.model_dir)
+    fout.write('  -DossHost=%s\n' % args.oss_host)
+    fout.write('  -Deval_method=separate;\n')
+
+  print('write to %s' % args.sql_path)

easy_rec/python/hpo/pai_hpo.py

@@ -0,0 +1,297 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""Hyperparameter search demo for easy_rec on pai."""
+import json
+import logging
+import os
+import shutil
+import time
+
+from pai.automl import hpo
+
+from easy_rec.python.utils import hpo_util
+
+file_dir, _ = os.path.split(os.path.abspath(__file__))
+logging.basicConfig(
+    level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s')
+
+try:
+  import subprocess
+
+  subprocess.check_output('which odpscmd', shell=True)
+except Exception:
+  logging.error(
+      'odpscmd is not in path, please install from https://help.aliyun.com/document_detail/27971.html'
+  )
+
+
+def get_tuner(data, max_parallel, max_trial_num):
+  param_dict = json.loads(data)
+  if 'environment' in param_dict.keys():
+    hpo.register_env(**param_dict['environment'])
+
+  # hyper param
+  params = []
+  for h in param_dict['hyperparams']:
+    param = hpo.hyperparam.create(**h)
+    params.append(param)
+
+  # tasks
+  tasks = []
+  for t in param_dict['tasks']:
+    r = None
+    if 'metric_reader' in t.keys():
+      r = hpo.reader.create(**t['metric_reader'])
+      t.pop('metric_reader')
+    if r:
+      subtask = hpo.task.create(metric_reader=r, **t)
+    else:
+      subtask = hpo.task.create(**t)
+    tasks.append(subtask)
+
+  # earlystop & algo
+  early_stop = None
+  if 'earlystop' in param_dict.keys():
+    early_stop = hpo.earlystop.create(**param_dict['earlystop'])
+
+  algo = None
+  if 'algorithm' in param_dict.keys():
+    algo = hpo.algorithm.create(**param_dict['algorithm'])
+
+  tuner = hpo.autotuner.AutoTuner(
+      earlystop=early_stop,
+      algorithm=algo,
+      hyperparams=params,
+      task_list=tasks,
+      max_parallel=max_parallel,
+      max_trial_num=max_trial_num,
+      mode='local',
+      user_id='your_cloud_id')
+  return tuner
+
+
+def hpo_config(config_path, hyperparams, environment, exp_dir, tables,
+               train_tables, eval_tables, cluster, algo_proj_name,
+               algo_res_proj, algo_version, metric_name, odps_config_path):
+  earlystop = {'type': 'large_is_better', 'max_runtime': 3600 * 12}
+  algorithm = {
+      'type': 'gp',
+      'initial_trials_num': 4,
+      'stop_when_exception': True
+  }
+
+  if exp_dir.startswith('oss://'):
+    exp_dir = exp_dir.replace('oss://', '')
+    exp_dir = exp_dir[exp_dir.find('/') + 1:]
+
+  param_path = '%s/hpo_test_{{ trial.id }}.json' % exp_dir
+  metric_path = '%s/easy_rec_hpo_{{ trial.id }}.metric' % exp_dir
+  model_path = '%s/easy_rec_hpo_{{ trial.id }}' % exp_dir
+  bucket = 'oss://' + environment['bucket'].strip('/') + '/'
+
+  adapter_task = {
+      'type': 'ossadaptertask',
+      # hpo_param_path for easy_rec
+      'param_file': param_path,
+  }
+
+  tmp_dir = '/tmp/pai_easy_rec_hpo_%d' % time.time()
+  os.makedirs(tmp_dir)
+  logging.info('local temporary path: %s' % tmp_dir)
+
+  def _add_prefix(table_name):
+    table_name = table_name.strip()
+    if not table_name.startswith('odps://'):
+      return 'odps://%s/tables/%s' % (environment['project'], table_name)
+    else:
+      return table_name
+
+  if tables:
+    tables = [_add_prefix(x) for x in tables.split(',') if x != '']
+    tables = ','.join(tables)
+    logging.info('will tune on data: %s' % tables)
+  else:
+    train_tables = [_add_prefix(x) for x in train_tables.split(',') if x != '']
+    train_tables = ','.join(train_tables)
+    eval_tables = [_add_prefix(x) for x in eval_tables.split(',') if x != '']
+    eval_tables = ','.join(eval_tables)
+
+  sql_path = '%s/train_ext_hpo_{{ trial.id }}.sql' % tmp_dir
+  cmd_args = [
+      'python', '-m', 'easy_rec.python.hpo.generate_hpo_sql', '--sql_path',
+      sql_path, '--config_path', config_path, '--cluster', cluster, '--bucket',
+      bucket, '--hpo_param_path',
+      os.path.join(bucket, param_path), '--hpo_metric_save_path',
+      os.path.join(bucket, metric_path), '--model_dir',
+      os.path.join(bucket,
+                   model_path), '--oss_host', environment['oss_endpoint'],
+      '--role_arn', environment['role_arn'], '--algo_proj_name', algo_proj_name
+  ]
+
+  if tables:
+    cmd_args.extend(['--tables', tables])
+  if train_tables and eval_tables:
+    cmd_args.extend(
+        ['--train_tables', train_tables, '--eval_tables', eval_tables])
+
+  if algo_res_proj:
+    cmd_args.extend(['--algo_res_proj', algo_res_proj])
+  if algo_version:
+    cmd_args.extend(['--algo_version', algo_version])
+  prepare_sql_task = {'type': 'BashTask', 'cmd': cmd_args}
+
+  train_task = {
+      'type': 'BashTask',
+      'cmd': ['odpscmd',
+              '--config=%s' % odps_config_path, '-f', sql_path],
+      'metric_reader': {
+          'type': 'oss_reader',
+          'location': metric_path,
+          'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name
+      }
+  }
+
+  tasks = [adapter_task, prepare_sql_task, train_task]
+  data = {
+      'earlystop': earlystop,
+      'algorithm': algorithm,
+      'hyperparams': hyperparams,
+      'tasks': tasks,
+      'environment': environment
+  }
+  return data, tmp_dir
+
+
+if __name__ == '__main__':
+  import argparse
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--odps_config', type=str, help='odps_config.ini', default=None)
+  parser.add_argument(
+      '--oss_config', type=str, help='excel config path', default='')
+  parser.add_argument('--bucket', type=str, help='bucket name', default=None)
+  parser.add_argument('--role_arn', type=str, help='role arn', default=None)
+  parser.add_argument(
+      '--hyperparams', type=str, help='hyper parameters', default=None)
+  parser.add_argument(
+      '--config_path', type=str, help='pipeline config', default=None)
+  parser.add_argument(
+      '--tables', type=str, help='train table and test table', default=None)
+  parser.add_argument(
+      '--train_tables', type=str, help='train tables', default=None)
+  parser.add_argument(
+      '--eval_tables', type=str, help='eval tables', default=None)
+  parser.add_argument(
+      '--exp_dir', type=str, help='hpo experiment directory', default=None)
+  parser.add_argument(
+      '--cluster',
+      type=str,
+      help='cluster spec',
+      default='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}'
+  )
+  parser.add_argument(
+      '--algo_proj_name',
+      type=str,
+      help='algo project name',
+      default='algo_public')
+  parser.add_argument(
+      '--algo_version', type=str, help='algo version', default=None)
+  parser.add_argument(
+      '--algo_res_proj', type=str, help='algo resource project', default=None)
+  parser.add_argument(
+      '--metric_name', type=str, help='evaluate metric name', default='auc')
+  parser.add_argument(
+      '--max_parallel',
+      type=int,
+      help='max number of trials run at the same time',
+      default=4)
+  parser.add_argument(
+      '--total_trial_num',
+      type=int,
+      help='total number of trials will run',
+      default=6)
+  parser.add_argument(
+      '--debug',
+      action='store_true',
+      help='debug mode, will keep the temporary folder')
+
+  args = parser.parse_args()
+
+  assert os.path.exists(args.odps_config)
+  odps_config = {}
+  with open(args.odps_config, 'r') as fin:
+    for line_str in fin:
+      line_str = line_str.strip()
+      if len(line_str) == 0:
+        continue
+      if line_str[0] == '#':
+        continue
+      if '=' in line_str:
+        tmp_id = line_str.find('=')
+        key = line_str[:tmp_id].strip()
+        val = line_str[(tmp_id + 1):].strip()
+        odps_config[key] = val
+
+  if args.oss_config is None:
+    args.oss_config = os.path.join(os.environ['HOME'], '.ossutilconfig')
+  assert os.path.exists(args.oss_config)
+  oss_config = {}
+  with open(args.oss_config, 'r') as fin:
+    for line_str in fin:
+      line_str = line_str.strip()
+      if len(line_str) == 0:
+        continue
+      if line_str[0] == '#':
+        continue
+      if '=' in line_str:
+        tmp_id = line_str.find('=')
+        key = line_str[:tmp_id].strip()
+        val = line_str[(tmp_id + 1):].strip()
+        oss_config[key] = val
+
+  assert args.bucket is not None
+  assert args.role_arn is not None
+
+  if args.bucket.startswith('oss://'):
+    args.bucket = args.bucket[len('oss://'):]
+  args.bucket = args.bucket.strip('/')
+
+  environment = {
+      'access_id': odps_config['access_id'],
+      'access_key': odps_config['access_key'],
+      'oss_access_id': oss_config['accessKeyID'],
+      'oss_access_key': oss_config['accessKeySecret'],
+      'project': odps_config['project_name'],
+      'odps_endpoint': odps_config['end_point'],
+      'biz_id': '147331^paistudio^xxxxxxx^2020-03-18',
+      'role_arn': args.role_arn,
+      'bucket': args.bucket,
+      'oss_endpoint': oss_config['endpoint']
+  }
+
+  assert args.hyperparams is not None
+  with open(args.hyperparams, 'r') as fin:
+    hyperparams = json.load(fin)
+
+  assert args.config_path is not None
+  assert args.exp_dir is not None
+  assert args.tables is not None or (args.train_tables is not None and
+                                     args.eval_tables is not None)
+
+  data, tmp_dir = hpo_config(args.config_path, hyperparams, environment,
+                             args.exp_dir, args.tables, args.train_tables,
+                             args.eval_tables, args.cluster,
+                             args.algo_proj_name, args.algo_res_proj,
+                             args.algo_version, args.metric_name,
+                             args.odps_config)
+  hpo_util.kill_old_proc(tmp_dir, platform='pai')
+
+  data_json = json.dumps(data)
+  tuner = get_tuner(data_json, args.max_parallel, args.total_trial_num)
+  tuner.fit(synchronize=True)
+
+  if not args.debug:
+    shutil.rmtree(tmp_dir)
+  else:
+    logging.info('temporary directory is: %s' % tmp_dir)