easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/utils/hpo_util.py

@@ -0,0 +1,137 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import json
+import logging
+import os
+
+import psutil
+import tensorflow as tf
+from tensorflow.python.summary import summary_iterator
+
+if tf.__version__ >= '2.0':
+  gfile = tf.compat.v1.gfile
+else:
+  gfile = tf.gfile
+
+
+def get_all_eval_result(event_file_pattern):
+  """Get the best eval result from event files.
+
+  Args:
+    event_files: Absolute pattern of event files.
+
+  Returns:
+    The best eval result.
+  """
+  all_eval_result = []
+  for event_file in gfile.Glob(os.path.join(event_file_pattern)):
+    for event in summary_iterator.summary_iterator(event_file):
+      if event.HasField('summary'):
+        event_eval_result = {}
+        event_eval_result['global_step'] = event.step
+        for value in event.summary.value:
+          if value.HasField('simple_value'):
+            event_eval_result[value.tag] = value.simple_value
+        if len(event_eval_result) >= 2:
+          all_eval_result.append(event_eval_result)
+  return all_eval_result
+
+
+def save_eval_metrics(model_dir, metric_save_path, has_evaluator=True):
+  """Save evaluation metrics.
+
+  Args:
+    model_dir: train model directory
+    metric_save_path: metric saving path
+    has_evaluator: evaluation is done on a separate evaluator, not on master.
+  """
+
+  def _get_eval_event_file_pattern():
+    eval_dir = os.path.join(model_dir, 'eval_val/')
+    if not gfile.Exists(eval_dir):
+      eval_dir = os.path.join(model_dir, 'eval/')
+      assert gfile.Exists(eval_dir), 'eval_val or eval does exists'
+    event_file_pattern = os.path.join(eval_dir, '*.tfevents.*')
+    logging.info('event_file_pattern: %s' % event_file_pattern)
+    return event_file_pattern
+
+  all_eval_res = []
+  if 'TF_CONFIG' in os.environ:
+    # check whether evaluator exists
+    tf_config = json.loads(os.environ['TF_CONFIG'])
+    logging.info('tf_config = %s' % json.dumps(tf_config))
+    logging.info('model_dir = %s' % model_dir)
+    if has_evaluator:
+      if tf_config['task']['type'] == 'evaluator':
+        event_file_pattern = _get_eval_event_file_pattern()
+        all_eval_res = get_all_eval_result(event_file_pattern)
+    elif 'master' in tf_config['cluster'] or 'chief' in tf_config['cluster']:
+      if tf_config['task']['type'] in ['master', 'chief']:
+        event_file_pattern = _get_eval_event_file_pattern()
+        all_eval_res = get_all_eval_result(event_file_pattern)
+    else:
+      assert False, 'invalid cluster config, could not find master or chief or evaluator'
+  else:
+    # standalone mode
+    event_file_pattern = _get_eval_event_file_pattern()
+    all_eval_res = get_all_eval_result(event_file_pattern)
+
+  logging.info('all_eval_res num = %d' % len(all_eval_res))
+  if len(all_eval_res) > 0:
+    with gfile.GFile(metric_save_path, 'w') as fout:
+      for eval_res in all_eval_res:
+        fout.write(json.dumps(eval_res) + '\n')
+    logging.info('save all evaluation result to %s' % metric_save_path)
+
+
+def kill_old_proc(tmp_dir, platform='pai'):
+  curr_pid = os.getpid()
+  if platform == 'pai':
+    for p in psutil.process_iter():
+      try:
+        cmd = ' '.join(p.cmdline())
+        if 'easy_rec.python.hpo.pai_hpo' in cmd and 'python' in cmd:
+          if p.pid != curr_pid:
+            logging.info('will kill: [%d] %s' % (p.pid, cmd))
+            p.terminate()
+        if 'client/experiment_main.py' in cmd and 'python' in cmd:
+          if p.pid != curr_pid:
+            logging.info('will kill: [%d] %s' % (p.pid, cmd))
+            p.terminate()
+      except Exception:
+        pass
+  else:
+    for p in psutil.process_iter():
+      try:
+        cmd = ' '.join(p.cmdline())
+        if 'easy_rec.python.hpo.emr_hpo' in cmd and 'python' in cmd:
+          if p.pid != curr_pid:
+            logging.info('will kill: [%d] %s' % (p.pid, cmd))
+            p.terminate()
+        if 'client/experiment_main.py' in cmd and 'python' in cmd:
+          if p.pid != curr_pid:
+            logging.info('will kill: [%d] %s' % (p.pid, cmd))
+            p.terminate()
+        if 'el_submit' in cmd and 'easy_rec_hpo' in cmd:
+          if p.pid != curr_pid:
+            logging.info('will kill: [%d] %s' % (p.pid, cmd))
+            p.terminate()
+      except Exception:
+        pass
+
+  if platform == 'emr':
+    # clear easy_rec_hpo yarn jobs
+    yarn_job_file = os.path.join(tmp_dir, 'yarn_job.txt')
+    os.system(
+        "yarn application -list | awk '{ if ($2 == \"easy_rec_hpo\") print $1 }' > %s"
+        % yarn_job_file)
+    yarn_job_arr = []
+    with open(yarn_job_file, 'r') as fin:
+      for line_str in fin:
+        line_str = line_str.strip()
+        yarn_job_arr.append(line_str)
+    yarn_job_arr = list(set(yarn_job_arr))
+    if len(yarn_job_arr) > 0:
+      logging.info('will kill the easy_rec_hpo yarn jobs: %s' %
+                   ','.join(yarn_job_arr))
+      os.system('yarn application -kill %s' % ' '.join(yarn_job_arr))

easy_rec/python/utils/hvd_utils.py

@@ -0,0 +1,56 @@
+# -*- encoding: utf-8 -*-
+import logging
+
+import tensorflow as tf
+from tensorflow.python.framework import ops
+from tensorflow.python.training import session_run_hook
+
+from easy_rec.python.utils import constant
+
+# from horovod.tensorflow.compression import Compression
+try:
+  from horovod.tensorflow.functions import broadcast_variables
+except Exception:
+  pass
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class BroadcastGlobalVariablesHook(session_run_hook.SessionRunHook):
+  """SessionRunHook that will broadcast all global variables from root rank to all other processes during initialization.
+
+  This is necessary to ensure consistent initialization of all workers when
+  training is started with random weights or restored from a checkpoint.
+  """  # noqa: E501
+
+  def __init__(self, root_rank, device=''):
+    """Construct a new BroadcastGlobalVariablesHook that will broadcast all global variables from root rank to all other processes during initialization.
+
+    Args:
+      root_rank:
+        Rank that will send data, other ranks will receive data.
+      device:
+        Device to be used for broadcasting. Uses GPU by default
+        if Horovod was built with HOROVOD_GPU_OPERATIONS.
+    """  # noqa: E501
+    super(BroadcastGlobalVariablesHook, self).__init__()
+    self.root_rank = root_rank
+    self.bcast_op = None
+    self.device = device
+
+  def begin(self):
+    bcast_vars = []
+    embed_para_vars = ops.get_collection(constant.EmbeddingParallel)
+    for x in tf.global_variables():
+      # if '/embedding' not in x.name and 'DynamicVariable' not in str(type(x)):
+      if x.name not in embed_para_vars:
+        bcast_vars.append(x)
+        logging.info('will broadcast variable: name=%s shape=%s' %
+                     (x.name, x.get_shape()))
+    if not self.bcast_op or self.bcast_op.graph != tf.get_default_graph():
+      with tf.device(self.device):
+        self.bcast_op = broadcast_variables(bcast_vars, self.root_rank)
+
+  def after_create_session(self, session, coord):
+    session.run(self.bcast_op)

easy_rec/python/utils/input_utils.py

@@ -0,0 +1,108 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+
+from easy_rec.python.protos.dataset_pb2 import DatasetConfig
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def get_type_defaults(field_type, default_val=''):
+  type_defaults = {
+      DatasetConfig.INT32: 0,
+      DatasetConfig.INT64: 0,
+      DatasetConfig.STRING: '',
+      DatasetConfig.BOOL: False,
+      DatasetConfig.FLOAT: 0.0,
+      DatasetConfig.DOUBLE: 0.0
+  }
+  assert field_type in type_defaults, 'invalid type: %s' % field_type
+  if default_val == '':
+    default_val = type_defaults[field_type]
+  if field_type == DatasetConfig.INT32:
+    return int(default_val)
+  elif field_type == DatasetConfig.INT64:
+    return np.int64(default_val)
+  elif field_type == DatasetConfig.STRING:
+    return default_val
+  elif field_type == DatasetConfig.BOOL:
+    return default_val.lower() == 'true'
+  elif field_type in [DatasetConfig.FLOAT]:
+    return float(default_val)
+  elif field_type in [DatasetConfig.DOUBLE]:
+    return np.float64(default_val)
+
+  return type_defaults[field_type]
+
+
+def string_to_number(field, ftype, default_value, name=''):
+  """Type conversion for parsing rtp fg input format.
+
+  Args:
+    field: field to be converted.
+    ftype: field dtype set in DatasetConfig.
+    default_value: default value for this field
+    name: field name for
+  Returns: A name for the operation (optional).
+  """
+  default_vals = tf.tile(tf.constant([str(default_value)]), tf.shape(field))
+  field = tf.where(tf.greater(tf.strings.length(field), 0), field, default_vals)
+
+  if ftype in [DatasetConfig.INT32, DatasetConfig.INT64]:
+    # Int type is not supported in fg.
+    # If you specify INT32, INT64 in DatasetConfig, you need to perform a cast at here.
+    tmp_field = tf.string_to_number(
+        field, tf.double, name='field_as_flt_%s' % name)
+    if ftype in [DatasetConfig.INT64]:
+      tmp_field = tf.cast(tmp_field, tf.int64)
+    else:
+      tmp_field = tf.cast(tmp_field, tf.int32)
+  elif ftype in [DatasetConfig.FLOAT]:
+    tmp_field = tf.string_to_number(
+        field, tf.float32, name='field_as_flt_%s' % name)
+  elif ftype in [DatasetConfig.DOUBLE]:
+    tmp_field = tf.string_to_number(
+        field, tf.float64, name='field_as_flt_%s' % name)
+  elif ftype in [DatasetConfig.BOOL]:
+    tmp_field = tf.logical_or(tf.equal(field, 'True'), tf.equal(field, 'true'))
+  elif ftype in [DatasetConfig.STRING]:
+    tmp_field = field
+  else:
+    assert False, 'invalid types: %s' % str(ftype)
+  return tmp_field
+
+
+def np_to_tf_type(np_type):
+  _types_map = {
+      int: tf.int32,
+      np.int32: tf.int32,
+      np.int64: tf.int64,
+      str: tf.string,
+      np.float: tf.float32,
+      np.float32: tf.float32,
+      float: tf.float32,
+      np.double: tf.float64
+  }
+  if np_type in _types_map:
+    return _types_map[np_type]
+  else:
+    return tf.string
+
+
+def get_tf_type_from_parquet_file(cols, parquet_file):
+  # gfile not supported, read_parquet requires random access
+  input_data = pd.read_parquet(parquet_file, columns=cols)
+  tf_types = []
+  for col in cols:
+    obj = input_data[col][0]
+    if isinstance(obj, list):
+      data_type = type(obj[0])
+    elif isinstance(obj, np.ndarray):
+      data_type = type(obj[0])
+    else:
+      data_type = type(obj)
+    tf_types.append(np_to_tf_type(data_type))
+  return tf_types

easy_rec/python/utils/io_util.py

@@ -0,0 +1,282 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""IO utils.
+
+isort:skip_file
+"""
+import logging
+from future import standard_library
+standard_library.install_aliases()
+
+import os
+import traceback
+from subprocess import getstatusoutput
+
+import six
+import tensorflow as tf
+from six.moves import http_client
+from six.moves import urllib
+import json
+if six.PY2:
+  from urllib import quote
+else:
+  from urllib.parse import quote
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+EASY_REC_RES_DIR = 'easy_rec_user_resources'
+HTTP_MAX_NUM_RETRY = 5
+HTTP_MAX_TIMEOUT = 600
+
+
+def http_read(url, timeout=HTTP_MAX_TIMEOUT, max_retry=HTTP_MAX_NUM_RETRY):
+  """Read data from url with maximum retry.
+
+  Args:
+    url: http url to be read
+    timeout: specifies a timeout in seconds for blocking operations.
+    max_retry: http max retry times.
+  """
+  num_read_try = 0
+  data = None
+  while num_read_try < max_retry:
+    try:
+      if six.PY2:
+        url = url.encode('utf-8')
+      url = quote(url, safe='%/:?=&')
+      data = urllib.request.urlopen(url, timeout=timeout).read()
+      break
+    except http_client.IncompleteRead:
+      tf.logging.warning('incomplete read exception, will retry: %s' % url)
+      num_read_try += 1
+    except Exception:
+      tf.logging.error(traceback.format_exc())
+      break
+
+  if data is None:
+    tf.logging.error('http read %s failed' % url)
+
+  return data
+
+
+def download(oss_or_url, dst_dir=''):
+  """Download file.
+
+  Args:
+    oss_or_url: http or oss path
+    dst_dir: destination directory
+  Return:
+    dst_file: local path for the downloaded file
+  """
+  _, basename = os.path.split(oss_or_url)
+  if oss_or_url[:3] == 'oss':
+    with tf.gfile.GFile(oss_or_url, 'rb') as infile:
+      file_content = infile.read()
+  elif oss_or_url[:4] == 'http':
+    try:
+      response = urllib.request.urlopen(oss_or_url, timeout=HTTP_MAX_TIMEOUT)
+      file_content = response.read()
+    except Exception as e:
+      raise RuntimeError('Download %s failed: %s\n %s' %
+                         (oss_or_url, str(e), traceback.format_exc()))
+  else:
+    tf.logging.warning('skip downloading %s, seems to be a local file' %
+                       oss_or_url)
+    return oss_or_url
+
+  if dst_dir != '' and not os.path.exists(dst_dir):
+    os.makedirs(dst_dir)
+  dst_file = os.path.join(dst_dir, basename)
+  with tf.gfile.GFile(dst_file, 'wb') as ofile:
+    ofile.write(file_content)
+
+  return dst_file
+
+
+def create_module_dir(dst_dir):
+  if not os.path.exists(dst_dir):
+    os.makedirs(dst_dir)
+    with open(os.path.join(dst_dir, '__init__.py'), 'w') as ofile:
+      ofile.write('\n')
+
+
+def download_resource(resource_path, dst_dir=EASY_REC_RES_DIR):
+  """Download user resource.
+
+  Args:
+    resource_path: http or oss path
+    dst_dir: destination directory
+  """
+  create_module_dir(dst_dir)
+  _, basename = os.path.split(resource_path)
+  if not basename.endswith('.py'):
+    raise ValueError('resource %s should be python file' % resource_path)
+
+  target = download(resource_path, dst_dir)
+
+  return target
+
+
+def download_and_uncompress_resource(resource_path, dst_dir=EASY_REC_RES_DIR):
+  """Download user resource and uncompress it if necessary.
+
+  Args:
+    resource_path: http or oss path
+    dst_dir: download destination directory
+  """
+  create_module_dir(dst_dir)
+
+  _, basename = os.path.split(resource_path)
+  if not basename.endswith('.tar.gz') and not basename.endswith('.zip') and \
+     not basename.endswith('.py'):
+    raise ValueError('resource %s should be tar.gz or zip or py' %
+                     resource_path)
+
+  download(resource_path, dst_dir)
+
+  stat = 0
+  if basename.endswith('tar.gz'):
+    stat, output = getstatusoutput('cd %s && tar -zxf %s' % (dst_dir, basename))
+  elif basename.endswith('zip'):
+    stat, output = getstatusoutput('cd %s && unzip %s' % (dst_dir, basename))
+
+  if stat != 0:
+    raise ValueError('uncompress resoruce %s failed: %s' % resource_path,
+                     output)
+
+  return dst_dir
+
+
+def oss_has_t_mode(target_file):
+  """Test if current enviroment support t-mode written to oss."""
+  if 'PAI' not in tf.__version__:
+    return False
+  # test if running on cluster
+  test_file = target_file + '.tmp'
+  try:
+    with tf.gfile.GFile(test_file, 't') as ofile:
+      ofile.write('a')
+      pass
+    tf.gfile.Remove(test_file)
+    return True
+  except:  # noqa: E722
+    return False
+
+
+def fix_oss_dir(path):
+  """Make sure that oss dir endswith /."""
+  if path.startswith('oss://') and not path.endswith('/'):
+    return path + '/'
+  return path
+
+
+def save_data_to_json_path(json_path, data):
+  with tf.gfile.GFile(json_path, 'w') as fout:
+    fout.write(json.dumps(data))
+  assert tf.gfile.Exists(json_path), 'in_save_data_to_json_path, save_failed'
+
+
+def read_data_from_json_path(json_path):
+  if json_path and tf.gfile.Exists(json_path):
+    with tf.gfile.GFile(json_path, 'r') as fin:
+      data = json.loads(fin.read())
+    return data
+  else:
+    logging.info('json_path not exists, return None')
+    return None
+
+
+def convert_tf_flags_to_argparse(flags):
+  """Convert tf.app.flags.FLAGS to argparse.ArgumentParser.
+
+  Args:
+      flags: tf.app.flags.FLAGS
+  Returns:
+      argparse.ArgumentParser: configurate ArgumentParser object
+  """
+  import argparse
+  import ast
+  parser = argparse.ArgumentParser()
+
+  args = {}
+  for flag in flags._flags().values():
+    flag_name = flag.name
+    if flag_name in args:
+      args[flag_name][0] = True
+      continue
+    default = flag.value
+    flag_type = type(default)
+    help_str = flag.help or ''
+    args[flag_name] = [
+        False, flag_type, default, help_str,
+        flag.choices if hasattr(flag, 'choices') else None
+    ]
+
+  def str2bool(v):
+    if isinstance(v, bool):
+      return v
+    if v.lower() in ('yes', 'true', 't', 'y', '1'):
+      return True
+    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+      return False
+    else:
+      raise argparse.ArgumentTypeError('Boolean value expected.')
+
+  for flag_name, (multi, flag_type, default, help_str, choices) in args.items():
+    if flag_type == bool:
+      parser.add_argument(
+          '--' + flag_name,
+          type=str2bool,
+          nargs='?',
+          const=True,
+          default=False,
+          help=help_str)
+    elif flag_type == str:
+      if choices:
+        parser.add_argument(
+            '--' + flag_name,
+            type=str,
+            choices=choices,
+            default=default,
+            help=help_str)
+      elif multi:
+        parser.add_argument(
+            '--' + flag_name,
+            type=str,
+            action='append',
+            default=default,
+            help=help_str)
+      else:
+        parser.add_argument(
+            '--' + flag_name, type=str, default=default, help=help_str)
+    elif flag_type in (list, dict):
+      parser.add_argument(
+          '--' + flag_name,
+          type=lambda s: ast.literal_eval(s),
+          default=default,
+          help=help_str)
+    elif flag_type in (int, float):
+      parser.add_argument(
+          '--' + flag_name, type=flag_type, default=default, help=help_str)
+    else:
+      parser.add_argument(
+          '--' + flag_name, type=str, default=default, help=help_str)
+  return parser
+
+
+def filter_unknown_args(flags, args):
+  """Filter unknown args."""
+  known_args = [args[0]]
+  parser = convert_tf_flags_to_argparse(flags)
+  args, unknown = parser.parse_known_args(args)
+  if len(unknown) > 1:
+    logging.info('undefined arguments: %s', ', '.join(unknown[1:]))
+  for key, value in vars(args).items():
+    if value is None:
+      continue
+    if type(value) in (list, dict) and not value:
+      continue
+    known_args.append('--' + key + '=' + str(value))
+  logging.info('defined arguments: %s', ', '.join(known_args[1:]))
+  return known_args