easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/utils/multi_optimizer.py

@@ -0,0 +1,62 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Date: 2018-09-13
+import tensorflow as tf
+from tensorflow.python.training import optimizer
+
+
+class MultiOptimizer(optimizer.Optimizer):
+
+  def __init__(self, opts, grouped_vars, use_locking=False):
+    """Combine multiple optimizers for optimization, such as WideAndDeep.
+
+    Args:
+      opts: list of optimizer instance.
+      grouped_vars: list of list of vars, each list of vars are
+          optimized by each of the optimizers.
+      use_locking: be compatible, currently not used.
+    """
+    super(MultiOptimizer, self).__init__(use_locking, 'MultiOptimizer')
+    self._opts = opts
+    self._grouped_vars = grouped_vars
+
+  def compute_gradients(self, loss, variables, **kwargs):
+    grad_and_vars = []
+    for gid, opt in enumerate(self._opts):
+      grad_and_vars.extend(
+          opt.compute_gradients(loss, self._grouped_vars[gid], **kwargs))
+    return grad_and_vars
+
+  def apply_gradients(self, grads_and_vars, global_step=None, name=None):
+    update_ops = []
+    grads_and_vars = [x for x in grads_and_vars]
+    for gid, opt in enumerate(self._opts):
+      tmp = [x for x in grads_and_vars if x[1] in self._grouped_vars[gid]]
+      if gid == 0:
+        update_ops.append(opt.apply_gradients(tmp, global_step))
+      else:
+        update_ops.append(opt.apply_gradients(tmp, None))
+    return tf.group(update_ops)
+
+  def open_auto_record(self, flag=True):
+    super(MultiOptimizer, self).open_auto_record(flag)
+
+  def get_slot(self, var, name):
+    raise NotImplementedError('not implemented')
+    # for opt in self._opts:
+    #   tmp = opt.get_slot(var, name)
+    #   if tmp is not None:
+    #     return tmp
+    # return None
+
+  def variables(self):
+    all_vars = []
+    for opt in self._opts:
+      all_vars.extend(opt.variables())
+    return all_vars
+
+  def get_slot_names(self):
+    slot_names = []
+    for opt in self._opts:
+      slot_names.extend(opt.get_slot_names())
+    return slot_names

easy_rec/python/utils/numpy_utils.py

@@ -0,0 +1,18 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import json
+
+import numpy as np
+
+
+class NumpyEncoder(json.JSONEncoder):
+  """For encode numpy arrays."""
+
+  def default(self, obj):
+    if isinstance(obj, np.integer):
+      return int(obj)
+    elif isinstance(obj, np.floating):
+      return float(obj)
+    elif isinstance(obj, np.ndarray):
+      return obj.tolist()
+    return json.JSONEncoder.default(self, obj)

easy_rec/python/utils/odps_util.py

@@ -0,0 +1,79 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""Common functions used for odps input."""
+from tensorflow.python.framework import dtypes
+
+from easy_rec.python.protos.dataset_pb2 import DatasetConfig
+
+
+def is_type_compatiable(odps_type, input_type):
+  """Check that odps_type are compatiable with input_type."""
+  type_map = {
+      'bigint': DatasetConfig.INT64,
+      'string': DatasetConfig.STRING,
+      'double': DatasetConfig.DOUBLE
+  }
+  tmp_type = type_map[odps_type]
+  if tmp_type == input_type:
+    return True
+  else:
+    float_types = [DatasetConfig.FLOAT, DatasetConfig.DOUBLE]
+    int_types = [DatasetConfig.INT32, DatasetConfig.INT64]
+    if tmp_type in float_types and input_type in float_types:
+      return True
+    elif tmp_type in int_types and input_type in int_types:
+      return True
+    else:
+      return False
+
+
+def odps_type_to_input_type(odps_type):
+  """Check that odps_type are compatiable with input_type."""
+  odps_type_map = {
+      'bigint': DatasetConfig.INT64,
+      'string': DatasetConfig.STRING,
+      'double': DatasetConfig.DOUBLE
+  }
+  assert odps_type in odps_type_map, 'only support [bigint, string, double]'
+  input_type = odps_type_map[odps_type]
+  return input_type
+
+
+def check_input_field_and_types(data_config):
+  """Check compatibility of input in data_config.
+
+  check that data_config.input_fields are compatible with
+  data_config.selected_cols and data_config.selected_types.
+
+  Args:
+    data_config: instance of DatasetConfig
+  """
+  input_fields = [x.input_name for x in data_config.input_fields]
+  input_field_types = [x.input_type for x in data_config.input_fields]
+  selected_cols = data_config.selected_cols if data_config.selected_cols else None
+  selected_col_types = data_config.selected_col_types if data_config.selected_col_types else None
+  if not selected_cols:
+    return
+
+  selected_cols = selected_cols.split(',')
+  for x in input_fields:
+    assert x in selected_cols, 'column %s is not in table' % x
+  if selected_col_types:
+    selected_types = selected_col_types.split(',')
+    type_map = {x: y for x, y in zip(selected_cols, selected_types)}
+    for x, y in zip(input_fields, input_field_types):
+      tmp_type = type_map[x]
+      assert is_type_compatiable(tmp_type, y), \
+          'feature[%s] type error: odps %s is not compatible with input_type %s' % (
+              x, tmp_type, DatasetConfig.FieldType.Name(y))
+
+
+def odps_type_2_tf_type(odps_type):
+  if odps_type == 'string':
+    return dtypes.string
+  elif odps_type == 'bigint':
+    return dtypes.int64
+  elif odps_type in ['double', 'float']:
+    return dtypes.float32
+  else:
+    return dtypes.string

easy_rec/python/utils/pai_util.py

@@ -0,0 +1,86 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import logging
+import os
+import sys
+import traceback
+
+import tensorflow as tf
+
+if sys.version_info.major == 2:
+  from urllib2 import urlopen, Request, HTTPError
+else:
+  from urllib.request import urlopen, Request
+  from urllib.error import HTTPError
+
+
+def is_on_pai():
+  # IS_ON_PAI is set in pai_jobs/run.py
+  # which is the entry on pai platform
+  return 'IS_ON_PAI' in os.environ
+
+
+def set_on_pai():
+  logging.info('set on pai environment variable: IS_ON_PAI')
+  os.environ['IS_ON_PAI'] = '1'
+
+
+def download(url):
+  _, fname = os.path.split(url)
+  request = Request(url=url)
+  try:
+    response = urlopen(request, timeout=10)
+    with open(fname, 'w') as ofile:
+      ofile.write(response.read())
+    return fname
+  except HTTPError as e:
+    tf.logging.error('http error: ', e.code)
+    tf.logging.error('body:', e.read())
+    return None
+  except Exception as e:
+    tf.logging.error(e)
+    tf.logging.error(traceback.format_exc())
+    return None
+
+
+def process_config(configs, task_index=0, worker_num=1):
+  """Download config and select config for the worker.
+
+  Args:
+    configs: config paths, separated by ','
+    task_index: worker index
+    worker_num: total number of workers
+  """
+  configs = configs.split(',')
+  if len(configs) > 1:
+    assert len(configs) == worker_num, \
+        'number of configs must be equal to number of workers,' + \
+        ' when number of configs > 1'
+    config = configs[task_index]
+  else:
+    config = configs[0]
+
+  if config[:4] == 'http':
+    return download(config)
+  elif config[:3] == 'oss':
+    if '/##/' in config:
+      config = config.replace('/##/', '\x02')
+    if '/#/' in config:
+      config = config.replace('/#/', '\x01')
+    return config
+  else:
+    # allow to use this entry file to run experiments from local env
+    # to avoid uploading sample file
+    return config
+
+
+def test():
+  f = download(
+      'https://easy-rec.oss-cn-hangzhou.aliyuncs.com/config/MultiTower/dwd_avazu_ctr_deepmodel.config'
+  )
+  assert f == 'dwd_avazu_ctr_deepmodel.config'
+
+
+if __name__ == '__main__':
+  test()

easy_rec/python/utils/proto_util.py

@@ -0,0 +1,90 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+
+def copy_obj(proto_obj):
+  """Make a copy of proto_obj so that later modifications of tmp_obj will have no impact on proto_obj.
+
+  Args:
+    proto_obj: a protobuf message
+  Return:
+    a copy of proto_obj
+  """
+  tmp_obj = type(proto_obj)()
+  tmp_obj.CopyFrom(proto_obj)
+  return tmp_obj
+
+
+def get_norm_embed_name(name, verbose=False):
+  """For embedding export to redis.
+
+  Args:
+    name: variable name
+    verbose: whether to dump the embed_names
+  Return:
+    embedding_name: normalized embedding_name
+    embedding_part_id: normalized embedding part_id
+    if embedding_weights not in name, return None, None
+  """
+  name_toks = name.split('/')
+  for i in range(0, len(name_toks) - 1):
+    if name_toks[i + 1].startswith('embedding_weights:'):
+      var_id = name_toks[i + 1].replace('embedding_weights:', '')
+      tmp_name = '/'.join(name_toks[:i + 1])
+      if var_id != '0':
+        tmp_name = tmp_name + '_' + var_id
+      if verbose:
+        logging.info('norm %s to %s' % (name, tmp_name))
+      return tmp_name, 0
+    if i > 1 and name_toks[i + 1].startswith('part_') and \
+       name_toks[i] == 'embedding_weights':
+      tmp_name = '/'.join(name_toks[:i])
+      part_id = name_toks[i + 1].replace('part_', '')
+      part_toks = part_id.split(':')
+      if len(part_toks) >= 2 and part_toks[1] != '0':
+        tmp_name = tmp_name + '_' + part_toks[1]
+      if verbose:
+        logging.info('norm %s to %s' % (name, tmp_name))
+      return tmp_name, int(part_toks[0])
+
+  # input_layer/app_category_embedding/app_category_embedding_weights/SparseReshape
+  # => input_layer/app_category_embedding
+  for i in range(0, len(name_toks) - 1):
+    if name_toks[i + 1].endswith('_embedding_weights') or \
+       '_embedding_weights_' in name_toks[i + 1]:
+      tmp_name = '/'.join(name_toks[:i + 1])
+      if verbose:
+        logging.info('norm %s to %s' % (name, tmp_name))
+      return tmp_name, 0
+  # input_layer/app_category_embedding/embedding_weights
+  # => input_layer/app_category_embedding
+  for i in range(0, len(name_toks) - 1):
+    if name_toks[i + 1] == 'embedding_weights':
+      tmp_name = '/'.join(name_toks[:i + 1])
+      if verbose:
+        logging.info('norm %s to %s' % (name, tmp_name))
+      return tmp_name, 0
+  logging.warning('Failed to norm: %s' % name)
+  return None, None
+
+
+def is_cache_from_redis(name, redis_cache_names):
+  """Check whether name should be cached.
+
+  Args:
+    name: string, the variable name to be checked
+    redis_cache_names: list of string, names which should be cached.
+
+  Return:
+    True if need to be cached
+  """
+  tok = name.split('/')
+  if tok[0].startswith('input_layer'):
+    tok = tok[1:]
+  for y in redis_cache_names:
+    for k in tok:
+      if k.startswith(y):
+        logging.info('embedding %s will be cached[specified by %s]' % (name, y))
+        return True
+  return False

easy_rec/python/utils/restore_filter.py

@@ -0,0 +1,89 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""Define filters for restore."""
+
+from abc import ABCMeta
+from abc import abstractmethod
+from enum import Enum
+
+
+class Logical(Enum):
+  AND = 1
+  OR = 2
+
+
+class Filter:
+  __metaclass__ = ABCMeta
+
+  def __init__(self):
+    pass
+
+  @abstractmethod
+  def keep(self, var_name):
+    """Keep the var or not.
+
+    Args:
+      var_name: input name of the var
+
+    Returns:
+      True if the var will be kept, else False
+    """
+    return True
+
+
+class KeywordFilter(Filter):
+
+  def __init__(self, pattern, exclusive=False):
+    """Init KeywordFilter.
+
+    Args:
+      pattern: keyword to be matched
+      exclusive: if True, var_name should include the pattern
+          else, var_name should not include the pattern
+    """
+    self._pattern = pattern
+    self._exclusive = exclusive
+
+  def keep(self, var_name):
+    if not self._exclusive:
+      return self._pattern in var_name
+    else:
+      return self._pattern not in var_name
+
+
+class CombineFilter(Filter):
+
+  def __init__(self, filters, logical=Logical.AND):
+    """Init CombineFilter.
+
+    Args:
+      filters: a set of filters to be combined
+      logical: logical and/or combination of the filters
+    """
+    self._filters = filters
+    self._logical = logical
+
+  def keep(self, var_name):
+    if self._logical == Logical.AND:
+      for one_filter in self._filters:
+        if not one_filter.keep(var_name):
+          return False
+      return True
+    elif self._logical == Logical.OR:
+      for one_filter in self._filters:
+        if one_filter.keep(var_name):
+          return True
+      return False
+
+
+class ScopeDrop:
+  """For drop out scope prefix when restore variables from checkpoint."""
+
+  def __init__(self, scope_name):
+    self._scope_name = scope_name
+    if len(self._scope_name) >= 0:
+      if self._scope_name[-1] != '/':
+        self._scope_name += '/'
+
+  def update(self, var_name):
+    return var_name.replace(self._scope_name, '')