easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/utils/dag.py

@@ -0,0 +1,192 @@
+import logging
+from collections import OrderedDict
+from collections import defaultdict
+from copy import copy
+from copy import deepcopy
+
+
+class DAG(object):
+  """Directed acyclic graph implementation."""
+
+  def __init__(self):
+    """Construct a new DAG with no nodes or edges."""
+    self.reset_graph()
+
+  def add_node(self, node_name, graph=None):
+    """Add a node if it does not exist yet, or error out."""
+    if not graph:
+      graph = self.graph
+    if node_name in graph:
+      raise KeyError('node %s already exists' % node_name)
+    graph[node_name] = set()
+
+  def add_node_if_not_exists(self, node_name, graph=None):
+    try:
+      self.add_node(node_name, graph=graph)
+    except KeyError:
+      logging.info('node %s already exist' % node_name)
+
+  def delete_node(self, node_name, graph=None):
+    """Deletes this node and all edges referencing it."""
+    if not graph:
+      graph = self.graph
+    if node_name not in graph:
+      raise KeyError('node %s does not exist' % node_name)
+    graph.pop(node_name)
+
+    for node, edges in graph.items():
+      if node_name in edges:
+        edges.remove(node_name)
+
+  def delete_node_if_exists(self, node_name, graph=None):
+    try:
+      self.delete_node(node_name, graph=graph)
+    except KeyError:
+      logging.info('node %s does not exist' % node_name)
+
+  def add_edge(self, ind_node, dep_node, graph=None):
+    """Add an edge (dependency) between the specified nodes."""
+    if not graph:
+      graph = self.graph
+    if ind_node not in graph or dep_node not in graph:
+      raise KeyError('one or more nodes do not exist in graph')
+    test_graph = deepcopy(graph)
+    test_graph[ind_node].add(dep_node)
+    is_valid, message = self.validate(test_graph)
+    if is_valid:
+      graph[ind_node].add(dep_node)
+    else:
+      raise Exception('invalid DAG')
+
+  def delete_edge(self, ind_node, dep_node, graph=None):
+    """Delete an edge from the graph."""
+    if not graph:
+      graph = self.graph
+    if dep_node not in graph.get(ind_node, []):
+      raise KeyError('this edge does not exist in graph')
+    graph[ind_node].remove(dep_node)
+
+  def rename_edges(self, old_task_name, new_task_name, graph=None):
+    """Change references to a task in existing edges."""
+    if not graph:
+      graph = self.graph
+    for node, edges in graph.items():
+
+      if node == old_task_name:
+        graph[new_task_name] = copy(edges)
+        del graph[old_task_name]
+
+      else:
+        if old_task_name in edges:
+          edges.remove(old_task_name)
+          edges.add(new_task_name)
+
+  def predecessors(self, node, graph=None):
+    """Returns a list of all predecessors of the given node."""
+    if graph is None:
+      graph = self.graph
+    return [key for key in graph if node in graph[key]]
+
+  def downstream(self, node, graph=None):
+    """Returns a list of all nodes this node has edges towards."""
+    if graph is None:
+      graph = self.graph
+    if node not in graph:
+      raise KeyError('node %s is not in graph' % node)
+    return list(graph[node])
+
+  def all_downstreams(self, node, graph=None):
+    """Returns a list of all nodes ultimately downstream of the given node in the dependency graph.
+
+    in topological order.
+    """
+    if graph is None:
+      graph = self.graph
+    nodes = [node]
+    nodes_seen = set()
+    i = 0
+    while i < len(nodes):
+      downstreams = self.downstream(nodes[i], graph)
+      for downstream_node in downstreams:
+        if downstream_node not in nodes_seen:
+          nodes_seen.add(downstream_node)
+          nodes.append(downstream_node)
+      i += 1
+    return list(
+        filter(lambda node: node in nodes_seen,
+               self.topological_sort(graph=graph)))
+
+  def all_leaves(self, graph=None):
+    """Return a list of all leaves (nodes with no downstreams)."""
+    if graph is None:
+      graph = self.graph
+    return [key for key in graph if not graph[key]]
+
+  def from_dict(self, graph_dict):
+    """Reset the graph and build it from the passed dictionary.
+
+    The dictionary takes the form of {node_name: [directed edges]}
+    """
+    self.reset_graph()
+    for new_node in graph_dict.keys():
+      self.add_node(new_node)
+    for ind_node, dep_nodes in graph_dict.items():
+      if not isinstance(dep_nodes, list):
+        raise TypeError('dict values must be lists')
+      for dep_node in dep_nodes:
+        self.add_edge(ind_node, dep_node)
+
+  def reset_graph(self):
+    """Restore the graph to an empty state."""
+    self.graph = OrderedDict()
+
+  def independent_nodes(self, graph=None):
+    """Returns a list of all nodes in the graph with no dependencies."""
+    if graph is None:
+      graph = self.graph
+
+    dependent_nodes = set(
+        node for dependents in graph.values() for node in dependents)
+    return [node for node in graph.keys() if node not in dependent_nodes]
+
+  def validate(self, graph=None):
+    """Returns (Boolean, message) of whether DAG is valid."""
+    graph = graph if graph is not None else self.graph
+    if len(self.independent_nodes(graph)) == 0:
+      return False, 'no independent nodes detected'
+    try:
+      self.topological_sort(graph)
+    except ValueError:
+      return False, 'failed topological sort'
+    return True, 'valid'
+
+  def topological_sort(self, graph=None):
+    """Returns a topological ordering of the DAG.
+
+    Raises an error if this is not possible (graph is not valid).
+    """
+    if graph is None:
+      graph = self.graph
+    result = []
+    in_degree = defaultdict(lambda: 0)
+
+    for u in graph:
+      for v in graph[u]:
+        in_degree[v] += 1
+    ready = [node for node in graph if not in_degree[node]]
+
+    while ready:
+      u = ready.pop()
+      result.append(u)
+      for v in graph[u]:
+        in_degree[v] -= 1
+        if in_degree[v] == 0:
+          ready.append(v)
+
+    if len(result) == len(graph):
+      return result
+    else:
+      raise ValueError('graph is not acyclic')
+
+  def size(self):
+    return len(self.graph)

easy_rec/python/utils/distribution_utils.py

@@ -0,0 +1,268 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from __future__ import print_function
+
+import json
+import logging
+import os
+
+import tensorflow as tf
+
+from easy_rec.python.protos.train_pb2 import DistributionStrategy
+from easy_rec.python.utils import estimator_utils
+from easy_rec.python.utils.estimator_utils import chief_to_master
+from easy_rec.python.utils.estimator_utils import master_to_chief
+
+DistributionStrategyMap = {
+    '': DistributionStrategy.NoStrategy,
+    'ps': DistributionStrategy.PSStrategy,
+    'ess': DistributionStrategy.ExascaleStrategy,
+    'mirrored': DistributionStrategy.MirroredStrategy,
+    'collective': DistributionStrategy.CollectiveAllReduceStrategy
+}
+
+
+def set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker,
+                            distribute_strategy):
+  if distribute_strategy in [
+      DistributionStrategy.PSStrategy, DistributionStrategy.MirroredStrategy,
+      DistributionStrategy.CollectiveAllReduceStrategy,
+      DistributionStrategy.ExascaleStrategy
+  ]:
+    pipeline_config.train_config.sync_replicas = False
+    pipeline_config.train_config.train_distribute = distribute_strategy
+    pipeline_config.train_config.num_gpus_per_worker = num_gpus_per_worker
+  print('Dump pipeline_config.train_config:')
+  print(pipeline_config.train_config)
+
+
+def set_tf_config_and_get_train_worker_num(
+    ps_hosts,
+    worker_hosts,
+    task_index,
+    job_name,
+    distribute_strategy=DistributionStrategy.NoStrategy,
+    eval_method='none'):
+  logging.info(
+      'set_tf_config_and_get_train_worker_num: distribute_strategy = %d' %
+      distribute_strategy)
+  worker_hosts = worker_hosts.split(',')
+  ps_hosts = ps_hosts.split(',') if ps_hosts else []
+
+  total_worker_num = len(worker_hosts)
+  train_worker_num = total_worker_num
+
+  print('Original TF_CONFIG=%s' % os.environ.get('TF_CONFIG', ''))
+  print('worker_hosts=%s ps_hosts=%s task_index=%d job_name=%s' %
+        (','.join(worker_hosts), ','.join(ps_hosts), task_index, job_name))
+  print('eval_method=%s' % eval_method)
+
+  if distribute_strategy == DistributionStrategy.MirroredStrategy:
+    assert total_worker_num == 1, 'mirrored distribute strategy only need 1 worker'
+  elif distribute_strategy in [
+      DistributionStrategy.NoStrategy, DistributionStrategy.PSStrategy,
+      DistributionStrategy.CollectiveAllReduceStrategy,
+      DistributionStrategy.ExascaleStrategy
+  ]:
+    cluster, task_type, task_index_ = estimator_utils.parse_tf_config()
+    train_worker_num = 0
+    if eval_method == 'separate':
+      if 'evaluator' in cluster:
+        # 'evaluator' in cluster indicates user use new-style cluster content
+        if 'chief' in cluster:
+          train_worker_num += len(cluster['chief'])
+        elif 'master' in cluster:
+          train_worker_num += len(cluster['master'])
+        if 'worker' in cluster:
+          train_worker_num += len(cluster['worker'])
+        # drop evaluator to avoid hang
+        if distribute_strategy == DistributionStrategy.NoStrategy:
+          del cluster['evaluator']
+        tf_config = {
+            'cluster': cluster,
+            'task': {
+                'type': task_type,
+                'index': task_index_
+            }
+        }
+        os.environ['TF_CONFIG'] = json.dumps(tf_config)
+      else:
+        # backward compatibility, if user does not assign one evaluator in
+        # -Dcluster, we use first worker for chief, second for evaluation
+        train_worker_num = total_worker_num - 1
+        assert train_worker_num > 0, 'in distribution mode worker num must be greater than 1, ' \
+                                     'the second worker will be used as evaluator'
+        if len(worker_hosts) > 1:
+          cluster = {'chief': [worker_hosts[0]], 'worker': worker_hosts[2:]}
+          if distribute_strategy != DistributionStrategy.NoStrategy:
+            cluster['evaluator'] = [worker_hosts[1]]
+          if len(ps_hosts) > 0:
+            cluster['ps'] = ps_hosts
+          if job_name == 'ps':
+            os.environ['TF_CONFIG'] = json.dumps({
+                'cluster': cluster,
+                'task': {
+                    'type': job_name,
+                    'index': task_index
+                }
+            })
+          elif job_name == 'worker':
+            if task_index == 0:
+              os.environ['TF_CONFIG'] = json.dumps({
+                  'cluster': cluster,
+                  'task': {
+                      'type': 'chief',
+                      'index': 0
+                  }
+              })
+            elif task_index == 1:
+              os.environ['TF_CONFIG'] = json.dumps({
+                  'cluster': cluster,
+                  'task': {
+                      'type': 'evaluator',
+                      'index': 0
+                  }
+              })
+            else:
+              os.environ['TF_CONFIG'] = json.dumps({
+                  'cluster': cluster,
+                  'task': {
+                      'type': job_name,
+                      'index': task_index - 2
+                  }
+              })
+    else:
+      if 'evaluator' in cluster:
+        evaluator = cluster['evaluator']
+        del cluster['evaluator']
+        # 'evaluator' in cluster indicates user use new-style cluster content
+        train_worker_num += 1
+        if 'chief' in cluster:
+          train_worker_num += len(cluster['chief'])
+        elif 'master' in cluster:
+          train_worker_num += len(cluster['master'])
+        if 'worker' in cluster:
+          train_worker_num += len(cluster['worker'])
+          cluster['worker'].append(evaluator[0])
+        else:
+          cluster['worker'] = [evaluator[0]]
+        if task_type == 'evaluator':
+          tf_config = {
+              'cluster': cluster,
+              'task': {
+                  'type': 'worker',
+                  'index': train_worker_num - 2
+              }
+          }
+        else:
+          tf_config = {
+              'cluster': cluster,
+              'task': {
+                  'type': task_type,
+                  'index': task_index_
+              }
+          }
+        os.environ['TF_CONFIG'] = json.dumps(tf_config)
+      else:
+        cluster = {'chief': [worker_hosts[0]], 'worker': worker_hosts[1:]}
+        train_worker_num = len(worker_hosts)
+        if len(ps_hosts) > 0:
+          cluster['ps'] = ps_hosts
+        if job_name == 'ps':
+          os.environ['TF_CONFIG'] = json.dumps({
+              'cluster': cluster,
+              'task': {
+                  'type': job_name,
+                  'index': task_index
+              }
+          })
+        else:
+          if task_index == 0:
+            os.environ['TF_CONFIG'] = json.dumps({
+                'cluster': cluster,
+                'task': {
+                    'type': 'chief',
+                    'index': 0
+                }
+            })
+          else:
+            os.environ['TF_CONFIG'] = json.dumps({
+                'cluster': cluster,
+                'task': {
+                    'type': 'worker',
+                    'index': task_index - 1
+                }
+            })
+      if eval_method == 'none':
+        # change master to chief, will not evaluate
+        master_to_chief()
+      elif eval_method == 'master':
+        # change chief to master, will evaluate on master
+        chief_to_master()
+  else:
+    assert distribute_strategy == '', 'invalid distribute_strategy %s'\
+           % distribute_strategy
+    cluster, task_type, task_index = estimator_utils.parse_tf_config()
+  print('Final TF_CONFIG = %s' % os.environ.get('TF_CONFIG', ''))
+  tf.logging.info('TF_CONFIG %s' % os.environ.get('TF_CONFIG', ''))
+  tf.logging.info('distribute_stategy %s, train_worker_num: %d' %
+                  (distribute_strategy, train_worker_num))
+
+  # remove pai chief-worker waiting strategy
+  # which is conflicted with worker waiting strategy in easyrec
+  if 'TF_WRITE_WORKER_STATUS_FILE' in os.environ:
+    del os.environ['TF_WRITE_WORKER_STATUS_FILE']
+  return train_worker_num
+
+
+def set_tf_config_and_get_train_worker_num_on_ds():
+  if 'TF_CONFIG' not in os.environ:
+    return
+  tf_config = json.loads(os.environ['TF_CONFIG'])
+  if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and (
+      'evaluator' not in tf_config['cluster']):
+    easyrec_tf_config = dict()
+    easyrec_tf_config['cluster'] = {}
+    easyrec_tf_config['task'] = {}
+    easyrec_tf_config['cluster']['ps'] = tf_config['cluster']['ps']
+    easyrec_tf_config['cluster']['chief'] = [tf_config['cluster']['worker'][0]]
+    easyrec_tf_config['cluster']['worker'] = tf_config['cluster']['worker'][2:]
+
+    if tf_config['task']['type'] == 'worker' and tf_config['task']['index'] == 0:
+      easyrec_tf_config['task']['type'] = 'chief'
+      easyrec_tf_config['task']['index'] = 0
+    elif tf_config['task']['type'] == 'worker' and tf_config['task'][
+        'index'] == 1:
+      easyrec_tf_config['task']['type'] = 'evaluator'
+      easyrec_tf_config['task']['index'] = 0
+    elif tf_config['task']['type'] == 'worker':
+      easyrec_tf_config['task']['type'] = tf_config['task']['type']
+      easyrec_tf_config['task']['index'] = tf_config['task']['index'] - 2
+    else:
+      easyrec_tf_config['task']['type'] = tf_config['task']['type']
+      easyrec_tf_config['task']['index'] = tf_config['task']['index']
+    os.environ['TF_CONFIG'] = json.dumps(easyrec_tf_config)
+
+
+def set_tf_config_and_get_distribute_eval_worker_num_on_ds():
+  assert 'TF_CONFIG' in os.environ, "'TF_CONFIG' must in os.environ"
+  tf_config = json.loads(os.environ['TF_CONFIG'])
+  if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and (
+      'evaluator' not in tf_config['cluster']):
+    easyrec_tf_config = dict()
+    easyrec_tf_config['cluster'] = {}
+    easyrec_tf_config['task'] = {}
+    easyrec_tf_config['cluster']['ps'] = tf_config['cluster']['ps']
+    easyrec_tf_config['cluster']['chief'] = [tf_config['cluster']['worker'][0]]
+    easyrec_tf_config['cluster']['worker'] = tf_config['cluster']['worker'][1:]
+
+    if tf_config['task']['type'] == 'worker' and tf_config['task']['index'] == 0:
+      easyrec_tf_config['task']['type'] = 'chief'
+      easyrec_tf_config['task']['index'] = 0
+    elif tf_config['task']['type'] == 'worker':
+      easyrec_tf_config['task']['type'] = tf_config['task']['type']
+      easyrec_tf_config['task']['index'] = tf_config['task']['index'] - 1
+    else:
+      easyrec_tf_config['task']['type'] = tf_config['task']['type']
+      easyrec_tf_config['task']['index'] = tf_config['task']['index']
+    os.environ['TF_CONFIG'] = json.dumps(easyrec_tf_config)

easy_rec/python/utils/ds_util.py

@@ -0,0 +1,65 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+import os
+import subprocess
+import traceback
+
+from tensorflow.python.platform import gfile
+
+from easy_rec.python.utils import estimator_utils
+
+
+def is_on_ds():
+  # IS_ON_PAI is set in train_eval
+  # which is the entry on DataScience platform
+  return 'IS_ON_DS' in os.environ
+
+
+def set_on_ds():
+  logging.info('set on ds environment variable: IS_ON_DS')
+  os.environ['IS_ON_DS'] = '1'
+
+
+def cache_ckpt(pipeline_config):
+  fine_tune_ckpt_path = pipeline_config.train_config.fine_tune_checkpoint
+  if not fine_tune_ckpt_path.startswith('hdfs://'):
+    # there is no need to cache if remote directories are mounted
+    return
+
+  if estimator_utils.is_ps() or estimator_utils.is_chief(
+  ) or estimator_utils.is_master():
+    tmpdir = os.path.dirname(fine_tune_ckpt_path.replace('hdfs://', ''))
+    tmpdir = os.path.join('/tmp/experiments', tmpdir)
+    logging.info('will cache fine_tune_ckpt to local dir: %s' % tmpdir)
+    if gfile.IsDirectory(tmpdir):
+      gfile.DeleteRecursively(tmpdir)
+    gfile.MakeDirs(tmpdir)
+    src_files = gfile.Glob(fine_tune_ckpt_path + '*')
+    src_files.sort()
+    data_files = [x for x in src_files if '.data-' in x]
+    meta_files = [x for x in src_files if '.data-' not in x]
+    if estimator_utils.is_ps():
+      _, _, ps_id = estimator_utils.parse_tf_config()
+      ps_id = (ps_id % len(data_files))
+      data_files = data_files[ps_id:] + data_files[:ps_id]
+      src_files = meta_files + data_files
+    else:
+      src_files = meta_files
+    for src_path in src_files:
+      _, file_name = os.path.split(src_path)
+      dst_path = os.path.join(tmpdir, os.path.basename(src_path))
+      logging.info('will copy %s to local path %s' % (src_path, dst_path))
+      try:
+        output = subprocess.check_output(
+            'hadoop fs -get %s %s' % (src_path, dst_path), shell=True)
+        logging.info('copy succeed: %s' % output)
+      except Exception:
+        logging.warning('exception: %s' % traceback.format_exc())
+    ckpt_filename = os.path.basename(fine_tune_ckpt_path)
+    fine_tune_ckpt_path = os.path.join(tmpdir, ckpt_filename)
+    pipeline_config.train_config.fine_tune_checkpoint = fine_tune_ckpt_path
+    logging.info('will restore from %s' % fine_tune_ckpt_path)
+  else:
+    # workers do not have to create the restore graph
+    pipeline_config.train_config.ClearField('fine_tune_checkpoint')

easy_rec/python/utils/embedding_utils.py

@@ -0,0 +1,73 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+
+import tensorflow as tf
+from tensorflow.python.framework import ops
+
+from easy_rec.python.utils import constant
+from easy_rec.python.utils import proto_util
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def get_norm_name_to_ids():
+  """Get normalize embedding name(including kv variables) to ids.
+
+  Return:
+    normalized names to ids mapping.
+  """
+  norm_name_to_ids = {}
+  for x in ops.get_collection(constant.SPARSE_UPDATE_VARIABLES):
+    norm_name, part_id = proto_util.get_norm_embed_name(x[0].name)
+    norm_name_to_ids[norm_name] = 1
+
+  for tid, t in enumerate(norm_name_to_ids.keys()):
+    norm_name_to_ids[t] = str(tid)
+  return norm_name_to_ids
+
+
+def get_sparse_name_to_ids():
+  """Get embedding variable(including kv variables) name to ids mapping.
+
+  Return:
+     variable names to ids mappping.
+  """
+  norm_name_to_ids = get_norm_name_to_ids()
+  name_to_ids = {}
+  for x in ops.get_collection(constant.SPARSE_UPDATE_VARIABLES):
+    norm_name, _ = proto_util.get_norm_embed_name(x[0].name)
+    name_to_ids[x[0].name] = norm_name_to_ids[norm_name]
+  return name_to_ids
+
+
+def get_dense_name_to_ids():
+  dense_train_vars = ops.get_collection(constant.DENSE_UPDATE_VARIABLES)
+  norm_name_to_ids = {}
+  for tid, x in enumerate(dense_train_vars):
+    norm_name_to_ids[x.op.name] = tid
+  return norm_name_to_ids
+
+
+embedding_parallel = False
+
+
+def set_embedding_parallel():
+  global embedding_parallel
+  embedding_parallel = True
+
+
+def is_embedding_parallel():
+  global embedding_parallel
+  return embedding_parallel
+
+
+def sort_col_by_name():
+  return constant.SORT_COL_BY_NAME in os.environ
+
+
+def embedding_on_cpu():
+  place_on_cpu = os.getenv(constant.EmbeddingOnCPU)
+  place_on_cpu = eval(place_on_cpu) if place_on_cpu else False
+  return place_on_cpu