easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/utils/expr_util.py

@@ -0,0 +1,118 @@
+from collections import deque
+
+
+def _process_multi_expr(expr):
+  expr = expr.strip()
+  size = len(expr)
+  idx = 0
+  two_expr = ['>=', '<=', '==']
+  expr_list = []
+  while (idx < size):
+    if idx + 2 <= size and expr[idx:idx + 2] in two_expr:
+      expr_list.append(expr[idx:idx + 2])
+      idx += 2
+    else:
+      expr_list.append(expr[idx])
+      idx += 1
+  return expr_list
+
+
+def _process_enum(enum, input_names, prefix=''):
+  enum = enum.strip()
+  if enum in input_names:
+    enum = "parsed_dict['%s']" % (prefix + enum)
+  return enum
+
+
+def _get_expression_list(expression, input_names, prefix=''):
+  ops = [
+      '+', '-', '*', '/', '(', ')', '>', '>=', '<', '<=', '==', '=', '&', '|'
+  ]
+  expression_list = []
+  eunm = ''
+  pre_expr = ''
+
+  for i in expression:
+    if i in ops:
+      if eunm:
+        expression_list.append(_process_enum(eunm, input_names, prefix=prefix))
+        eunm = ''
+      pre_expr += i
+    else:
+      eunm += i
+      if pre_expr:
+        expression_list.extend(_process_multi_expr(pre_expr))
+        pre_expr = ''
+  if eunm:
+    expression_list.append(_process_enum(eunm, input_names, prefix=prefix))
+  if pre_expr:
+    expression_list.extend(_process_multi_expr(pre_expr))
+
+  final_expression_list = ['']
+  ops = ['(', ')', '>=', '<=', '==', '>', '<', '&', '|']
+  for expr in expression_list:
+    if expr in ops:
+      final_expression_list.append(expr)
+    elif final_expression_list[-1] not in ops:
+      final_expression_list[-1] += expr
+    else:
+      final_expression_list.append(expr)
+  final_expression_list = [expr for expr in final_expression_list if expr]
+  return final_expression_list
+
+
+def _solve(enum, sign, stack):
+  if len(stack) == 0 or enum == '' or sign == '':
+    return enum
+  op1 = stack.pop()
+  op2 = enum
+  if sign == '>':
+    result = 'tf.greater(%s, %s)' % (op1, op2)
+  elif sign == '>=':
+    result = 'tf.greater_equal(%s, %s)' % (op1, op2)
+  elif sign == '<':
+    result = 'tf.less(%s, %s)' % (op1, op2)
+  elif sign == '<=':
+    result = 'tf.less_equal(%s, %s)' % (op1, op2)
+  elif sign == '==':
+    result = 'tf.equal(%s, %s)' % (op1, op2)
+  elif sign == '&':
+    result = '%s & %s' % (op1, op2)
+  elif sign == '|':
+    result = '%s | %s' % (op1, op2)
+  else:
+    assert False
+  return result
+
+
+def _expression_eval(expr_list):
+  ops = ['>', '>=', '<', '<=', '==', '&', '|', '(', ')']
+  stack = deque()
+  sign = ''
+  operand = ''
+  for c in expr_list:
+    if c == ' ':
+      continue
+    elif c not in ops:
+      operand = c
+    elif c == '(':
+      stack.append(sign)
+      sign = ''
+    else:
+      result = _solve(operand, sign, stack)
+      operand = ''
+      if c == ')':
+        sign = stack.pop()
+        operand = _solve(result, sign, stack)
+        sign = ''
+      else:
+        sign = c
+        stack.append(result)
+  expr_str = _solve(operand, sign, stack)
+  return expr_str
+
+
+def get_expression(expression, input_names, prefix=''):
+  expression_list = _get_expression_list(expression, input_names, prefix=prefix)
+  expression = _expression_eval(expression_list)
+  return expression

easy_rec/python/utils/fg_util.py

@@ -0,0 +1,53 @@
+import json
+import logging
+
+import tensorflow as tf
+
+from easy_rec.python.protos.dataset_pb2 import DatasetConfig
+from easy_rec.python.protos.feature_config_pb2 import FeatureConfig
+from easy_rec.python.utils.config_util import get_compatible_feature_configs
+
+from easy_rec.python.utils.convert_rtp_fg import load_input_field_and_feature_config  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def load_fg_json_to_config(pipeline_config):
+  fg_json_path = pipeline_config.fg_json_path
+  if not fg_json_path:
+    return
+
+  if fg_json_path.startswith('!'):
+    # already loaded
+    return
+
+  label_fields = pipeline_config.data_config.label_fields
+  with tf.gfile.GFile(fg_json_path, 'r') as fin:
+    rtp_fg = json.load(fin)
+
+  fg_config = load_input_field_and_feature_config(
+      rtp_fg, label_fields=label_fields)
+
+  pipeline_config.data_config.ClearField('input_fields')
+  pipeline_config.ClearField('feature_configs')
+
+  # not clear features so that we could define extra features
+  # which is not defined in fg.json
+  # pipeline_config.feature_config.ClearField('features')
+
+  for input_config in fg_config.data_config.input_fields:
+    in_config = DatasetConfig.Field()
+    in_config.CopyFrom(input_config)
+    pipeline_config.data_config.input_fields.append(in_config)
+  fg_fea_config = get_compatible_feature_configs(fg_config)
+  for fc in fg_fea_config:
+    fea_config = FeatureConfig()
+    fea_config.CopyFrom(fc)
+    pipeline_config.feature_config.features.append(fea_config)
+  logging.info('data_config and feature_config has been replaced by fg_json.')
+
+  # signal that it is already loaded
+  pipeline_config.fg_json_path = '!' + pipeline_config.fg_json_path
+
+  return pipeline_config

easy_rec/python/utils/hit_rate_utils.py

@@ -0,0 +1,220 @@
+import logging
+
+import graphlearn as gl
+import numpy as np
+import tensorflow as tf
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def load_graph(i_emb_table, emb_dim, knn_metric, timeout, knn_strict):
+  """Load embedding tables in GL.
+
+  that used to lookup embedding and do knn search.
+  """
+  gl.set_knn_metric(knn_metric)
+  gl.set_timeout(timeout)
+  option = gl.IndexOption()
+  option.name = 'knn'
+  if knn_strict:
+    # option.index_type = "flat"
+    option.index_type = 'ivfflat'
+    option.nlist = 5
+    option.nprobe = 5
+  else:
+    option.index_type = 'ivfflat'
+    option.nlist = 5
+    option.nprobe = 2
+  g = gl.Graph().node(
+      i_emb_table,
+      node_type='i',
+      decoder=gl.Decoder(attr_types=['float'] * emb_dim, attr_delimiter=','),
+      option=option)
+  return g
+
+
+def batch_hitrate(src_ids,
+                  recall_ids,
+                  recall_distances,
+                  gt_items,
+                  num_interests,
+                  mask=None):
+  """Compute hitrate of a batch of src ids.
+
+  Args:
+    src_ids: trigger id, a numpy array.
+    recall_ids: recalled ids by src_ids, a numpy array.
+    recall_distances: corresponding distances of recalled ids, a numpy array.
+    gt_items: batch of ground truth item ids list, a list of list.
+    num_interests: max number of interests.
+    mask: some models have different number of interests.
+
+  Returns:
+    hitrates: hitrate of src_ids, a list.
+    bad_cases: bad cases, a list of list.
+    bad_dsts: distances of bad cases, a list of list.
+    hits: total hit counts of a batch of src ids, a scalar.
+    gt_count: total ground truth items num of a batch of src ids, a scalar.
+  """
+  hitrates = []
+  bad_cases = []
+  bad_dists = []
+  hits = 0.0
+  gt_count = 0.0
+  for idx, src_id in enumerate(src_ids):
+    recall_id = recall_ids[idx]
+    recall_distance = recall_distances[idx]
+
+    bad_case = {}
+    gt_items_size = len(gt_items[idx])
+    hit_ids = []
+    if gt_items_size == 0:  # just skip invalid record.
+      print('Id {:d} has no related items sequence, just skip.'.format(src_id))
+      continue
+    for interest_id in range(num_interests):
+      if not mask[idx, interest_id]:
+        continue
+      for k, id in enumerate(recall_id[interest_id]):
+        if id in gt_items[idx]:
+          if id not in hit_ids:
+            hit_ids.append(id)
+        else:
+          dis = recall_distance[interest_id][k]
+          if id not in bad_case:
+            bad_case[id] = dis
+          elif dis < bad_case[id]:
+            bad_case[id] = dis
+    hit_count = float(len(hit_ids))
+    hitrates.append(hit_count / gt_items_size)
+    hits += hit_count
+    gt_count += gt_items_size
+    bad_cases.append([x for x in bad_case])
+    bad_dists.append([bad_case[x] for x in bad_case])
+  return hitrates, bad_cases, bad_dists, hits, gt_count
+
+
+def reduce_hitrate(cluster, hits, count, task_index):
+  """Reduce hitrate of all workers.
+
+  Args:
+    cluster: tf cluster.
+    hits: total_hits of each worker.
+    count: total count of ground truth items of each worker.
+    task_index: worker index.
+
+  Returns:
+    var_total_hitrate: variable of total hitrate.
+    var_worker_count: variable used to mark the number of worker that
+    have completed the calculation of hitrate.
+  """
+  with tf.device(
+      tf.train.replica_device_setter(
+          worker_device='/job:worker/task:%d' % task_index, cluster=cluster)):
+    with tf.variable_scope('hitrate_var', reuse=tf.AUTO_REUSE):
+      var_worker_count = tf.get_variable(
+          'worker_count',
+          shape=(),
+          dtype=tf.int32,
+          initializer=tf.zeros_initializer())
+      var_hits = tf.get_variable(
+          'hits',
+          shape=(),
+          dtype=tf.float32,
+          initializer=tf.zeros_initializer())
+      var_gt_count = tf.get_variable(
+          'gt_count',
+          shape=(),
+          dtype=tf.float32,
+          initializer=tf.zeros_initializer())
+      var_total_hitrate = tf.get_variable(
+          'total_hitate',
+          shape=(),
+          dtype=tf.float32,
+          initializer=tf.zeros_initializer())
+
+      var_hits = tf.assign_add(var_hits, hits, use_locking=True)
+      var_gt_count = tf.assign_add(var_gt_count, count, use_locking=True)
+      var_gt_count = tf.Print(
+          var_gt_count, [var_gt_count, var_hits],
+          message='var_gt_count/var_hits')
+      var_total_hitrate = tf.assign(
+          var_total_hitrate, var_hits / var_gt_count, use_locking=True)
+      with tf.control_dependencies([var_total_hitrate]):
+        var_worker_count = tf.assign_add(var_worker_count, 1, use_locking=True)
+  return var_total_hitrate, var_worker_count
+
+
+def compute_hitrate_batch(g, gt_record, emb_dim, num_interests, top_k):
+  """Reduce hitrate of one batch.
+
+  Args:
+    g: a GL Graph instance.
+    gt_record: record list of groung truth.
+    emb_dim: embedding dim.
+    num_interests: max number of interests.
+    top_k: top_k hitrate.
+
+  Returns:
+    hits: total hit counts of a batch of src ids, a scalar.
+    gt_count: total ground truth items num of a batch of src ids, a scalar.
+    src_ids: src ids, a list.
+    recall_ids: recall ids, a list.
+    recall_distances: recall distances, a list.
+    hitrates: hitrate of a batch of src_ids, a list.
+    bad_cases: bad cases, a list of list.
+    bad_dsts: distances of bad cases, a list of list.
+  """
+
+  def _to_float_attrs(x):
+    # incase user embedding is not present
+    if x == '':
+      return np.zeros([emb_dim], dtype=np.float32)
+    embed = np.array(x.split(','), dtype=np.float32)
+    assert len(embed) == emb_dim, 'invalid embed len=%d, x=%s' % (len(embed), x)
+    return embed
+
+  def _to_multi_float_attrs(x, userid):
+    if x == '':
+      arr = [_to_float_attrs(x) for i in range(num_interests)]
+    else:
+      arr = [_to_float_attrs(sub_x) for sub_x in x.split('|')]
+    assert len(arr) == num_interests, 'invalid arr len=%d, x=%s, userid=%s' % (
+        len(arr), x, userid)
+    return arr
+
+  src_ids = np.array([src_items[0] for src_items in gt_record])
+  user_embedding = np.array([
+      _to_multi_float_attrs(src_items[2], src_items[0])
+      for src_items in gt_record
+  ])
+  user_emb_num = [src_items[3] for src_items in gt_record]
+
+  print('max(user_emb_num) = %d len(src_ids) = %d' %
+        (np.max(user_emb_num), len(src_ids)))
+
+  # a list of list.
+  gt_items = [
+      list(map(int, src_items[1].split(','))) for src_items in gt_record
+  ]
+
+  logging.info('src_nodes.float_attrs.shape=%s' % str(user_embedding.shape))
+  user_embedding = user_embedding.reshape([-1, user_embedding.shape[-1]])
+  # numpy array
+  recall_ids, recall_distances = g.search('i', user_embedding,
+                                          gl.KnnOption(k=top_k))
+  logging.info('recall_ids.shape=%s' % str(recall_ids.shape))
+
+  def _make_mask(lens):
+    mask = np.ones([len(lens), num_interests], dtype=np.float32)
+    for tmp_id, tmp_len in enumerate(lens):
+      mask[tmp_id, int(tmp_len):] = 0
+    return mask
+
+  mask = _make_mask(user_emb_num)
+  recall_ids = recall_ids.reshape([-1, num_interests, recall_ids.shape[-1]])
+  recall_distances = recall_distances.reshape(
+      [-1, num_interests, recall_distances.shape[-1]])
+  hitrates, bad_cases, bad_dists, hits, gt_count = batch_hitrate(
+      src_ids, recall_ids, recall_distances, gt_items, num_interests, mask)
+  return hits, gt_count, src_ids, recall_ids, recall_distances, hitrates, bad_cases, bad_dists

easy_rec/python/utils/hive_utils.py

@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*-
+import logging
+
+try:
+  from pyhive import hive
+  from pyhive.exc import ProgrammingError
+except ImportError:
+  logging.warning('pyhive is not installed.')
+
+
+class TableInfo(object):
+
+  def __init__(self, tablename, selected_cols, partition_kv, limit_num):
+    self.tablename = tablename
+    self.selected_cols = selected_cols
+    self.partition_kv = partition_kv
+    self.limit_num = limit_num
+
+  def gen_sql(self):
+    part = ''
+    if self.partition_kv and len(self.partition_kv) > 0:
+      res = []
+      for k, v in self.partition_kv.items():
+        res.append('{}={}'.format(k, v))
+      part = ' '.join(res)
+    sql = """select {}
+    from {}""".format(self.selected_cols, self.tablename)
+
+    if part:
+      sql += """
+    where {}
+    """.format(part)
+    if self.limit_num is not None and self.limit_num > 0:
+      sql += ' limit {}'.format(self.limit_num)
+    return sql
+
+
+class HiveUtils(object):
+  """Common IO based interface, could run at local or on data science."""
+
+  def __init__(self,
+               data_config,
+               hive_config,
+               selected_cols='',
+               record_defaults=[],
+               task_index=0,
+               task_num=1):
+
+    self._data_config = data_config
+    self._hive_config = hive_config
+
+    self._num_epoch = data_config.num_epochs
+    self._num_epoch_record = 0
+    self._task_index = task_index
+    self._task_num = task_num
+    self._selected_cols = selected_cols
+    self._record_defaults = record_defaults
+
+  def _construct_table_info(self, table_name, limit_num):
+    # sample_table/dt=2014-11-23/name=a
+    segs = table_name.split('/')
+    table_name = segs[0].strip()
+    if len(segs) > 0:
+      partition_kv = {i.split('=')[0]: i.split('=')[1] for i in segs[1:]}
+    else:
+      partition_kv = None
+
+    table_info = TableInfo(table_name, self._selected_cols, partition_kv,
+                           limit_num)
+    return table_info
+
+  def _construct_hive_connect(self):
+    conn = hive.Connection(
+        host=self._hive_config.host,
+        port=self._hive_config.port,
+        username=self._hive_config.username,
+        database=self._hive_config.database)
+    return conn
+
+  def hive_read_line(self, input_path, limit_num=None):
+    table_info = self._construct_table_info(input_path, limit_num)
+    conn = self._construct_hive_connect()
+    cursor = conn.cursor()
+    sql = table_info.gen_sql()
+    cursor.execute(sql)
+
+    while True:
+      data = cursor.fetchmany(size=1)
+      if len(data) == 0:
+        break
+      yield data
+
+    cursor.close()
+    conn.close()
+
+  def hive_read_lines(self, input_path, batch_size, limit_num=None):
+    table_info = self._construct_table_info(input_path, limit_num)
+    conn = self._construct_hive_connect()
+    cursor = conn.cursor()
+    sql = table_info.gen_sql()
+    cursor.execute(sql)
+
+    while True:
+      data = cursor.fetchmany(size=batch_size)
+      if len(data) == 0:
+        break
+      yield data
+
+    cursor.close()
+    conn.close()
+
+  def run_sql(self, sql):
+    conn = self._construct_hive_connect()
+    cursor = conn.cursor()
+    cursor.execute(sql)
+    try:
+      data = cursor.fetchall()
+    except ProgrammingError:
+      data = []
+    return data
+
+  def is_table_or_partition_exist(self,
+                                  table_name,
+                                  partition_name=None,
+                                  partition_val=None):
+    if partition_name and partition_val:
+      sql = 'show partitions %s partition(%s=%s)' % (table_name, partition_name,
+                                                     partition_val)
+      try:
+        res = self.run_sql(sql)
+        if not res:
+          return False
+        else:
+          return True
+      except:  # noqa: E722
+        return False
+
+    else:
+      sql = 'desc %s' % table_name
+      try:
+        self.run_sql(sql)
+        return True
+      except:  # noqa: E722
+        return False
+
+  def get_table_location(self, input_path):
+    conn = self._construct_hive_connect()
+    cursor = conn.cursor()
+    partition = ''
+    if len(input_path.split('/')) == 2:
+      table_name, partition = input_path.split('/')
+      partition += '/'
+    else:
+      table_name = input_path
+    sql = 'desc formatted %s' % table_name
+    cursor.execute(sql)
+    data = cursor.fetchmany()
+    for line in data:
+      if line[0].startswith('Location'):
+        return line[1].strip() + '/' + partition
+    return None
+
+  def get_all_cols(self, input_path):
+    conn = self._construct_hive_connect()
+    cursor = conn.cursor()
+    sql = 'desc %s' % input_path.split('/')[0]
+    cursor.execute(sql)
+    data = cursor.fetchmany()
+    col_names = []
+    cols_types = []
+    pt_name = ''
+    if len(input_path.split('/')) == 2:
+      pt_name = input_path.split('/')[1].split('=')[0]
+
+    for col in data:
+      col_name = col[0].strip()
+      if col_name and (not col_name.startswith('#')) and (col_name
+                                                          not in col_names):
+        if col_name != pt_name:
+          col_names.append(col_name)
+          cols_types.append(col[1].strip())
+
+    return col_names, cols_types