easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/model/simple_multi_task.py

@@ -0,0 +1,54 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import tensorflow as tf
+
+from easy_rec.python.layers import dnn
+from easy_rec.python.model.multi_task_model import MultiTaskModel
+
+from easy_rec.python.protos.simple_multi_task_pb2 import SimpleMultiTask as SimpleMultiTaskConfig  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class SimpleMultiTask(MultiTaskModel):
+
+  def __init__(self,
+               model_config,
+               feature_configs,
+               features,
+               labels=None,
+               is_training=False):
+    super(SimpleMultiTask, self).__init__(model_config, feature_configs,
+                                          features, labels, is_training)
+
+    assert self._model_config.WhichOneof('model') == 'simple_multi_task', \
+        'invalid model config: %s' % self._model_config.WhichOneof('model')
+    self._model_config = self._model_config.simple_multi_task
+    assert isinstance(self._model_config, SimpleMultiTaskConfig)
+
+    if self.has_backbone:
+      self._features = self.backbone
+    else:
+      self._features, _ = self._input_layer(self._feature_dict, 'all')
+    self._init_towers(self._model_config.task_towers)
+
+  def build_predict_graph(self):
+    tower_outputs = {}
+    for i, task_tower_cfg in enumerate(self._task_towers):
+      tower_name = task_tower_cfg.tower_name
+      task_dnn = dnn.DNN(
+          task_tower_cfg.dnn,
+          self._l2_reg,
+          name=tower_name,
+          is_training=self._is_training)
+      task_fea = task_dnn(self._features)
+      task_output = tf.layers.dense(
+          inputs=task_fea,
+          units=task_tower_cfg.num_class,
+          kernel_regularizer=self._l2_reg,
+          name='dnn_output_%d' % i)
+      tower_outputs[tower_name] = task_output
+
+    self._add_to_prediction_dict(tower_outputs)
+    return self._prediction_dict

easy_rec/python/model/uniter.py

@@ -0,0 +1,46 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import tensorflow as tf
+
+from easy_rec.python.layers import dnn
+from easy_rec.python.layers import uniter
+from easy_rec.python.model.rank_model import RankModel
+
+from easy_rec.python.protos.uniter_pb2 import Uniter as UNITERConfig  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class Uniter(RankModel):
+  """UNITER: UNiversal Image-TExt Representation Learning.
+
+  See the original paper:
+  https://arxiv.org/abs/1909.11740
+  """
+
+  def __init__(self,
+               model_config,
+               feature_configs,
+               features,
+               labels=None,
+               is_training=False):
+    super(Uniter, self).__init__(model_config, feature_configs, features,
+                                 labels, is_training)
+    assert self._model_config.WhichOneof('model') == 'uniter', (
+        'invalid model config: %s' % self._model_config.WhichOneof('model'))
+
+    self._uniter_layer = uniter.Uniter(model_config, feature_configs, features,
+                                       self._model_config.uniter.config,
+                                       self._input_layer)
+    self._model_config = self._model_config.uniter
+
+  def build_predict_graph(self):
+    hidden = self._uniter_layer(self._is_training, l2_reg=self._l2_reg)
+    final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg,
+                              'final_dnn', self._is_training)
+    all_fea = final_dnn_layer(hidden)
+
+    final = tf.layers.dense(all_fea, self._num_class, name='output')
+    self._add_to_prediction_dict(final)
+    return self._prediction_dict

easy_rec/python/model/wide_and_deep.py

@@ -0,0 +1,121 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import tensorflow as tf
+
+from easy_rec.python.layers import dnn
+from easy_rec.python.model.rank_model import RankModel
+
+from easy_rec.python.protos.wide_and_deep_pb2 import WideAndDeep as WideAndDeepConfig  # NOQA
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class WideAndDeep(RankModel):
+
+  def __init__(self,
+               model_config,
+               feature_configs,
+               features,
+               labels=None,
+               is_training=False):
+    super(WideAndDeep, self).__init__(model_config, feature_configs, features,
+                                      labels, is_training)
+    assert model_config.WhichOneof('model') == 'wide_and_deep', \
+        'invalid model config: %s' % model_config.WhichOneof('model')
+    self._model_config = model_config.wide_and_deep
+    assert isinstance(self._model_config, WideAndDeepConfig)
+    assert self._input_layer.has_group('wide')
+    _, self._wide_features = self._input_layer(self._feature_dict, 'wide')
+    assert self._input_layer.has_group('deep')
+    _, self._deep_features = self._input_layer(self._feature_dict, 'deep')
+
+  def build_input_layer(self, model_config, feature_configs):
+    # overwrite create input_layer to support wide_output_dim
+    has_final = len(model_config.wide_and_deep.final_dnn.hidden_units) > 0
+    self._wide_output_dim = model_config.wide_and_deep.wide_output_dim
+    if not has_final:
+      model_config.wide_and_deep.wide_output_dim = model_config.num_class
+      self._wide_output_dim = model_config.num_class
+    super(WideAndDeep, self).build_input_layer(model_config, feature_configs)
+
+  def build_predict_graph(self):
+    wide_fea = tf.add_n(self._wide_features)
+    logging.info('wide features dimension: %d' % wide_fea.get_shape()[-1])
+
+    self._deep_features = tf.concat(self._deep_features, axis=1)
+    logging.info('input deep features dimension: %d' %
+                 self._deep_features.get_shape()[-1])
+
+    deep_layer = dnn.DNN(self._model_config.dnn, self._l2_reg, 'deep_feature',
+                         self._is_training)
+    deep_fea = deep_layer(self._deep_features)
+    logging.info('output deep features dimension: %d' %
+                 deep_fea.get_shape()[-1])
+
+    has_final = len(self._model_config.final_dnn.hidden_units) > 0
+    print('wide_deep has_final_dnn layers = %d' % has_final)
+    if has_final:
+      all_fea = tf.concat([wide_fea, deep_fea], axis=1)
+      final_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg,
+                            'final_dnn', self._is_training)
+      all_fea = final_layer(all_fea)
+      output = tf.layers.dense(
+          all_fea,
+          self._num_class,
+          kernel_regularizer=self._l2_reg,
+          name='output')
+    else:
+      deep_out = tf.layers.dense(
+          deep_fea,
+          self._num_class,
+          kernel_regularizer=self._l2_reg,
+          name='deep_out')
+      output = deep_out + wide_fea
+
+    self._add_to_prediction_dict(output)
+
+    return self._prediction_dict
+
+  def get_grouped_vars(self, opt_num):
+    """Group the vars into different optimization groups.
+
+    Each group will be optimized by a separate optimizer.
+
+    Args:
+      opt_num: number of optimizers from easyrec config.
+
+    Return:
+      list of list of variables.
+    """
+    assert opt_num <= 3, 'could only support 2 or 3 optimizers, ' + \
+        'if opt_num = 2, one for the wide , and one for the others, ' + \
+        'if opt_num = 3, one for the wide, second for the deep embeddings, ' + \
+        'and third for the other layers.'
+
+    if opt_num == 2:
+      wide_vars = []
+      deep_vars = []
+      for tmp_var in tf.trainable_variables():
+        if tmp_var.name.startswith('input_layer') and \
+            (not tmp_var.name.startswith('input_layer_1')):
+          wide_vars.append(tmp_var)
+        else:
+          deep_vars.append(tmp_var)
+      return [wide_vars, deep_vars]
+    elif opt_num == 3:
+      wide_vars = []
+      embedding_vars = []
+      deep_vars = []
+      for tmp_var in tf.trainable_variables():
+        if tmp_var.name.startswith('input_layer') and \
+            (not tmp_var.name.startswith('input_layer_1')):
+          wide_vars.append(tmp_var)
+        elif tmp_var.name.startswith(
+            'input_layer') or '/embedding_weights' in tmp_var.name:
+          embedding_vars.append(tmp_var)
+        else:
+          deep_vars.append(tmp_var)
+      return [wide_vars, embedding_vars, deep_vars]

easy_rec/python/ops/gen_kafka_ops.py

@@ -0,0 +1,193 @@
+"""Python wrappers around TensorFlow ops.
+
+This file is MACHINE GENERATED! Do not edit.
+Original C++ source file: kafka_ops_deprecated.cc
+"""
+
+import logging
+import os
+import traceback
+
+import six as _six
+import tensorflow as tf
+from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow
+from tensorflow.python.eager import context as _context
+from tensorflow.python.eager import core as _core
+from tensorflow.python.eager import execute as _execute
+# Needed to trigger the call to _set_call_cpp_shape_fn.
+from tensorflow.python.framework import dtypes as _dtypes
+from tensorflow.python.framework import ops as _ops
+from tensorflow.python.util.tf_export import tf_export
+
+import easy_rec
+
+kafka_module = None
+if easy_rec.ops_dir is not None:
+  kafka_ops_path = os.path.join(easy_rec.ops_dir, 'kafka.so')
+  if os.path.exists(kafka_ops_path):
+    try:
+      kafka_module = tf.load_op_library(kafka_ops_path)
+    except Exception:
+      logging.warning('load %s failed: %s' %
+                      (kafka_ops_path, traceback.format_exc()))
+
+
+@tf_export('io_kafka_dataset_v2')
+def io_kafka_dataset_v2(topics,
+                        servers,
+                        group,
+                        eof,
+                        timeout,
+                        config_global,
+                        config_topic,
+                        message_key,
+                        message_offset,
+                        name=None):
+  """Creates a dataset that emits the messages of one or more Kafka topics.
+
+  Args:
+    topics: A `Tensor` of type `string`.
+      A `tf.string` tensor containing one or more subscriptions,
+      in the format of [topic:partition:offset].
+    servers: A `Tensor` of type `string`. A list of bootstrap servers.
+    group: A `Tensor` of type `string`. The consumer group id.
+    eof: A `Tensor` of type `bool`.
+      If True, the kafka reader will stop on EOF.
+    timeout: A `Tensor` of type `int64`.
+      The timeout value for the Kafka Consumer to wait
+      (in millisecond).
+    config_global: A `Tensor` of type `string`.
+      A `tf.string` tensor containing global configuration
+      properties in [Key=Value] format,
+      eg. ["enable.auto.commit=false", "heartbeat.interval.ms=2000"],
+      please refer to 'Global configuration properties' in librdkafka doc.
+    config_topic: A `Tensor` of type `string`.
+      A `tf.string` tensor containing topic configuration
+      properties in [Key=Value] format, eg. ["auto.offset.reset=earliest"],
+      please refer to 'Topic configuration properties' in librdkafka doc.
+    message_key: A `Tensor` of type `bool`.
+    message_offset: A `Tensor` of type `bool`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of type `variant`.
+  """
+  return kafka_module.io_kafka_dataset_v2(
+      topics=topics,
+      servers=servers,
+      group=group,
+      eof=eof,
+      timeout=timeout,
+      config_global=config_global,
+      config_topic=config_topic,
+      message_key=message_key,
+      message_offset=message_offset,
+      name=name)
+
+
+def io_kafka_dataset_eager_fallback(topics,
+                                    servers,
+                                    group,
+                                    eof,
+                                    timeout,
+                                    config_global,
+                                    config_topic,
+                                    message_key,
+                                    message_offset,
+                                    name=None,
+                                    ctx=None):
+  """This is the slowpath function for Eager mode.
+
+  This is for function io_kafka_dataset
+  """
+  _ctx = ctx if ctx else _context.context()
+  topics = _ops.convert_to_tensor(topics, _dtypes.string)
+  servers = _ops.convert_to_tensor(servers, _dtypes.string)
+  group = _ops.convert_to_tensor(group, _dtypes.string)
+  eof = _ops.convert_to_tensor(eof, _dtypes.bool)
+  timeout = _ops.convert_to_tensor(timeout, _dtypes.int64)
+  config_global = _ops.convert_to_tensor(config_global, _dtypes.string)
+  config_topic = _ops.convert_to_tensor(config_topic, _dtypes.string)
+  message_key = _ops.convert_to_tensor(message_key, _dtypes.bool)
+  message_offset = _ops.convert_to_tensor(message_offset, _dtypes.bool)
+  _inputs_flat = [
+      topics, servers, group, eof, timeout, config_global, config_topic,
+      message_key, message_offset
+  ]
+  _attrs = None
+  _result = _execute.execute(
+      b'IOKafkaDataset',
+      1,
+      inputs=_inputs_flat,
+      attrs=_attrs,
+      ctx=_ctx,
+      name=name)
+  _execute.record_gradient('IOKafkaDataset', _inputs_flat, _attrs, _result,
+                           name)
+  _result, = _result
+  return _result
+
+
+@tf_export('io_write_kafka_v2')
+def io_write_kafka_v2(message, topic, servers, name=None):
+  r"""TODO: add doc.
+
+  Args:
+    message: A `Tensor` of type `string`.
+    topic: A `Tensor` of type `string`.
+    servers: A `Tensor` of type `string`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of type `string`.
+  """
+  _ctx = _context._context
+  if _ctx is None or not _ctx._eager_context.is_eager:
+    _op = kafka_module.io_write_kafka_v2(
+        message=message, topic=topic, servers=servers, name=name)
+    _result = _op.outputs[:]
+    _inputs_flat = _op.inputs
+    _attrs = None
+    _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result,
+                             name)
+    _result, = _result
+    return _result
+
+  else:
+    try:
+      _result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
+          _ctx._context_handle, _ctx._eager_context.device_name, 'IOWriteKafka',
+          name, _ctx._post_execution_callbacks, message, topic, servers)
+      return _result
+    except _core._FallbackException:
+      return io_write_kafka_eager_fallback(
+          message, topic, servers, name=name, ctx=_ctx)
+    except _core._NotOkStatusException as e:
+      if name is not None:
+        message = e.message + ' name: ' + name
+      else:
+        message = e.message
+      _six.raise_from(_core._status_to_exception(e.code, message), None)
+
+
+def io_write_kafka_eager_fallback(message, topic, servers, name=None, ctx=None):
+  """This is the slowpath function for Eager mode.
+
+  This is for function io_write_kafka
+  """
+  _ctx = ctx if ctx else _context.context()
+  message = _ops.convert_to_tensor(message, _dtypes.string)
+  topic = _ops.convert_to_tensor(topic, _dtypes.string)
+  servers = _ops.convert_to_tensor(servers, _dtypes.string)
+  _inputs_flat = [message, topic, servers]
+  _attrs = None
+  _result = _execute.execute(
+      b'IOWriteKafka',
+      1,
+      inputs=_inputs_flat,
+      attrs=_attrs,
+      ctx=_ctx,
+      name=name)
+  _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result, name)
+  _result, = _result
+  return _result

easy_rec/python/ops/gen_str_avx_op.py

@@ -0,0 +1,28 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+import os
+
+import tensorflow as tf
+from tensorflow.python.ops import string_ops
+
+import easy_rec
+from easy_rec.python.utils import constant
+
+try:
+  str_avx_op_path = os.path.join(easy_rec.ops_dir, 'libstr_avx_op.so')
+  str_avx_op = tf.load_op_library(str_avx_op_path)
+  logging.info('load avx string_split op from %s succeed' % str_avx_op_path)
+except Exception as ex:
+  logging.warning('load avx string_split op failed: %s' % str(ex))
+  str_avx_op = None
+
+
+def str_split_by_chr(input_str, sep, skip_empty):
+  if constant.has_avx_str_split() and str_avx_op is not None:
+    assert len(sep) == 1, \
+        'invalid data_config.separator(%s) len(%d) != 1' % (
+        sep, len(sep))
+    return str_avx_op.avx512_string_split(input_str, sep, skip_empty=skip_empty)
+  else:
+    return string_ops.string_split(input_str, sep, skip_empty=skip_empty)

easy_rec/python/ops/incr_record.py

@@ -0,0 +1,30 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+import os
+
+import tensorflow as tf
+
+import easy_rec
+
+try:
+  op_path = os.path.join(easy_rec.ops_dir, 'incr_record.so')
+  op = tf.load_op_library(op_path)
+  get_sparse_indices = op.get_sparse_indices
+  set_sparse_indices = op.set_sparse_indices
+  if 'kv_resource_incr_gather' in dir(op):
+    kv_resource_incr_gather = getattr(op, 'kv_resource_incr_gather')
+  else:
+    kv_resource_incr_gather = None
+except ImportError as ex:
+  get_sparse_indices = None
+  set_sparse_indices = None
+  kv_resource_incr_gather = None
+  logging.warning('failed to import gen_io_ops.collect_sparse_indices: %s' %
+                  str(ex))
+except Exception as ex:
+  get_sparse_indices = None
+  set_sparse_indices = None
+  kv_resource_incr_gather = None
+  logging.warning('failed to import gen_io_ops.collect_sparse_indices: %s' %
+                  str(ex))