easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/input/odps_input.py

@@ -0,0 +1,101 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import tensorflow as tf
+
+from easy_rec.python.input.input import Input
+from easy_rec.python.utils import odps_util
+
+try:
+  import pai
+except Exception:
+  pass
+
+
+class OdpsInput(Input):
+
+  def __init__(self,
+               data_config,
+               feature_config,
+               input_path,
+               task_index=0,
+               task_num=1,
+               check_mode=False,
+               pipeline_config=None):
+    super(OdpsInput,
+          self).__init__(data_config, feature_config, input_path, task_index,
+                         task_num, check_mode, pipeline_config)
+
+  def _build(self, mode, params):
+    # check data_config are consistent with odps tables
+    odps_util.check_input_field_and_types(self._data_config)
+
+    selected_cols = ','.join(self._input_fields)
+    if self._data_config.chief_redundant and \
+        mode == tf.estimator.ModeKeys.TRAIN:
+      reader = tf.TableRecordReader(
+          csv_delimiter=self._data_config.separator,
+          selected_cols=selected_cols,
+          slice_count=max(self._task_num - 1, 1),
+          slice_id=max(self._task_index - 1, 0))
+    else:
+      reader = tf.TableRecordReader(
+          csv_delimiter=self._data_config.separator,
+          selected_cols=selected_cols,
+          slice_count=self._task_num,
+          slice_id=self._task_index)
+
+    if type(self._input_path) != list:
+      self._input_path = self._input_path.split(',')
+    assert len(
+        self._input_path) > 0, 'match no files with %s' % self._input_path
+
+    if mode == tf.estimator.ModeKeys.TRAIN:
+      if self._data_config.pai_worker_queue:
+        work_queue = pai.data.WorkQueue(
+            self._input_path,
+            num_epochs=self.num_epochs,
+            shuffle=self._data_config.shuffle,
+            num_slices=self._data_config.pai_worker_slice_num * self._task_num)
+        work_queue.add_summary()
+        file_queue = work_queue.input_producer()
+        reader = tf.TableRecordReader()
+      else:
+        file_queue = tf.train.string_input_producer(
+            self._input_path,
+            num_epochs=self.num_epochs,
+            capacity=1000,
+            shuffle=self._data_config.shuffle)
+    else:
+      file_queue = tf.train.string_input_producer(
+          self._input_path, num_epochs=1, capacity=1000, shuffle=False)
+    key, value = reader.read_up_to(file_queue, self._batch_size)
+
+    record_defaults = [
+        self.get_type_defaults(t, v)
+        for t, v in zip(self._input_field_types, self._input_field_defaults)
+    ]
+    fields = tf.decode_csv(
+        value,
+        record_defaults=record_defaults,
+        field_delim=self._data_config.separator,
+        name='decode_csv')
+
+    inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids}
+    for x in self._label_fids:
+      inputs[self._input_fields[x]] = fields[x]
+
+    fields = self._preprocess(inputs)
+
+    features = self._get_features(fields)
+    # import pai
+    if mode != tf.estimator.ModeKeys.PREDICT:
+      labels = self._get_labels(fields)
+      # features, labels = pai.data.prefetch(features=(features, labels),
+      #                         capacity=self._prefetch_size, num_threads=2,
+      #         closed_exception_types=(tuple([tf.errors.InternalError])))
+      return features, labels
+    else:
+      # features = pai.data.prefetch(features=(features,),
+      #                         capacity=self._prefetch_size, num_threads=2,
+      #         closed_exception_types=(tuple([tf.errors.InternalError])))
+      return features

easy_rec/python/input/odps_input_v2.py

@@ -0,0 +1,110 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import tensorflow as tf
+
+from easy_rec.python.input.input import Input
+from easy_rec.python.utils import odps_util
+
+try:
+  import pai
+except Exception:
+  pass
+
+
+class OdpsInputV2(Input):
+
+  def __init__(self,
+               data_config,
+               feature_config,
+               input_path,
+               task_index=0,
+               task_num=1,
+               check_mode=False,
+               pipeline_config=None):
+    super(OdpsInputV2,
+          self).__init__(data_config, feature_config, input_path, task_index,
+                         task_num, check_mode, pipeline_config)
+
+  def _parse_table(self, *fields):
+    fields = list(fields)
+    inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids}
+    for x in self._label_fids:
+      inputs[self._input_fields[x]] = fields[x]
+    return inputs
+
+  def _build(self, mode, params):
+    if type(self._input_path) != list:
+      self._input_path = self._input_path.split(',')
+    assert len(
+        self._input_path) > 0, 'match no files with %s' % self._input_path
+    # check data_config are consistent with odps tables
+    odps_util.check_input_field_and_types(self._data_config)
+
+    selected_cols = ','.join(self._input_fields)
+    record_defaults = [
+        self.get_type_defaults(x, v)
+        for x, v in zip(self._input_field_types, self._input_field_defaults)
+    ]
+
+    if self._data_config.pai_worker_queue and \
+        mode == tf.estimator.ModeKeys.TRAIN:
+      logging.info('pai_worker_slice_num = %d' %
+                   self._data_config.pai_worker_slice_num)
+      work_queue = pai.data.WorkQueue(
+          self._input_path,
+          num_epochs=self.num_epochs,
+          shuffle=self._data_config.shuffle,
+          num_slices=self._data_config.pai_worker_slice_num * self._task_num)
+      que_paths = work_queue.input_dataset()
+      dataset = tf.data.TableRecordDataset(
+          que_paths,
+          record_defaults=record_defaults,
+          selected_cols=selected_cols)
+    elif self._data_config.chief_redundant and \
+        mode == tf.estimator.ModeKeys.TRAIN:
+      dataset = tf.data.TableRecordDataset(
+          self._input_path,
+          record_defaults=record_defaults,
+          selected_cols=selected_cols,
+          slice_id=max(self._task_index - 1, 0),
+          slice_count=max(self._task_num - 1, 1))
+    else:
+      dataset = tf.data.TableRecordDataset(
+          self._input_path,
+          record_defaults=record_defaults,
+          selected_cols=selected_cols,
+          slice_id=self._task_index,
+          slice_count=self._task_num)
+
+    if mode == tf.estimator.ModeKeys.TRAIN:
+      if self._data_config.shuffle:
+        dataset = dataset.shuffle(
+            self._data_config.shuffle_buffer_size,
+            seed=2020,
+            reshuffle_each_iteration=True)
+      dataset = dataset.repeat(self.num_epochs)
+    else:
+      dataset = dataset.repeat(1)
+
+    dataset = dataset.batch(batch_size=self._data_config.batch_size)
+
+    dataset = dataset.map(
+        self._parse_table,
+        num_parallel_calls=self._data_config.num_parallel_calls)
+
+    # preprocess is necessary to transform data
+    # so that they could be feed into FeatureColumns
+    dataset = dataset.map(
+        map_func=self._preprocess,
+        num_parallel_calls=self._data_config.num_parallel_calls)
+
+    dataset = dataset.prefetch(buffer_size=self._prefetch_size)
+
+    if mode != tf.estimator.ModeKeys.PREDICT:
+      dataset = dataset.map(lambda x:
+                            (self._get_features(x), self._get_labels(x)))
+    else:
+      dataset = dataset.map(lambda x: (self._get_features(x)))
+    return dataset

easy_rec/python/input/odps_input_v3.py

@@ -0,0 +1,132 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import logging
+import sys
+
+import tensorflow as tf
+
+from easy_rec.python.input.input import Input
+from easy_rec.python.utils import odps_util
+from easy_rec.python.utils.tf_utils import get_tf_type
+
+try:
+  import common_io
+except Exception:
+  common_io = None
+
+
+class OdpsInputV3(Input):
+  """Common IO based interface, could run at local or on data science."""
+
+  def __init__(self,
+               data_config,
+               feature_config,
+               input_path,
+               task_index=0,
+               task_num=1,
+               check_mode=False,
+               pipeline_config=None):
+    super(OdpsInputV3,
+          self).__init__(data_config, feature_config, input_path, task_index,
+                         task_num, check_mode, pipeline_config)
+    self._num_epoch = 0
+    if common_io is None:
+      logging.error('''
+        please install common_io pip install
+        https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl'''
+                    )
+      sys.exit(1)
+
+  def _parse_table(self, *fields):
+    fields = list(fields)
+    inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids}
+    for x in self._label_fids:
+      inputs[self._input_fields[x]] = fields[x]
+    return inputs
+
+  def _odps_read(self):
+    logging.info('start epoch[%d]' % self._num_epoch)
+    self._num_epoch += 1
+    if type(self._input_path) != list:
+      self._input_path = self._input_path.split(',')
+    assert len(
+        self._input_path) > 0, 'match no files with %s' % self._input_path
+
+    # check data_config are consistent with odps tables
+    odps_util.check_input_field_and_types(self._data_config)
+
+    record_defaults = [
+        self.get_type_defaults(x, v)
+        for x, v in zip(self._input_field_types, self._input_field_defaults)
+    ]
+
+    selected_cols = ','.join(self._input_fields)
+    for table_path in self._input_path:
+      reader = common_io.table.TableReader(
+          table_path,
+          selected_cols=selected_cols,
+          slice_id=self._task_index,
+          slice_count=self._task_num)
+      total_records_num = reader.get_row_count()
+      batch_num = int(total_records_num / self._data_config.batch_size)
+      res_num = total_records_num - batch_num * self._data_config.batch_size
+      batch_defaults = [
+          [x] * self._data_config.batch_size for x in record_defaults
+      ]
+      for batch_id in range(batch_num):
+        batch_data_np = [x.copy() for x in batch_defaults]
+        for row_id, one_data in enumerate(
+            reader.read(self._data_config.batch_size)):
+          for col_id in range(len(record_defaults)):
+            if one_data[col_id] not in ['', 'NULL', None]:
+              batch_data_np[col_id][row_id] = one_data[col_id]
+        yield tuple(batch_data_np)
+      if res_num > 0:
+        batch_data_np = [x[:res_num] for x in batch_defaults]
+        for row_id, one_data in enumerate(reader.read(res_num)):
+          for col_id in range(len(record_defaults)):
+            if one_data[col_id] not in ['', 'NULL', None]:
+              batch_data_np[col_id][row_id] = one_data[col_id]
+        yield tuple(batch_data_np)
+      reader.close()
+    logging.info('finish epoch[%d]' % self._num_epoch)
+
+  def _build(self, mode, params):
+    # get input type
+    list_type = [get_tf_type(x) for x in self._input_field_types]
+    list_type = tuple(list_type)
+    list_shapes = [tf.TensorShape([None]) for x in range(0, len(list_type))]
+    list_shapes = tuple(list_shapes)
+
+    # read odps tables
+    dataset = tf.data.Dataset.from_generator(
+        self._odps_read, output_types=list_type, output_shapes=list_shapes)
+
+    if mode == tf.estimator.ModeKeys.TRAIN:
+      dataset = dataset.shuffle(
+          self._data_config.shuffle_buffer_size,
+          seed=2020,
+          reshuffle_each_iteration=True)
+      dataset = dataset.repeat(self.num_epochs)
+    else:
+      dataset = dataset.repeat(1)
+
+    dataset = dataset.map(
+        self._parse_table,
+        num_parallel_calls=self._data_config.num_parallel_calls)
+
+    # preprocess is necessary to transform data
+    # so that they could be feed into FeatureColumns
+    dataset = dataset.map(
+        map_func=self._preprocess,
+        num_parallel_calls=self._data_config.num_parallel_calls)
+
+    dataset = dataset.prefetch(buffer_size=self._prefetch_size)
+
+    if mode != tf.estimator.ModeKeys.PREDICT:
+      dataset = dataset.map(lambda x:
+                            (self._get_features(x), self._get_labels(x)))
+    else:
+      dataset = dataset.map(lambda x: (self._get_features(x)))
+    return dataset

easy_rec/python/input/odps_rtp_input.py

@@ -0,0 +1,187 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import tensorflow as tf
+
+from easy_rec.python.input.input import Input
+from easy_rec.python.ops.gen_str_avx_op import str_split_by_chr
+from easy_rec.python.utils.check_utils import check_split
+from easy_rec.python.utils.input_utils import string_to_number
+
+try:
+  import pai
+except Exception:
+  pass
+
+
+class OdpsRTPInput(Input):
+  """RTPInput for parsing rtp fg new input format on odps.
+
+  Our new format(csv in table) of rtp output:
+     label0, item_id, ..., user_id, features
+  For the feature column, features are separated by ,
+     multiple values of one feature are separated by , such as:
+     ...20beautysmartParis...
+  The features column and labels are specified by data_config.selected_cols,
+     columns are selected by names in the table
+     such as: clk,features, the last selected column is features, the first
+     selected columns are labels
+  """
+
+  def __init__(self,
+               data_config,
+               feature_config,
+               input_path,
+               task_index=0,
+               task_num=1,
+               check_mode=False,
+               pipeline_config=None):
+    super(OdpsRTPInput,
+          self).__init__(data_config, feature_config, input_path, task_index,
+                         task_num, check_mode, pipeline_config)
+    logging.info('input_fields: %s label_fields: %s' %
+                 (','.join(self._input_fields), ','.join(self._label_fields)))
+
+  def _parse_table(self, *fields):
+    fields = list(fields)
+    labels = fields[:-1]
+
+    selected_cols = self._data_config.selected_cols \
+        if self._data_config.selected_cols else None
+    non_feature_cols = self._label_fields
+    if selected_cols:
+      cols = [c.strip() for c in selected_cols.split(',')]
+      non_feature_cols = cols[:-1]
+    # only for features, labels and sample_weight excluded
+    record_types = [
+        t for x, t in zip(self._input_fields, self._input_field_types)
+        if x not in non_feature_cols
+    ]
+    record_defaults = [
+        self.get_type_defaults(t, v)
+        for x, t, v in zip(self._input_fields, self._input_field_types,
+                           self._input_field_defaults)
+        if x not in non_feature_cols
+    ]
+
+    feature_num = len(record_types)
+    # assume that the last field is the generated feature column
+    print('field_delim = %s, feature_num = %d' %
+          (self._data_config.separator, feature_num))
+    logging.info('field_delim = %s, input_field_name = %d' %
+                 (self._data_config.separator, len(record_types)))
+
+    check_list = [
+        tf.py_func(
+            check_split,
+            [fields[-1], self._data_config.separator,
+             len(record_types)],
+            Tout=tf.bool)
+    ] if self._check_mode else []
+    with tf.control_dependencies(check_list):
+      fields = str_split_by_chr(
+          fields[-1], self._data_config.separator, skip_empty=False)
+    tmp_fields = tf.reshape(fields.values, [-1, feature_num])
+    fields = labels[len(self._label_fields):]
+    for i in range(feature_num):
+      field = string_to_number(tmp_fields[:, i], record_types[i],
+                               record_defaults[i], i)
+      fields.append(field)
+
+    field_keys = [x for x in self._input_fields if x not in self._label_fields]
+    effective_fids = [field_keys.index(x) for x in self._effective_fields]
+    inputs = {field_keys[x]: fields[x] for x in effective_fids}
+
+    for x in range(len(self._label_fields)):
+      inputs[self._label_fields[x]] = labels[x]
+    print('effective field num = %d, input_num = %d' %
+          (len(fields), len(inputs)))
+    return inputs
+
+  def _build(self, mode, params):
+    if type(self._input_path) != list:
+      self._input_path = self._input_path.split(',')
+    assert len(
+        self._input_path) > 0, 'match no files with %s' % self._input_path
+
+    selected_cols = self._data_config.selected_cols \
+        if self._data_config.selected_cols else None
+    if selected_cols:
+      cols = [c.strip() for c in selected_cols.split(',')]
+      record_defaults = [
+          self.get_type_defaults(t, v)
+          for x, t, v in zip(self._input_fields, self._input_field_types,
+                             self._input_field_defaults)
+          if x in cols[:-1]
+      ]
+      print('selected_cols: %s; defaults num: %d' %
+            (','.join(cols), len(record_defaults)))
+    else:
+      record_defaults = [
+          self.get_type_defaults(t, v)
+          for x, t, v in zip(self._input_fields, self._input_field_types,
+                             self._input_field_defaults)
+          if x in self._label_fields
+      ]
+    # the actual features are in one single column
+    record_defaults.append(
+        self._data_config.separator.join([
+            str(self.get_type_defaults(t, v))
+            for x, t, v in zip(self._input_fields, self._input_field_types,
+                               self._input_field_defaults)
+            if x not in self._label_fields
+        ]))
+
+    if self._data_config.pai_worker_queue and \
+        mode == tf.estimator.ModeKeys.TRAIN:
+      logging.info('pai_worker_slice_num = %d' %
+                   self._data_config.pai_worker_slice_num)
+      work_queue = pai.data.WorkQueue(
+          self._input_path,
+          num_epochs=self.num_epochs,
+          shuffle=self._data_config.shuffle,
+          num_slices=self._data_config.pai_worker_slice_num * self._task_num)
+      que_paths = work_queue.input_dataset()
+      dataset = tf.data.TableRecordDataset(
+          que_paths,
+          record_defaults=record_defaults,
+          selected_cols=selected_cols)
+    else:
+      dataset = tf.data.TableRecordDataset(
+          self._input_path,
+          record_defaults=record_defaults,
+          selected_cols=selected_cols,
+          slice_id=self._task_index,
+          slice_count=self._task_num)
+
+    if mode == tf.estimator.ModeKeys.TRAIN:
+      if self._data_config.shuffle:
+        dataset = dataset.shuffle(
+            self._data_config.shuffle_buffer_size,
+            seed=2020,
+            reshuffle_each_iteration=True)
+      dataset = dataset.repeat(self.num_epochs)
+    else:
+      dataset = dataset.repeat(1)
+
+    dataset = dataset.batch(batch_size=self._data_config.batch_size)
+
+    dataset = dataset.map(
+        self._parse_table,
+        num_parallel_calls=self._data_config.num_parallel_calls)
+
+    # preprocess is necessary to transform data
+    # so that they could be feed into FeatureColumns
+    dataset = dataset.map(
+        map_func=self._preprocess,
+        num_parallel_calls=self._data_config.num_parallel_calls)
+
+    dataset = dataset.prefetch(buffer_size=self._prefetch_size)
+
+    if mode != tf.estimator.ModeKeys.PREDICT:
+      dataset = dataset.map(lambda x:
+                            (self._get_features(x), self._get_labels(x)))
+    else:
+      dataset = dataset.map(lambda x: (self._get_features(x)))
+    return dataset

easy_rec/python/input/odps_rtp_input_v2.py

@@ -0,0 +1,104 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import json
+import logging
+
+import tensorflow as tf
+
+from easy_rec.python.input.odps_rtp_input import OdpsRTPInput
+
+if tf.__version__.startswith('1.'):
+  from tensorflow.python.platform import gfile
+else:
+  import tensorflow.io.gfile as gfile
+try:
+  import pai
+  import rtp_fg
+except Exception:
+  pai = None
+  rtp_fg = None
+
+
+class OdpsRTPInputV2(OdpsRTPInput):
+  """RTPInput for parsing rtp fg new input format on odps.
+
+  Our new format(csv in table) of rtp output:
+     label0, item_id, ..., user_id, features
+  Where features is in default RTP-tensorflow format.
+  The features column and labels are specified by data_config.selected_cols,
+     columns are selected by names in the table
+     such as: clk,features, the last selected column is features, the first
+     selected columns are labels
+  """
+
+  def __init__(self,
+               data_config,
+               feature_config,
+               input_path,
+               task_index=0,
+               task_num=1,
+               check_mode=False,
+               fg_json_path=None,
+               pipeline_config=None):
+    super(OdpsRTPInputV2,
+          self).__init__(data_config, feature_config, input_path, task_index,
+                         task_num, check_mode, pipeline_config)
+    if fg_json_path.startswith('!'):
+      fg_json_path = fg_json_path[1:]
+    self._fg_config_path = fg_json_path
+    logging.info('fg config path: {}'.format(self._fg_config_path))
+    if self._fg_config_path is None:
+      raise ValueError('fg_json_path is not set')
+    with gfile.GFile(self._fg_config_path, 'r') as f:
+      self._fg_config = json.load(f)
+
+  def _parse_table(self, *fields):
+    self.check_rtp()
+
+    fields = list(fields)
+    labels = fields[:-1]
+
+    # assume that the last field is the generated feature column
+    features = rtp_fg.parse_genreated_fg(self._fg_config, fields[-1])
+
+    field_keys = [x for x in self._input_fields if x not in self._label_fields]
+    for feature_key in features:
+      if feature_key not in field_keys or feature_key not in self._effective_fields:
+        del features[feature_key]
+    inputs = {x: features[x] for x in features.keys()}
+
+    for x in range(len(self._label_fields)):
+      inputs[self._label_fields[x]] = labels[x]
+    return inputs
+
+  def create_placeholders(self, *args, **kwargs):
+    """Create serving placeholders with rtp_fg."""
+    self.check_rtp()
+    self._mode = tf.estimator.ModeKeys.PREDICT
+    inputs_placeholder = tf.placeholder(tf.string, [None], name='features')
+    print('[OdpsRTPInputV2] building placeholders.')
+    print('[OdpsRTPInputV2] fg_config: {}'.format(self._fg_config))
+    features = rtp_fg.parse_genreated_fg(self._fg_config, inputs_placeholder)
+    print('[OdpsRTPInputV2] built features: {}'.format(features.keys()))
+    features = self._preprocess(features)
+    print('[OdpsRTPInputV2] processed features: {}'.format(features.keys()))
+    return {'features': inputs_placeholder}, features['feature']
+
+  def create_multi_placeholders(self, *args, **kwargs):
+    """Create serving multi-placeholders with rtp_fg."""
+    raise NotImplementedError(
+        'create_multi_placeholders is not supported for OdpsRTPInputV2')
+
+  def check_rtp(self):
+    if rtp_fg is None:
+      raise NotImplementedError(
+          'OdpsRTPInputV2 cannot run without rtp_fg, which is not installed')
+
+  def _pre_build(self, mode, params):
+    try:
+      # Prevent TF from replacing the shape tensor to a constant tensor. This will
+      # cause the batch size being fixed. And RTP will be not able to recognize
+      # the input shape.
+      tf.get_default_graph().set_shape_optimize(False)
+    except AttributeError as e:
+      logging.warning('failed to disable shape optimization:', e)