easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/input/criteo_input.py

@@ -0,0 +1,107 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import tensorflow as tf
+
+from easy_rec.python.input.criteo_binary_reader import BinaryDataset
+from easy_rec.python.input.input import Input
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class CriteoInput(Input):
+
+  def __init__(self,
+               data_config,
+               feature_config,
+               input_path,
+               task_index=0,
+               task_num=1,
+               check_mode=False,
+               pipeline_config=None):
+    super(CriteoInput,
+          self).__init__(data_config, feature_config, input_path, task_index,
+                         task_num, check_mode, pipeline_config)
+    all_label_paths = []
+    all_dense_paths = []
+    all_category_paths = []
+
+    if input_path is not None:
+      assert len(input_path.label_path) == len(input_path.dense_path) and \
+          len(input_path.label_path) == len(input_path.category_path), \
+          'label_path_num(%d), dense_path_num(%d), category_path_num(%d) must be the same' % \
+          (len(input_path.label_path), len(input_path.dense_path), len(input_path.category_path))
+
+      for label_path, dense_path, category_path in zip(
+          input_path.label_path, input_path.dense_path,
+          input_path.category_path):
+        label_paths = tf.gfile.Glob(input_path.label_path)
+        dense_paths = tf.gfile.Glob(input_path.dense_path)
+        category_paths = tf.gfile.Glob(input_path.category_path)
+        assert len(label_paths) == len(dense_paths) and len(label_paths) == \
+            len(category_paths), 'label_path(%s) dense_path(%s) category_path(%s) ' + \
+            'matched different number of files(%d %d %d)' % (
+            len(label_paths), len(dense_paths), len(category_paths))
+        label_paths.sort()
+        dense_paths.sort()
+        category_paths.sort()
+        all_label_paths.extend(label_paths)
+        all_dense_paths.extend(dense_paths)
+        all_category_paths.extend(category_paths)
+
+      logging.info('total number of input parts: %s' % len(all_label_paths))
+
+      self._binary_reader = BinaryDataset(
+          all_label_paths,
+          all_dense_paths,
+          all_category_paths,
+          self._batch_size,
+          prefetch=self._prefetch_size,
+          global_rank=self._task_index,
+          global_size=self._task_num)
+    else:
+      self._binary_reader = None
+
+  def _sample_generator(self):
+    num_epoch = 0
+    while not self.should_stop(num_epoch):
+      logging.info('start epoch: %d' % num_epoch)
+      for dense, category, labels in self._binary_reader:
+        yield dense, category, labels.reshape([-1])
+      logging.info('finish epoch: %d' % num_epoch)
+      num_epoch += 1
+
+  def _to_fea_dict(self, dense, category, labels):
+    field_dict = {}
+    for fid in range(1, 14):
+      fea_name = 'f%d' % fid
+      field_dict[fea_name] = dense[:, fid - 1]
+
+    for cid in range(1, 27):
+      fea_name = 'c%d' % cid
+      field_dict[fea_name] = category[:, cid - 1]
+    field_dict['label'] = labels
+    return field_dict
+
+  def _build(self, mode, params):
+    dataset = tf.data.Dataset.from_generator(
+        self._sample_generator,
+        output_types=(tf.float32, tf.int32, tf.int32),
+        output_shapes=(tf.TensorShape([None, 13]), tf.TensorShape([None, 26]),
+                       tf.TensorShape([None])))
+    num_parallel_calls = self._data_config.num_parallel_calls
+    dataset = dataset.map(
+        self._to_fea_dict, num_parallel_calls=num_parallel_calls)
+    dataset = dataset.prefetch(buffer_size=self._prefetch_size)
+    dataset = dataset.map(
+        map_func=self._preprocess, num_parallel_calls=num_parallel_calls)
+    dataset = dataset.prefetch(buffer_size=self._prefetch_size)
+
+    if mode != tf.estimator.ModeKeys.PREDICT:
+      dataset = dataset.map(lambda x:
+                            (self._get_features(x), self._get_labels(x)))
+    else:
+      dataset = dataset.map(lambda x: (self._get_features(x)))
+    return dataset

easy_rec/python/input/csv_input.py

@@ -0,0 +1,175 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import tensorflow as tf
+
+from easy_rec.python.input.input import Input
+from easy_rec.python.utils.check_utils import check_split
+
+if tf.__version__ >= '2.0':
+  ignore_errors = tf.data.experimental.ignore_errors()
+  tf = tf.compat.v1
+else:
+  ignore_errors = tf.contrib.data.ignore_errors()
+
+
+class CSVInput(Input):
+
+  def __init__(self,
+               data_config,
+               feature_config,
+               input_path,
+               task_index=0,
+               task_num=1,
+               check_mode=False,
+               pipeline_config=None):
+    super(CSVInput,
+          self).__init__(data_config, feature_config, input_path, task_index,
+                         task_num, check_mode, pipeline_config)
+    self._with_header = data_config.with_header
+    self._field_names = None
+
+  def _parse_csv(self, line):
+    record_defaults = [
+        self.get_type_defaults(t, v)
+        for t, v in zip(self._input_field_types, self._input_field_defaults)
+    ]
+
+    if self._field_names:
+      # decode by csv header
+      record_defaults = []
+      for field_name in self._field_names:
+        if field_name in self._input_fields:
+          tid = self._input_fields.index(field_name)
+          record_defaults.append(
+              self.get_type_defaults(self._input_field_types[tid],
+                                     self._input_field_defaults[tid]))
+        else:
+          record_defaults.append('')
+
+    check_list = [
+        tf.py_func(
+            check_split, [
+                line, self._data_config.separator,
+                len(record_defaults), self._check_mode
+            ],
+            Tout=tf.bool)
+    ] if self._check_mode else []
+    with tf.control_dependencies(check_list):
+      fields = tf.decode_csv(
+          line,
+          field_delim=self._data_config.separator,
+          record_defaults=record_defaults,
+          name='decode_csv')
+      if self._field_names is not None:
+        fields = [
+            fields[self._field_names.index(x)] for x in self._input_fields
+        ]
+
+    # filter only valid fields
+    inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids}
+
+    # filter only valid labels
+    for x in self._label_fids:
+      inputs[self._input_fields[x]] = fields[x]
+    return inputs
+
+  def _build(self, mode, params):
+    if type(self._input_path) != list:
+      self._input_path = self._input_path.split(',')
+    file_paths = []
+    for path in self._input_path:
+      for x in tf.gfile.Glob(path):
+        if not x.endswith('_SUCCESS'):
+          file_paths.append(x)
+    assert len(file_paths) > 0, 'match no files with %s' % self._input_path
+
+    assert not file_paths[0].endswith(
+        '.tar.gz'), 'could only support .csv or .gz(not .tar.gz) files.'
+
+    compression_type = 'GZIP' if file_paths[0].endswith('.gz') else ''
+    if compression_type:
+      logging.info('compression_type = %s' % compression_type)
+
+    if self._with_header:
+      with tf.gfile.GFile(file_paths[0], 'r') as fin:
+        for line_str in fin:
+          line_str = line_str.strip()
+          self._field_names = line_str.split(self._data_config.separator)
+          break
+        print('field_names: %s' % ','.join(self._field_names))
+
+    num_parallel_calls = self._data_config.num_parallel_calls
+    if mode == tf.estimator.ModeKeys.TRAIN:
+      logging.info('train files[%d]: %s' %
+                   (len(file_paths), ','.join(file_paths)))
+      dataset = tf.data.Dataset.from_tensor_slices(file_paths)
+
+      if self._data_config.file_shard:
+        dataset = self._safe_shard(dataset)
+
+      if self._data_config.shuffle:
+        # shuffle input files
+        dataset = dataset.shuffle(len(file_paths))
+
+      # too many readers read the same file will cause performance issues
+      # as the same data will be read multiple times
+      parallel_num = min(num_parallel_calls, len(file_paths))
+      dataset = dataset.interleave(
+          lambda x: tf.data.TextLineDataset(
+              x, compression_type=compression_type).skip(
+                  int(self._with_header)),
+          cycle_length=parallel_num,
+          num_parallel_calls=parallel_num)
+
+      if not self._data_config.file_shard:
+        dataset = self._safe_shard(dataset)
+
+      if self._data_config.shuffle:
+        dataset = dataset.shuffle(
+            self._data_config.shuffle_buffer_size,
+            seed=2020,
+            reshuffle_each_iteration=True)
+      dataset = dataset.repeat(self.num_epochs)
+    elif self._task_num > 1:  # For distribute evaluate
+      dataset = tf.data.Dataset.from_tensor_slices(file_paths)
+      parallel_num = min(num_parallel_calls, len(file_paths))
+      dataset = dataset.interleave(
+          lambda x: tf.data.TextLineDataset(
+              x, compression_type=compression_type).skip(
+                  int(self._with_header)),
+          cycle_length=parallel_num,
+          num_parallel_calls=parallel_num)
+      dataset = self._safe_shard(dataset)
+      dataset = dataset.repeat(1)
+    else:
+      logging.info('eval files[%d]: %s' %
+                   (len(file_paths), ','.join(file_paths)))
+      dataset = tf.data.Dataset.from_tensor_slices(file_paths)
+      parallel_num = min(num_parallel_calls, len(file_paths))
+      dataset = dataset.interleave(
+          lambda x: tf.data.TextLineDataset(
+              x, compression_type=compression_type).skip(
+                  int(self._with_header)),
+          cycle_length=parallel_num,
+          num_parallel_calls=parallel_num)
+      dataset = dataset.repeat(1)
+
+    dataset = dataset.batch(self._data_config.batch_size)
+    dataset = dataset.map(
+        self._parse_csv, num_parallel_calls=num_parallel_calls)
+    if self._data_config.ignore_error:
+      dataset = dataset.apply(ignore_errors)
+    dataset = dataset.prefetch(buffer_size=self._prefetch_size)
+    dataset = dataset.map(
+        map_func=self._preprocess, num_parallel_calls=num_parallel_calls)
+
+    dataset = dataset.prefetch(buffer_size=self._prefetch_size)
+
+    if mode != tf.estimator.ModeKeys.PREDICT:
+      dataset = dataset.map(lambda x:
+                            (self._get_features(x), self._get_labels(x)))
+    else:
+      dataset = dataset.map(lambda x: (self._get_features(x)))
+    return dataset

easy_rec/python/input/csv_input_ex.py

@@ -0,0 +1,72 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import numpy as np
+import tensorflow as tf
+
+from easy_rec.python.input.csv_input import CSVInput
+from easy_rec.python.ops.gen_str_avx_op import str_split_by_chr
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+class CSVInputEx(CSVInput):
+
+  def __init__(self,
+               data_config,
+               feature_config,
+               input_path,
+               task_index=0,
+               task_num=1,
+               check_mode=False,
+               pipeline_config=None):
+    super(CSVInputEx,
+          self).__init__(data_config, feature_config, input_path, task_index,
+                         task_num, check_mode, pipeline_config)
+
+  def _parse_csv(self, line):
+    record_defaults = [
+        self.get_type_defaults(t, v)
+        for t, v in zip(self._input_field_types, self._input_field_defaults)
+    ]
+
+    def _check_data(line):
+      sep = self._data_config.separator
+      if type(sep) != type(str):
+        sep = sep.encode('utf-8')
+      field_num = len(line[0].split(sep))
+      assert field_num == len(record_defaults), \
+          'sep[%s] maybe invalid: field_num=%d, required_num=%d' % \
+          (sep, field_num, len(record_defaults))
+      return True
+
+    fields = str_split_by_chr(
+        line, self._data_config.separator, skip_empty=False)
+    tmp_fields = tf.reshape(fields.values, [-1, len(record_defaults)])
+    fields = []
+    for i in range(len(record_defaults)):
+      if type(record_defaults[i]) == int:
+        fields.append(
+            tf.string_to_number(
+                tmp_fields[:, i], tf.int64, name='field_as_int_%d' % i))
+      elif type(record_defaults[i]) in [float, np.float32, np.float64]:
+        fields.append(
+            tf.string_to_number(
+                tmp_fields[:, i], tf.float32, name='field_as_flt_%d' % i))
+      elif type(record_defaults[i]) in [str, type(u''), bytes]:
+        fields.append(tmp_fields[:, i])
+      elif type(record_defaults[i]) == bool:
+        fields.append(
+            tf.logical_or(
+                tf.equal(tmp_fields[:, i], 'True'),
+                tf.equal(tmp_fields[:, i], 'true')))
+      else:
+        assert 'invalid types: %s' % str(type(record_defaults[i]))
+
+    keep_ids = [
+        self._input_fields.index(x)
+        for x in self._label_fields + self._effective_fields
+    ]
+    inputs = {self._input_fields[x]: fields[x] for x in keep_ids}
+
+    return inputs

easy_rec/python/input/csv_input_v2.py

@@ -0,0 +1,68 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import tensorflow as tf
+
+from easy_rec.python.input.input import Input
+
+
+class CSVInputV2(Input):
+
+  def __init__(self,
+               data_config,
+               feature_config,
+               input_path,
+               task_index=0,
+               task_num=1,
+               check_mode=False,
+               pipeline_config=None):
+    super(CSVInputV2,
+          self).__init__(data_config, feature_config, input_path, task_index,
+                         task_num, check_mode, pipeline_config)
+
+  def _build(self, mode, params):
+    if type(self._input_path) != list:
+      self._input_path = self._input_path.split(',')
+    assert len(
+        self._input_path) > 0, 'match no files with %s' % self._input_path
+
+    if self._input_path[0].startswith('hdfs://'):
+      # support hdfs input
+      dataset = tf.data.TextLineDataset(self._input_path)
+    else:
+      num_epochs = self.num_epochs if mode == tf.estimator.ModeKeys.TRAIN else 1
+      is_train = (mode == tf.estimator.ModeKeys.TRAIN)
+      record_defaults = [
+          self.get_type_defaults(x, v)
+          for x, v in zip(self._input_field_types, self._input_field_defaults)
+      ]
+      dataset = tf.data.experimental.make_csv_dataset(
+          self._input_path,
+          self._data_config.batch_size,
+          column_names=self._input_fields,
+          field_delim=self._data_config.separator,
+          column_defaults=record_defaults,
+          header=False,
+          num_epochs=num_epochs,
+          shuffle=is_train and self._data_config.shuffle,
+          num_parallel_reads=8,
+          sloppy=is_train)
+
+    if mode == tf.estimator.ModeKeys.TRAIN:
+      if self._data_config.chief_redundant:
+        dataset = dataset.shard(
+            max(self._task_num - 1, 1), max(self._task_index - 1, 0))
+      else:
+        dataset = dataset.shard(self._task_num, self._task_index)
+    else:
+      dataset = dataset.repeat(1)
+
+    dataset = dataset.prefetch(buffer_size=self._prefetch_size)
+    dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=8)
+    dataset = dataset.prefetch(buffer_size=self._prefetch_size)
+
+    if mode != tf.estimator.ModeKeys.PREDICT:
+      dataset = dataset.map(lambda x:
+                            (self._get_features(x), self._get_labels(x)))
+    else:
+      dataset = dataset.map(lambda x: (self._get_features(x)))
+    return dataset