array-record 0.8.2__cp314-cp314-macosx_11_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- array_record/__init__.py +0 -0
- array_record/beam/__init__.py +20 -0
- array_record/beam/arrayrecordio.py +69 -0
- array_record/beam/demo.py +43 -0
- array_record/beam/dofns.py +57 -0
- array_record/beam/example.py +106 -0
- array_record/beam/options.py +26 -0
- array_record/beam/pipelines.py +179 -0
- array_record/beam/testdata.py +40 -0
- array_record/python/__init__.py +0 -0
- array_record/python/array_record_data_source.py +416 -0
- array_record/python/array_record_data_source_test.py +320 -0
- array_record/python/array_record_module.so +0 -0
- array_record/python/array_record_module_test.py +147 -0
- array_record/setup.py +54 -0
- array_record-0.8.2.dist-info/METADATA +34 -0
- array_record-0.8.2.dist-info/RECORD +20 -0
- array_record-0.8.2.dist-info/WHEEL +5 -0
- array_record-0.8.2.dist-info/licenses/LICENSE +201 -0
- array_record-0.8.2.dist-info/top_level.txt +1 -0
array_record/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Apache Beam module for array_record.
|
|
2
|
+
|
|
3
|
+
This module provides both core components and
|
|
4
|
+
helper functions to enable users to convert different file formats to AR.
|
|
5
|
+
|
|
6
|
+
To keep dependencies light, we'll import Beam on module usage so any errors
|
|
7
|
+
occur early.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import apache_beam as beam
|
|
11
|
+
|
|
12
|
+
# I'd really like a PEP8 compatible conditional import here with a more
|
|
13
|
+
# explicit error message. Example below:
|
|
14
|
+
|
|
15
|
+
# try:
|
|
16
|
+
# import apache_beam as beam
|
|
17
|
+
# except Exception as e:
|
|
18
|
+
# raise ImportError(
|
|
19
|
+
# ('Beam functionality requires extra dependencies. '
|
|
20
|
+
# 'Install apache-beam or run "pip install array_record[beam]".')) from e
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""An IO module for ArrayRecord.
|
|
2
|
+
|
|
3
|
+
CURRENTLY ONLY SINK IS IMPLEMENTED, AND IT DOESN'T WORK WITH NON-DISK WRITES
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from apache_beam import io
|
|
7
|
+
from apache_beam import transforms
|
|
8
|
+
from apache_beam.coders import coders
|
|
9
|
+
from apache_beam.io import filebasedsink
|
|
10
|
+
from apache_beam.io import filesystem
|
|
11
|
+
from array_record.python import array_record_module
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class _ArrayRecordSink(filebasedsink.FileBasedSink):
|
|
15
|
+
"""Sink Class for use in Arrayrecord PTransform."""
|
|
16
|
+
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
file_path_prefix,
|
|
20
|
+
file_name_suffix=None,
|
|
21
|
+
num_shards=0,
|
|
22
|
+
shard_name_template=None,
|
|
23
|
+
coder=coders.ToBytesCoder(),
|
|
24
|
+
compression_type=filesystem.CompressionTypes.AUTO):
|
|
25
|
+
|
|
26
|
+
super().__init__(
|
|
27
|
+
file_path_prefix,
|
|
28
|
+
file_name_suffix=file_name_suffix,
|
|
29
|
+
num_shards=num_shards,
|
|
30
|
+
shard_name_template=shard_name_template,
|
|
31
|
+
coder=coder,
|
|
32
|
+
mime_type='application/octet-stream',
|
|
33
|
+
compression_type=compression_type)
|
|
34
|
+
|
|
35
|
+
def open(self, temp_path):
|
|
36
|
+
array_writer = array_record_module.ArrayRecordWriter(
|
|
37
|
+
temp_path, 'group_size:1'
|
|
38
|
+
)
|
|
39
|
+
return array_writer
|
|
40
|
+
|
|
41
|
+
def close(self, file_handle):
|
|
42
|
+
file_handle.close()
|
|
43
|
+
|
|
44
|
+
def write_encoded_record(self, file_handle, value):
|
|
45
|
+
file_handle.write(value)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class WriteToArrayRecord(transforms.PTransform):
|
|
49
|
+
"""PTransform for a disk-based write to ArrayRecord."""
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
file_path_prefix,
|
|
54
|
+
file_name_suffix='',
|
|
55
|
+
num_shards=0,
|
|
56
|
+
shard_name_template=None,
|
|
57
|
+
coder=coders.ToBytesCoder(),
|
|
58
|
+
compression_type=filesystem.CompressionTypes.AUTO):
|
|
59
|
+
|
|
60
|
+
self._sink = _ArrayRecordSink(
|
|
61
|
+
file_path_prefix,
|
|
62
|
+
file_name_suffix,
|
|
63
|
+
num_shards,
|
|
64
|
+
shard_name_template,
|
|
65
|
+
coder,
|
|
66
|
+
compression_type)
|
|
67
|
+
|
|
68
|
+
def expand(self, pcoll):
|
|
69
|
+
return pcoll | io.iobase.Write(self._sink)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Demo Pipeline.
|
|
2
|
+
|
|
3
|
+
This file creates a TFrecord dataset and converts it to ArrayRecord on GCS
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import apache_beam as beam
|
|
7
|
+
from apache_beam.coders import coders
|
|
8
|
+
from . import dofns
|
|
9
|
+
from . import example
|
|
10
|
+
from . import options
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
## Grab CLI arguments.
|
|
14
|
+
## Override by passing args/pipeline_options to the function manually.
|
|
15
|
+
args, pipeline_options = options.get_arguments()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def main():
|
|
19
|
+
p1 = beam.Pipeline(options=pipeline_options)
|
|
20
|
+
initial = (p1
|
|
21
|
+
| 'Create a set of TFExamples' >> beam.Create(
|
|
22
|
+
example.generate_movie_examples()
|
|
23
|
+
)
|
|
24
|
+
| 'Write TFRecords' >> beam.io.WriteToTFRecord(
|
|
25
|
+
args['input'],
|
|
26
|
+
coder=coders.ToBytesCoder(),
|
|
27
|
+
num_shards=4,
|
|
28
|
+
file_name_suffix='.tfrecord'
|
|
29
|
+
)
|
|
30
|
+
| 'Read shards from GCS' >> beam.io.ReadAllFromTFRecord(
|
|
31
|
+
with_filename=True)
|
|
32
|
+
| 'Group with Filename' >> beam.GroupByKey()
|
|
33
|
+
| 'Write to ArrayRecord in GCS' >> beam.ParDo(
|
|
34
|
+
dofns.ConvertToArrayRecordGCS(),
|
|
35
|
+
args['output'],
|
|
36
|
+
overwrite_extension=True))
|
|
37
|
+
|
|
38
|
+
return p1, initial
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
if __name__ == '__main__':
|
|
42
|
+
demo_pipeline = main()
|
|
43
|
+
demo_pipeline.run()
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""DoFn's for parallel processing."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import urllib
|
|
5
|
+
import apache_beam as beam
|
|
6
|
+
from array_record.python.array_record_module import ArrayRecordWriter
|
|
7
|
+
from google.cloud import storage
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ConvertToArrayRecordGCS(beam.DoFn):
|
|
11
|
+
"""Write a tuple consisting of a filename and records to GCS ArrayRecords."""
|
|
12
|
+
|
|
13
|
+
_WRITE_DIR = '/tmp/'
|
|
14
|
+
|
|
15
|
+
def process(
|
|
16
|
+
self,
|
|
17
|
+
element,
|
|
18
|
+
path,
|
|
19
|
+
write_dir=_WRITE_DIR,
|
|
20
|
+
file_path_suffix='.arrayrecord',
|
|
21
|
+
overwrite_extension=False,
|
|
22
|
+
):
|
|
23
|
+
|
|
24
|
+
## Upload to GCS
|
|
25
|
+
def upload_to_gcs(bucket_name, filename, prefix='', source_dir=self._WRITE_DIR):
|
|
26
|
+
source_filename = os.path.join(source_dir, filename)
|
|
27
|
+
blob_name = os.path.join(prefix, filename)
|
|
28
|
+
storage_client = storage.Client()
|
|
29
|
+
bucket = storage_client.get_bucket(bucket_name)
|
|
30
|
+
blob = bucket.blob(blob_name)
|
|
31
|
+
blob.upload_from_filename(source_filename)
|
|
32
|
+
|
|
33
|
+
## Simple logic for stripping a file extension and replacing it
|
|
34
|
+
def fix_filename(filename):
|
|
35
|
+
base_name = os.path.splitext(filename)[0]
|
|
36
|
+
new_filename = base_name + file_path_suffix
|
|
37
|
+
return new_filename
|
|
38
|
+
|
|
39
|
+
parsed_gcs_path = urllib.parse.urlparse(path)
|
|
40
|
+
bucket_name = parsed_gcs_path.hostname
|
|
41
|
+
gcs_prefix = parsed_gcs_path.path.lstrip('/')
|
|
42
|
+
|
|
43
|
+
if overwrite_extension:
|
|
44
|
+
filename = fix_filename(os.path.basename(element[0]))
|
|
45
|
+
else:
|
|
46
|
+
filename = '{}{}'.format(os.path.basename(element[0]), file_path_suffix)
|
|
47
|
+
|
|
48
|
+
write_path = os.path.join(write_dir, filename)
|
|
49
|
+
writer = ArrayRecordWriter(write_path, 'group_size:1')
|
|
50
|
+
|
|
51
|
+
for item in element[1]:
|
|
52
|
+
writer.write(item)
|
|
53
|
+
|
|
54
|
+
writer.close()
|
|
55
|
+
|
|
56
|
+
upload_to_gcs(bucket_name, filename, prefix=gcs_prefix)
|
|
57
|
+
os.remove(os.path.join(write_dir, filename))
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Helper file for generating TF/ArrayRecords and writing them to disk."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from array_record.python.array_record_module import ArrayRecordWriter
|
|
5
|
+
import tensorflow as tf
|
|
6
|
+
from . import testdata
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def generate_movie_examples():
|
|
10
|
+
"""Create a list of TF examples from the dummy data above and return it.
|
|
11
|
+
|
|
12
|
+
Returns:
|
|
13
|
+
TFExample object
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
examples = []
|
|
17
|
+
for example in testdata.data:
|
|
18
|
+
examples.append(
|
|
19
|
+
tf.train.Example(
|
|
20
|
+
features=tf.train.Features(
|
|
21
|
+
feature={
|
|
22
|
+
'Age': tf.train.Feature(
|
|
23
|
+
int64_list=tf.train.Int64List(value=[example['Age']])),
|
|
24
|
+
'Movie': tf.train.Feature(
|
|
25
|
+
bytes_list=tf.train.BytesList(
|
|
26
|
+
value=[
|
|
27
|
+
m.encode('utf-8') for m in example['Movie']])),
|
|
28
|
+
'Movie Ratings': tf.train.Feature(
|
|
29
|
+
float_list=tf.train.FloatList(
|
|
30
|
+
value=example['Movie Ratings'])),
|
|
31
|
+
'Suggestion': tf.train.Feature(
|
|
32
|
+
bytes_list=tf.train.BytesList(
|
|
33
|
+
value=[example['Suggestion'].encode('utf-8')])),
|
|
34
|
+
'Suggestion Purchased': tf.train.Feature(
|
|
35
|
+
float_list=tf.train.FloatList(
|
|
36
|
+
value=[example['Suggestion Purchased']])),
|
|
37
|
+
'Purchase Price': tf.train.Feature(
|
|
38
|
+
float_list=tf.train.FloatList(
|
|
39
|
+
value=[example['Purchase Price']]))
|
|
40
|
+
}
|
|
41
|
+
)
|
|
42
|
+
)
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
return(examples)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def generate_serialized_movie_examples():
|
|
49
|
+
"""Return a serialized version of the above data for byte insertion."""
|
|
50
|
+
|
|
51
|
+
return [example.SerializeToString() for example in generate_movie_examples()]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def write_example_to_tfrecord(example, file_path):
|
|
55
|
+
"""Write example(s) to a single TFrecord file."""
|
|
56
|
+
|
|
57
|
+
with tf.io.TFRecordWriter(file_path) as writer:
|
|
58
|
+
writer.write(example.SerializeToString())
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# Write example(s) to a single ArrayRecord file
|
|
62
|
+
def write_example_to_arrayrecord(example, file_path):
|
|
63
|
+
writer = ArrayRecordWriter(file_path, 'group_size:1')
|
|
64
|
+
writer.write(example.SerializeToString())
|
|
65
|
+
writer.close()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def kitty_tfrecord(prefix=''):
|
|
69
|
+
"""Create a TFRecord from a cat pic on the Internet.
|
|
70
|
+
|
|
71
|
+
This is mainly for testing; probably don't use it.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
prefix: A file directory in string format.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
cat_in_snow = tf.keras.utils.get_file(
|
|
78
|
+
'320px-Felis_catus-cat_on_snow.jpg',
|
|
79
|
+
'https://storage.googleapis.com/download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg')
|
|
80
|
+
|
|
81
|
+
image_labels = {
|
|
82
|
+
cat_in_snow: 0
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
image_string = open(cat_in_snow, 'rb').read()
|
|
86
|
+
label = image_labels[cat_in_snow]
|
|
87
|
+
image_shape = tf.io.decode_jpeg(image_string).shape
|
|
88
|
+
|
|
89
|
+
feature = {
|
|
90
|
+
'height': tf.train.Feature(int64_list=tf.train.Int64List(
|
|
91
|
+
value=[image_shape[0]])),
|
|
92
|
+
'width': tf.train.Feature(int64_list=tf.train.Int64List(
|
|
93
|
+
value=[image_shape[1]])),
|
|
94
|
+
'depth': tf.train.Feature(int64_list=tf.train.Int64List(
|
|
95
|
+
value=[image_shape[2]])),
|
|
96
|
+
'label': tf.train.Feature(int64_list=tf.train.Int64List(
|
|
97
|
+
value=[label])),
|
|
98
|
+
'image_raw': tf.train.Feature(bytes_list=tf.train.BytesList(
|
|
99
|
+
value=[image_string]))
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
example = tf.train.Example(features=tf.train.Features(feature=feature))
|
|
103
|
+
|
|
104
|
+
record_file = os.path.join(prefix, 'kittymeow.tfrecord')
|
|
105
|
+
with tf.io.TFRecordWriter(record_file) as writer:
|
|
106
|
+
writer.write(example.SerializeToString())
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Handler for Pipeline and Beam options that allows for cleaner importing."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
from apache_beam.options import pipeline_options
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def get_arguments():
|
|
9
|
+
"""Simple external wrapper for argparse that allows for manual construction.
|
|
10
|
+
|
|
11
|
+
Returns:
|
|
12
|
+
1. A dictionary of known args for use in pipelines
|
|
13
|
+
2. The remainder of the arguments in PipelineOptions format
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
parser = argparse.ArgumentParser()
|
|
18
|
+
parser.add_argument(
|
|
19
|
+
'--input',
|
|
20
|
+
help='The file pattern for the input TFRecords.',)
|
|
21
|
+
parser.add_argument(
|
|
22
|
+
'--output',
|
|
23
|
+
help='The path prefix for output ArrayRecords.')
|
|
24
|
+
|
|
25
|
+
args, beam_args = parser.parse_known_args()
|
|
26
|
+
return(args.__dict__, pipeline_options.PipelineOptions(beam_args))
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Various opinionated Beam pipelines for testing different functionality."""
|
|
2
|
+
|
|
3
|
+
import apache_beam as beam
|
|
4
|
+
from apache_beam.coders import coders
|
|
5
|
+
from . import arrayrecordio
|
|
6
|
+
from . import dofns
|
|
7
|
+
from . import example
|
|
8
|
+
from . import options
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
## Grab CLI arguments.
|
|
12
|
+
## Override by passing args/pipeline_options to the function manually.
|
|
13
|
+
def_args, def_pipeline_options = options.get_arguments()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def example_to_tfrecord(
|
|
17
|
+
num_shards=1,
|
|
18
|
+
args=def_args,
|
|
19
|
+
pipeline_options=def_pipeline_options):
|
|
20
|
+
"""Beam pipeline for creating example TFRecord data.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
num_shards: Number of files
|
|
24
|
+
args: Custom arguments
|
|
25
|
+
pipeline_options: Beam arguments in dict format
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
Beam Pipeline object
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
p1 = beam.Pipeline(options=pipeline_options)
|
|
32
|
+
_ = (
|
|
33
|
+
p1
|
|
34
|
+
| 'Create' >> beam.Create(example.generate_movie_examples())
|
|
35
|
+
| 'Write'
|
|
36
|
+
>> beam.io.WriteToTFRecord(
|
|
37
|
+
args['output'],
|
|
38
|
+
coder=coders.ToBytesCoder(),
|
|
39
|
+
num_shards=num_shards,
|
|
40
|
+
file_name_suffix='.tfrecord',
|
|
41
|
+
)
|
|
42
|
+
)
|
|
43
|
+
return p1
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def example_to_arrayrecord(
|
|
47
|
+
num_shards=1, args=def_args, pipeline_options=def_pipeline_options
|
|
48
|
+
):
|
|
49
|
+
"""Beam pipeline for creating example ArrayRecord data.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
num_shards: Number of files
|
|
53
|
+
args: Custom arguments
|
|
54
|
+
pipeline_options: Beam arguments in dict format
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
Beam Pipeline object
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
p1 = beam.Pipeline(options=pipeline_options)
|
|
61
|
+
_ = (
|
|
62
|
+
p1
|
|
63
|
+
| 'Create' >> beam.Create(example.generate_movie_examples())
|
|
64
|
+
| 'Write'
|
|
65
|
+
>> arrayrecordio.WriteToArrayRecord(
|
|
66
|
+
args['output'],
|
|
67
|
+
coder=coders.ToBytesCoder(),
|
|
68
|
+
num_shards=num_shards,
|
|
69
|
+
file_name_suffix='.arrayrecord',
|
|
70
|
+
)
|
|
71
|
+
)
|
|
72
|
+
return p1
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def convert_tf_to_arrayrecord_disk(
|
|
76
|
+
num_shards=1, args=def_args, pipeline_options=def_pipeline_options
|
|
77
|
+
):
|
|
78
|
+
"""Convert TFRecords to ArrayRecords using sink/sharding functionality.
|
|
79
|
+
|
|
80
|
+
THIS ONLY WORKS FOR DISK ARRAYRECORD WRITES
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
num_shards: Number of files
|
|
84
|
+
args: Custom arguments
|
|
85
|
+
pipeline_options: Beam arguments in dict format
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
Beam Pipeline object
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
p1 = beam.Pipeline(options=pipeline_options)
|
|
92
|
+
_ = (
|
|
93
|
+
p1
|
|
94
|
+
| 'Read TFRecord' >> beam.io.ReadFromTFRecord(args['input'])
|
|
95
|
+
| 'Write ArrayRecord'
|
|
96
|
+
>> arrayrecordio.WriteToArrayRecord(
|
|
97
|
+
args['output'],
|
|
98
|
+
coder=coders.ToBytesCoder(),
|
|
99
|
+
num_shards=num_shards,
|
|
100
|
+
file_name_suffix='.arrayrecord',
|
|
101
|
+
)
|
|
102
|
+
)
|
|
103
|
+
return p1
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def convert_tf_to_arrayrecord_disk_match_shards(
|
|
107
|
+
args=def_args, pipeline_options=def_pipeline_options
|
|
108
|
+
):
|
|
109
|
+
"""Convert TFRecords to matching number of ArrayRecords.
|
|
110
|
+
|
|
111
|
+
THIS ONLY WORKS FOR DISK ARRAYRECORD WRITES
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
args: Custom arguments
|
|
115
|
+
pipeline_options: Beam arguments in dict format
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
Beam Pipeline object
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
p1 = beam.Pipeline(options=pipeline_options)
|
|
122
|
+
initial = (
|
|
123
|
+
p1
|
|
124
|
+
| 'Start' >> beam.Create([args['input']])
|
|
125
|
+
| 'Read' >> beam.io.ReadAllFromTFRecord(with_filename=True)
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
file_count = (
|
|
129
|
+
initial
|
|
130
|
+
| 'Group' >> beam.GroupByKey()
|
|
131
|
+
| 'Count Shards' >> beam.combiners.Count.Globally()
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
_ = (
|
|
135
|
+
initial
|
|
136
|
+
| 'Drop Filename' >> beam.Map(lambda x: x[1])
|
|
137
|
+
| 'Write ArrayRecord'
|
|
138
|
+
>> arrayrecordio.WriteToArrayRecord(
|
|
139
|
+
args['output'],
|
|
140
|
+
coder=coders.ToBytesCoder(),
|
|
141
|
+
num_shards=beam.pvalue.AsSingleton(file_count),
|
|
142
|
+
file_name_suffix='.arrayrecord',
|
|
143
|
+
)
|
|
144
|
+
)
|
|
145
|
+
return p1
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def convert_tf_to_arrayrecord_gcs(
|
|
149
|
+
overwrite_extension=False,
|
|
150
|
+
file_path_suffix='.arrayrecord',
|
|
151
|
+
args=def_args,
|
|
152
|
+
pipeline_options=def_pipeline_options):
|
|
153
|
+
"""Convert TFRecords to ArrayRecords in GCS 1:1.
|
|
154
|
+
|
|
155
|
+
Args:
|
|
156
|
+
overwrite_extension: Boolean making DoFn attempt to overwrite extension
|
|
157
|
+
file_path_suffix: Intended suffix for overwrite or append
|
|
158
|
+
args: Custom arguments
|
|
159
|
+
pipeline_options: Beam arguments in dict format
|
|
160
|
+
|
|
161
|
+
Returns:
|
|
162
|
+
Beam Pipeline object
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
p1 = beam.Pipeline(options=pipeline_options)
|
|
166
|
+
_ = (
|
|
167
|
+
p1
|
|
168
|
+
| 'Start' >> beam.Create([args['input']])
|
|
169
|
+
| 'Read' >> beam.io.ReadAllFromTFRecord(with_filename=True)
|
|
170
|
+
| 'Group' >> beam.GroupByKey()
|
|
171
|
+
| 'Write to ArrayRecord in GCS'
|
|
172
|
+
>> beam.ParDo(
|
|
173
|
+
dofns.ConvertToArrayRecordGCS(),
|
|
174
|
+
args['output'],
|
|
175
|
+
file_path_suffix=file_path_suffix,
|
|
176
|
+
overwrite_extension=overwrite_extension,
|
|
177
|
+
)
|
|
178
|
+
)
|
|
179
|
+
return p1
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Simple test data wrapper.
|
|
2
|
+
|
|
3
|
+
Separated to keep Tensorflow and Beam dependencies away from test data
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
# Hardcoded multirecord dataset in dict format for testing and demo.
|
|
7
|
+
data = [
|
|
8
|
+
{
|
|
9
|
+
'Age': 29,
|
|
10
|
+
'Movie': ['The Shawshank Redemption', 'Fight Club'],
|
|
11
|
+
'Movie Ratings': [9.0, 9.7],
|
|
12
|
+
'Suggestion': 'Inception',
|
|
13
|
+
'Suggestion Purchased': 1.0,
|
|
14
|
+
'Purchase Price': 9.99
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
'Age': 39,
|
|
18
|
+
'Movie': ['The Prestige', 'The Big Lebowski', 'The Fall'],
|
|
19
|
+
'Movie Ratings': [9.5, 8.5, 8.5],
|
|
20
|
+
'Suggestion': 'Interstellar',
|
|
21
|
+
'Suggestion Purchased': 1.0,
|
|
22
|
+
'Purchase Price': 14.99
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
'Age': 19,
|
|
26
|
+
'Movie': ['Barbie', 'The Batman', 'Boss Baby', 'Oppenheimer'],
|
|
27
|
+
'Movie Ratings': [9.6, 8.2, 10.0, 4.2],
|
|
28
|
+
'Suggestion': 'Secret Life of Pets',
|
|
29
|
+
'Suggestion Purchased': 0.0,
|
|
30
|
+
'Purchase Price': 25.99
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
'Age': 35,
|
|
34
|
+
'Movie': ['The Mothman Prophecies', 'Sinister'],
|
|
35
|
+
'Movie Ratings': [8.3, 9.0],
|
|
36
|
+
'Suggestion': 'Hereditary',
|
|
37
|
+
'Suggestion Purchased': 1.0,
|
|
38
|
+
'Purchase Price': 12.99
|
|
39
|
+
}
|
|
40
|
+
]
|
|
File without changes
|