array-record 0.8.2__cp314-cp314-macosx_11_0_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
File without changes
@@ -0,0 +1,20 @@
1
+ """Apache Beam module for array_record.
2
+
3
+ This module provides both core components and
4
+ helper functions to enable users to convert different file formats to AR.
5
+
6
+ To keep dependencies light, we'll import Beam on module usage so any errors
7
+ occur early.
8
+ """
9
+
10
+ import apache_beam as beam
11
+
12
+ # I'd really like a PEP8 compatible conditional import here with a more
13
+ # explicit error message. Example below:
14
+
15
+ # try:
16
+ # import apache_beam as beam
17
+ # except Exception as e:
18
+ # raise ImportError(
19
+ # ('Beam functionality requires extra dependencies. '
20
+ # 'Install apache-beam or run "pip install array_record[beam]".')) from e
@@ -0,0 +1,69 @@
1
+ """An IO module for ArrayRecord.
2
+
3
+ CURRENTLY ONLY SINK IS IMPLEMENTED, AND IT DOESN'T WORK WITH NON-DISK WRITES
4
+ """
5
+
6
+ from apache_beam import io
7
+ from apache_beam import transforms
8
+ from apache_beam.coders import coders
9
+ from apache_beam.io import filebasedsink
10
+ from apache_beam.io import filesystem
11
+ from array_record.python import array_record_module
12
+
13
+
14
+ class _ArrayRecordSink(filebasedsink.FileBasedSink):
15
+ """Sink Class for use in Arrayrecord PTransform."""
16
+
17
+ def __init__(
18
+ self,
19
+ file_path_prefix,
20
+ file_name_suffix=None,
21
+ num_shards=0,
22
+ shard_name_template=None,
23
+ coder=coders.ToBytesCoder(),
24
+ compression_type=filesystem.CompressionTypes.AUTO):
25
+
26
+ super().__init__(
27
+ file_path_prefix,
28
+ file_name_suffix=file_name_suffix,
29
+ num_shards=num_shards,
30
+ shard_name_template=shard_name_template,
31
+ coder=coder,
32
+ mime_type='application/octet-stream',
33
+ compression_type=compression_type)
34
+
35
+ def open(self, temp_path):
36
+ array_writer = array_record_module.ArrayRecordWriter(
37
+ temp_path, 'group_size:1'
38
+ )
39
+ return array_writer
40
+
41
+ def close(self, file_handle):
42
+ file_handle.close()
43
+
44
+ def write_encoded_record(self, file_handle, value):
45
+ file_handle.write(value)
46
+
47
+
48
+ class WriteToArrayRecord(transforms.PTransform):
49
+ """PTransform for a disk-based write to ArrayRecord."""
50
+
51
+ def __init__(
52
+ self,
53
+ file_path_prefix,
54
+ file_name_suffix='',
55
+ num_shards=0,
56
+ shard_name_template=None,
57
+ coder=coders.ToBytesCoder(),
58
+ compression_type=filesystem.CompressionTypes.AUTO):
59
+
60
+ self._sink = _ArrayRecordSink(
61
+ file_path_prefix,
62
+ file_name_suffix,
63
+ num_shards,
64
+ shard_name_template,
65
+ coder,
66
+ compression_type)
67
+
68
+ def expand(self, pcoll):
69
+ return pcoll | io.iobase.Write(self._sink)
@@ -0,0 +1,43 @@
1
+ """Demo Pipeline.
2
+
3
+ This file creates a TFrecord dataset and converts it to ArrayRecord on GCS
4
+ """
5
+
6
+ import apache_beam as beam
7
+ from apache_beam.coders import coders
8
+ from . import dofns
9
+ from . import example
10
+ from . import options
11
+
12
+
13
+ ## Grab CLI arguments.
14
+ ## Override by passing args/pipeline_options to the function manually.
15
+ args, pipeline_options = options.get_arguments()
16
+
17
+
18
+ def main():
19
+ p1 = beam.Pipeline(options=pipeline_options)
20
+ initial = (p1
21
+ | 'Create a set of TFExamples' >> beam.Create(
22
+ example.generate_movie_examples()
23
+ )
24
+ | 'Write TFRecords' >> beam.io.WriteToTFRecord(
25
+ args['input'],
26
+ coder=coders.ToBytesCoder(),
27
+ num_shards=4,
28
+ file_name_suffix='.tfrecord'
29
+ )
30
+ | 'Read shards from GCS' >> beam.io.ReadAllFromTFRecord(
31
+ with_filename=True)
32
+ | 'Group with Filename' >> beam.GroupByKey()
33
+ | 'Write to ArrayRecord in GCS' >> beam.ParDo(
34
+ dofns.ConvertToArrayRecordGCS(),
35
+ args['output'],
36
+ overwrite_extension=True))
37
+
38
+ return p1, initial
39
+
40
+
41
+ if __name__ == '__main__':
42
+ demo_pipeline = main()
43
+ demo_pipeline.run()
@@ -0,0 +1,57 @@
1
+ """DoFn's for parallel processing."""
2
+
3
+ import os
4
+ import urllib
5
+ import apache_beam as beam
6
+ from array_record.python.array_record_module import ArrayRecordWriter
7
+ from google.cloud import storage
8
+
9
+
10
+ class ConvertToArrayRecordGCS(beam.DoFn):
11
+ """Write a tuple consisting of a filename and records to GCS ArrayRecords."""
12
+
13
+ _WRITE_DIR = '/tmp/'
14
+
15
+ def process(
16
+ self,
17
+ element,
18
+ path,
19
+ write_dir=_WRITE_DIR,
20
+ file_path_suffix='.arrayrecord',
21
+ overwrite_extension=False,
22
+ ):
23
+
24
+ ## Upload to GCS
25
+ def upload_to_gcs(bucket_name, filename, prefix='', source_dir=self._WRITE_DIR):
26
+ source_filename = os.path.join(source_dir, filename)
27
+ blob_name = os.path.join(prefix, filename)
28
+ storage_client = storage.Client()
29
+ bucket = storage_client.get_bucket(bucket_name)
30
+ blob = bucket.blob(blob_name)
31
+ blob.upload_from_filename(source_filename)
32
+
33
+ ## Simple logic for stripping a file extension and replacing it
34
+ def fix_filename(filename):
35
+ base_name = os.path.splitext(filename)[0]
36
+ new_filename = base_name + file_path_suffix
37
+ return new_filename
38
+
39
+ parsed_gcs_path = urllib.parse.urlparse(path)
40
+ bucket_name = parsed_gcs_path.hostname
41
+ gcs_prefix = parsed_gcs_path.path.lstrip('/')
42
+
43
+ if overwrite_extension:
44
+ filename = fix_filename(os.path.basename(element[0]))
45
+ else:
46
+ filename = '{}{}'.format(os.path.basename(element[0]), file_path_suffix)
47
+
48
+ write_path = os.path.join(write_dir, filename)
49
+ writer = ArrayRecordWriter(write_path, 'group_size:1')
50
+
51
+ for item in element[1]:
52
+ writer.write(item)
53
+
54
+ writer.close()
55
+
56
+ upload_to_gcs(bucket_name, filename, prefix=gcs_prefix)
57
+ os.remove(os.path.join(write_dir, filename))
@@ -0,0 +1,106 @@
1
+ """Helper file for generating TF/ArrayRecords and writing them to disk."""
2
+
3
+ import os
4
+ from array_record.python.array_record_module import ArrayRecordWriter
5
+ import tensorflow as tf
6
+ from . import testdata
7
+
8
+
9
+ def generate_movie_examples():
10
+ """Create a list of TF examples from the dummy data above and return it.
11
+
12
+ Returns:
13
+ TFExample object
14
+ """
15
+
16
+ examples = []
17
+ for example in testdata.data:
18
+ examples.append(
19
+ tf.train.Example(
20
+ features=tf.train.Features(
21
+ feature={
22
+ 'Age': tf.train.Feature(
23
+ int64_list=tf.train.Int64List(value=[example['Age']])),
24
+ 'Movie': tf.train.Feature(
25
+ bytes_list=tf.train.BytesList(
26
+ value=[
27
+ m.encode('utf-8') for m in example['Movie']])),
28
+ 'Movie Ratings': tf.train.Feature(
29
+ float_list=tf.train.FloatList(
30
+ value=example['Movie Ratings'])),
31
+ 'Suggestion': tf.train.Feature(
32
+ bytes_list=tf.train.BytesList(
33
+ value=[example['Suggestion'].encode('utf-8')])),
34
+ 'Suggestion Purchased': tf.train.Feature(
35
+ float_list=tf.train.FloatList(
36
+ value=[example['Suggestion Purchased']])),
37
+ 'Purchase Price': tf.train.Feature(
38
+ float_list=tf.train.FloatList(
39
+ value=[example['Purchase Price']]))
40
+ }
41
+ )
42
+ )
43
+ )
44
+
45
+ return(examples)
46
+
47
+
48
+ def generate_serialized_movie_examples():
49
+ """Return a serialized version of the above data for byte insertion."""
50
+
51
+ return [example.SerializeToString() for example in generate_movie_examples()]
52
+
53
+
54
+ def write_example_to_tfrecord(example, file_path):
55
+ """Write example(s) to a single TFrecord file."""
56
+
57
+ with tf.io.TFRecordWriter(file_path) as writer:
58
+ writer.write(example.SerializeToString())
59
+
60
+
61
+ # Write example(s) to a single ArrayRecord file
62
+ def write_example_to_arrayrecord(example, file_path):
63
+ writer = ArrayRecordWriter(file_path, 'group_size:1')
64
+ writer.write(example.SerializeToString())
65
+ writer.close()
66
+
67
+
68
+ def kitty_tfrecord(prefix=''):
69
+ """Create a TFRecord from a cat pic on the Internet.
70
+
71
+ This is mainly for testing; probably don't use it.
72
+
73
+ Args:
74
+ prefix: A file directory in string format.
75
+ """
76
+
77
+ cat_in_snow = tf.keras.utils.get_file(
78
+ '320px-Felis_catus-cat_on_snow.jpg',
79
+ 'https://storage.googleapis.com/download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg')
80
+
81
+ image_labels = {
82
+ cat_in_snow: 0
83
+ }
84
+
85
+ image_string = open(cat_in_snow, 'rb').read()
86
+ label = image_labels[cat_in_snow]
87
+ image_shape = tf.io.decode_jpeg(image_string).shape
88
+
89
+ feature = {
90
+ 'height': tf.train.Feature(int64_list=tf.train.Int64List(
91
+ value=[image_shape[0]])),
92
+ 'width': tf.train.Feature(int64_list=tf.train.Int64List(
93
+ value=[image_shape[1]])),
94
+ 'depth': tf.train.Feature(int64_list=tf.train.Int64List(
95
+ value=[image_shape[2]])),
96
+ 'label': tf.train.Feature(int64_list=tf.train.Int64List(
97
+ value=[label])),
98
+ 'image_raw': tf.train.Feature(bytes_list=tf.train.BytesList(
99
+ value=[image_string]))
100
+ }
101
+
102
+ example = tf.train.Example(features=tf.train.Features(feature=feature))
103
+
104
+ record_file = os.path.join(prefix, 'kittymeow.tfrecord')
105
+ with tf.io.TFRecordWriter(record_file) as writer:
106
+ writer.write(example.SerializeToString())
@@ -0,0 +1,26 @@
1
+ """Handler for Pipeline and Beam options that allows for cleaner importing."""
2
+
3
+
4
+ import argparse
5
+ from apache_beam.options import pipeline_options
6
+
7
+
8
+ def get_arguments():
9
+ """Simple external wrapper for argparse that allows for manual construction.
10
+
11
+ Returns:
12
+ 1. A dictionary of known args for use in pipelines
13
+ 2. The remainder of the arguments in PipelineOptions format
14
+
15
+ """
16
+
17
+ parser = argparse.ArgumentParser()
18
+ parser.add_argument(
19
+ '--input',
20
+ help='The file pattern for the input TFRecords.',)
21
+ parser.add_argument(
22
+ '--output',
23
+ help='The path prefix for output ArrayRecords.')
24
+
25
+ args, beam_args = parser.parse_known_args()
26
+ return(args.__dict__, pipeline_options.PipelineOptions(beam_args))
@@ -0,0 +1,179 @@
1
+ """Various opinionated Beam pipelines for testing different functionality."""
2
+
3
+ import apache_beam as beam
4
+ from apache_beam.coders import coders
5
+ from . import arrayrecordio
6
+ from . import dofns
7
+ from . import example
8
+ from . import options
9
+
10
+
11
+ ## Grab CLI arguments.
12
+ ## Override by passing args/pipeline_options to the function manually.
13
+ def_args, def_pipeline_options = options.get_arguments()
14
+
15
+
16
+ def example_to_tfrecord(
17
+ num_shards=1,
18
+ args=def_args,
19
+ pipeline_options=def_pipeline_options):
20
+ """Beam pipeline for creating example TFRecord data.
21
+
22
+ Args:
23
+ num_shards: Number of files
24
+ args: Custom arguments
25
+ pipeline_options: Beam arguments in dict format
26
+
27
+ Returns:
28
+ Beam Pipeline object
29
+ """
30
+
31
+ p1 = beam.Pipeline(options=pipeline_options)
32
+ _ = (
33
+ p1
34
+ | 'Create' >> beam.Create(example.generate_movie_examples())
35
+ | 'Write'
36
+ >> beam.io.WriteToTFRecord(
37
+ args['output'],
38
+ coder=coders.ToBytesCoder(),
39
+ num_shards=num_shards,
40
+ file_name_suffix='.tfrecord',
41
+ )
42
+ )
43
+ return p1
44
+
45
+
46
+ def example_to_arrayrecord(
47
+ num_shards=1, args=def_args, pipeline_options=def_pipeline_options
48
+ ):
49
+ """Beam pipeline for creating example ArrayRecord data.
50
+
51
+ Args:
52
+ num_shards: Number of files
53
+ args: Custom arguments
54
+ pipeline_options: Beam arguments in dict format
55
+
56
+ Returns:
57
+ Beam Pipeline object
58
+ """
59
+
60
+ p1 = beam.Pipeline(options=pipeline_options)
61
+ _ = (
62
+ p1
63
+ | 'Create' >> beam.Create(example.generate_movie_examples())
64
+ | 'Write'
65
+ >> arrayrecordio.WriteToArrayRecord(
66
+ args['output'],
67
+ coder=coders.ToBytesCoder(),
68
+ num_shards=num_shards,
69
+ file_name_suffix='.arrayrecord',
70
+ )
71
+ )
72
+ return p1
73
+
74
+
75
+ def convert_tf_to_arrayrecord_disk(
76
+ num_shards=1, args=def_args, pipeline_options=def_pipeline_options
77
+ ):
78
+ """Convert TFRecords to ArrayRecords using sink/sharding functionality.
79
+
80
+ THIS ONLY WORKS FOR DISK ARRAYRECORD WRITES
81
+
82
+ Args:
83
+ num_shards: Number of files
84
+ args: Custom arguments
85
+ pipeline_options: Beam arguments in dict format
86
+
87
+ Returns:
88
+ Beam Pipeline object
89
+ """
90
+
91
+ p1 = beam.Pipeline(options=pipeline_options)
92
+ _ = (
93
+ p1
94
+ | 'Read TFRecord' >> beam.io.ReadFromTFRecord(args['input'])
95
+ | 'Write ArrayRecord'
96
+ >> arrayrecordio.WriteToArrayRecord(
97
+ args['output'],
98
+ coder=coders.ToBytesCoder(),
99
+ num_shards=num_shards,
100
+ file_name_suffix='.arrayrecord',
101
+ )
102
+ )
103
+ return p1
104
+
105
+
106
+ def convert_tf_to_arrayrecord_disk_match_shards(
107
+ args=def_args, pipeline_options=def_pipeline_options
108
+ ):
109
+ """Convert TFRecords to matching number of ArrayRecords.
110
+
111
+ THIS ONLY WORKS FOR DISK ARRAYRECORD WRITES
112
+
113
+ Args:
114
+ args: Custom arguments
115
+ pipeline_options: Beam arguments in dict format
116
+
117
+ Returns:
118
+ Beam Pipeline object
119
+ """
120
+
121
+ p1 = beam.Pipeline(options=pipeline_options)
122
+ initial = (
123
+ p1
124
+ | 'Start' >> beam.Create([args['input']])
125
+ | 'Read' >> beam.io.ReadAllFromTFRecord(with_filename=True)
126
+ )
127
+
128
+ file_count = (
129
+ initial
130
+ | 'Group' >> beam.GroupByKey()
131
+ | 'Count Shards' >> beam.combiners.Count.Globally()
132
+ )
133
+
134
+ _ = (
135
+ initial
136
+ | 'Drop Filename' >> beam.Map(lambda x: x[1])
137
+ | 'Write ArrayRecord'
138
+ >> arrayrecordio.WriteToArrayRecord(
139
+ args['output'],
140
+ coder=coders.ToBytesCoder(),
141
+ num_shards=beam.pvalue.AsSingleton(file_count),
142
+ file_name_suffix='.arrayrecord',
143
+ )
144
+ )
145
+ return p1
146
+
147
+
148
+ def convert_tf_to_arrayrecord_gcs(
149
+ overwrite_extension=False,
150
+ file_path_suffix='.arrayrecord',
151
+ args=def_args,
152
+ pipeline_options=def_pipeline_options):
153
+ """Convert TFRecords to ArrayRecords in GCS 1:1.
154
+
155
+ Args:
156
+ overwrite_extension: Boolean making DoFn attempt to overwrite extension
157
+ file_path_suffix: Intended suffix for overwrite or append
158
+ args: Custom arguments
159
+ pipeline_options: Beam arguments in dict format
160
+
161
+ Returns:
162
+ Beam Pipeline object
163
+ """
164
+
165
+ p1 = beam.Pipeline(options=pipeline_options)
166
+ _ = (
167
+ p1
168
+ | 'Start' >> beam.Create([args['input']])
169
+ | 'Read' >> beam.io.ReadAllFromTFRecord(with_filename=True)
170
+ | 'Group' >> beam.GroupByKey()
171
+ | 'Write to ArrayRecord in GCS'
172
+ >> beam.ParDo(
173
+ dofns.ConvertToArrayRecordGCS(),
174
+ args['output'],
175
+ file_path_suffix=file_path_suffix,
176
+ overwrite_extension=overwrite_extension,
177
+ )
178
+ )
179
+ return p1
@@ -0,0 +1,40 @@
1
+ """Simple test data wrapper.
2
+
3
+ Separated to keep Tensorflow and Beam dependencies away from test data
4
+ """
5
+
6
+ # Hardcoded multirecord dataset in dict format for testing and demo.
7
+ data = [
8
+ {
9
+ 'Age': 29,
10
+ 'Movie': ['The Shawshank Redemption', 'Fight Club'],
11
+ 'Movie Ratings': [9.0, 9.7],
12
+ 'Suggestion': 'Inception',
13
+ 'Suggestion Purchased': 1.0,
14
+ 'Purchase Price': 9.99
15
+ },
16
+ {
17
+ 'Age': 39,
18
+ 'Movie': ['The Prestige', 'The Big Lebowski', 'The Fall'],
19
+ 'Movie Ratings': [9.5, 8.5, 8.5],
20
+ 'Suggestion': 'Interstellar',
21
+ 'Suggestion Purchased': 1.0,
22
+ 'Purchase Price': 14.99
23
+ },
24
+ {
25
+ 'Age': 19,
26
+ 'Movie': ['Barbie', 'The Batman', 'Boss Baby', 'Oppenheimer'],
27
+ 'Movie Ratings': [9.6, 8.2, 10.0, 4.2],
28
+ 'Suggestion': 'Secret Life of Pets',
29
+ 'Suggestion Purchased': 0.0,
30
+ 'Purchase Price': 25.99
31
+ },
32
+ {
33
+ 'Age': 35,
34
+ 'Movie': ['The Mothman Prophecies', 'Sinister'],
35
+ 'Movie Ratings': [8.3, 9.0],
36
+ 'Suggestion': 'Hereditary',
37
+ 'Suggestion Purchased': 1.0,
38
+ 'Purchase Price': 12.99
39
+ }
40
+ ]
File without changes