grain-nightly 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of grain-nightly might be problematic. Click here for more details.
- grain/_src/core/config.py +109 -0
- grain/_src/core/constants.py +39 -0
- grain/_src/core/exceptions.py +5 -0
- grain/_src/core/grain_random.py +126 -0
- grain/_src/core/grain_random_test.py +83 -0
- grain/_src/core/parallel.py +72 -0
- grain/_src/core/parallel_test.py +110 -0
- grain/_src/core/sharding.py +103 -0
- grain/_src/core/sharding_test.py +42 -0
- grain/_src/core/transforms.py +112 -0
- grain/_src/core/usage_logging.py +18 -0
- grain/_src/python/checkpoint_handlers.py +54 -0
- grain/_src/python/data_loader.py +679 -0
- grain/_src/python/data_loader_test.py +578 -0
- grain/_src/python/data_sources.py +242 -0
- grain/_src/python/data_sources_test.py +184 -0
- grain/_src/python/experimental/example_packing/packing.py +204 -0
- grain/_src/python/experimental/example_packing/packing_test.py +490 -0
- grain/_src/python/experimental/index_shuffle/libindex_shuffle.so +0 -0
- grain/_src/python/experimental/index_shuffle/python/index_shuffle_module.so +0 -0
- grain/_src/python/experimental/index_shuffle/python/index_shuffle_test.py +42 -0
- grain/_src/python/grain_pool.py +373 -0
- grain/_src/python/grain_pool_test.py +153 -0
- grain/_src/python/lazy_dataset/data_sources.py +49 -0
- grain/_src/python/lazy_dataset/data_sources_test.py +66 -0
- grain/_src/python/lazy_dataset/grain_pool_v2.py +334 -0
- grain/_src/python/lazy_dataset/grain_pool_v2_test.py +233 -0
- grain/_src/python/lazy_dataset/lazy_dataset.py +288 -0
- grain/_src/python/lazy_dataset/lazy_dataset_test.py +116 -0
- grain/_src/python/lazy_dataset/transformations/batch.py +136 -0
- grain/_src/python/lazy_dataset/transformations/batch_test.py +76 -0
- grain/_src/python/lazy_dataset/transformations/filter.py +120 -0
- grain/_src/python/lazy_dataset/transformations/filter_test.py +125 -0
- grain/_src/python/lazy_dataset/transformations/flatmap.py +62 -0
- grain/_src/python/lazy_dataset/transformations/flatmap_test.py +127 -0
- grain/_src/python/lazy_dataset/transformations/map.py +214 -0
- grain/_src/python/lazy_dataset/transformations/map_test.py +134 -0
- grain/_src/python/lazy_dataset/transformations/mix.py +148 -0
- grain/_src/python/lazy_dataset/transformations/mix_test.py +75 -0
- grain/_src/python/lazy_dataset/transformations/ragged_batch.py +51 -0
- grain/_src/python/lazy_dataset/transformations/ragged_batch_test.py +106 -0
- grain/_src/python/lazy_dataset/transformations/shuffle.py +47 -0
- grain/_src/python/lazy_dataset/transformations/shuffle_test.py +63 -0
- grain/_src/python/lazy_dataset/transformations/slice.py +38 -0
- grain/_src/python/lazy_dataset/transformations/slice_test.py +39 -0
- grain/_src/python/load.py +65 -0
- grain/_src/python/load_test.py +74 -0
- grain/_src/python/multiprocessing_common.py +73 -0
- grain/_src/python/multiprocessing_common_test.py +94 -0
- grain/_src/python/operations.py +205 -0
- grain/_src/python/operations_test.py +393 -0
- grain/_src/python/options.py +60 -0
- grain/_src/python/record.py +53 -0
- grain/_src/python/record_test.py +41 -0
- grain/_src/python/samplers.py +156 -0
- grain/_src/python/samplers_test.py +657 -0
- grain/core.py +22 -0
- grain/python.py +63 -0
- grain/python_experimental.py +23 -0
- grain/python_lazy_dataset.py +45 -0
- grain_nightly-0.0.1.dist-info/LICENSE +201 -0
- grain_nightly-0.0.1.dist-info/METADATA +228 -0
- grain_nightly-0.0.1.dist-info/RECORD +65 -0
- grain_nightly-0.0.1.dist-info/WHEEL +5 -0
- grain_nightly-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# Copyright 2022 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Handle Grain config options.
|
|
15
|
+
|
|
16
|
+
Config options can be set via flags starting with '--grain_' or by calling
|
|
17
|
+
`grain.config.update(name, value)`.
|
|
18
|
+
"""
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from absl import flags
|
|
22
|
+
|
|
23
|
+
# Performance optimisations. Consider most of these experimental. We might
|
|
24
|
+
# remove them once we are confident that the default values work well for
|
|
25
|
+
# everyone.
|
|
26
|
+
_INTERLEAVED_SHUFFLE = flags.DEFINE_bool(
|
|
27
|
+
"grain_tf_interleaved_shuffle",
|
|
28
|
+
False,
|
|
29
|
+
(
|
|
30
|
+
"If enabled replace the global shuffle with an approximation that "
|
|
31
|
+
"interleaves small blocks. This can increase throughput of the input "
|
|
32
|
+
"pipeline. See index_dataset._interleaved_shuffle() for details."
|
|
33
|
+
),
|
|
34
|
+
)
|
|
35
|
+
_INTERLEAVED_SHUFFLE_BLOCK_SIZE = flags.DEFINE_integer(
|
|
36
|
+
"grain_tf_interleaved_shuffle_block_size",
|
|
37
|
+
10,
|
|
38
|
+
"Block size in index_dataset._interleaved_shuffle().",
|
|
39
|
+
)
|
|
40
|
+
_INTERLEAVED_SHUFFLE_NUM_BLOCKS_PER_PARTITION = flags.DEFINE_integer(
|
|
41
|
+
"grain_tf_interleaved_shuffle_parallel_blocks",
|
|
42
|
+
10,
|
|
43
|
+
"Number of blocks per partition in index_dataset._interleaved_shuffle().",
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
_LOOKUP_BATCH_SIZE = flags.DEFINE_integer(
|
|
47
|
+
"grain_tf_lookup_batch_size",
|
|
48
|
+
100,
|
|
49
|
+
(
|
|
50
|
+
"Number of keys to batch into a single lookup call."
|
|
51
|
+
" --grain_tf_num_parallel_calls controls on how many lookups to perform"
|
|
52
|
+
" in parallel."
|
|
53
|
+
),
|
|
54
|
+
)
|
|
55
|
+
_LOOKUP_NUM_PARALLEL_CALLS = flags.DEFINE_integer(
|
|
56
|
+
"grain_tf_lookup_num_parallel_calls",
|
|
57
|
+
-1,
|
|
58
|
+
(
|
|
59
|
+
"Number of parallel lookup calls. Each lookup will read"
|
|
60
|
+
" --grain_tf_lookup_batch_size records. This must be a positive integer"
|
|
61
|
+
" > 0 or -1. If -1 Grain will use tf.data.AUTOTUNE to tune the value"
|
|
62
|
+
" automatically."
|
|
63
|
+
),
|
|
64
|
+
)
|
|
65
|
+
_LOOKUP_FAST_WARMUP = flags.DEFINE_bool(
|
|
66
|
+
"grain_tf_lookup_fast_warmup",
|
|
67
|
+
False,
|
|
68
|
+
(
|
|
69
|
+
"Deprecated. If True will split up the first batch lookup into smaller"
|
|
70
|
+
" chunks. This can help to provide initial elements fast while allowing"
|
|
71
|
+
" when using a large tf_lookup_batch_size (which is usually better for"
|
|
72
|
+
" throughput)."
|
|
73
|
+
),
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
_GRAIN_FLAGS = (
|
|
77
|
+
_INTERLEAVED_SHUFFLE,
|
|
78
|
+
_INTERLEAVED_SHUFFLE_BLOCK_SIZE,
|
|
79
|
+
_INTERLEAVED_SHUFFLE_NUM_BLOCKS_PER_PARTITION,
|
|
80
|
+
_LOOKUP_BATCH_SIZE,
|
|
81
|
+
_LOOKUP_NUM_PARALLEL_CALLS,
|
|
82
|
+
_LOOKUP_FAST_WARMUP,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class Config:
|
|
87
|
+
"""Class for holding current Grain configuration."""
|
|
88
|
+
|
|
89
|
+
# Loosen the static type checking requirements.
|
|
90
|
+
_HAS_DYNAMIC_ATTRIBUTES = True
|
|
91
|
+
|
|
92
|
+
def __getattr__(self, name: str) -> Any:
|
|
93
|
+
flag_name = f"grain_{name}"
|
|
94
|
+
if any(f.name == flag_name for f in _GRAIN_FLAGS):
|
|
95
|
+
return getattr(flags.FLAGS, flag_name)
|
|
96
|
+
raise ValueError(f"Unrecognized config option: {name}")
|
|
97
|
+
|
|
98
|
+
def __setattr__(self, name: str, value: Any):
|
|
99
|
+
raise ValueError("Please use update().")
|
|
100
|
+
|
|
101
|
+
def update(self, name: str, value: Any):
|
|
102
|
+
flag_name = f"grain_{name}"
|
|
103
|
+
if any(f.name == flag_name for f in _GRAIN_FLAGS):
|
|
104
|
+
setattr(flags.FLAGS, flag_name, value)
|
|
105
|
+
return
|
|
106
|
+
raise ValueError(f"Unrecognized config option: {name}")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
config = Config()
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Copyright 2022 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Shared constants for various Grain APIs."""
|
|
15
|
+
|
|
16
|
+
# Below are names of meta features used by index_dataset (and pipelines building
|
|
17
|
+
# on top of it). These features are generated on the fly and help to track
|
|
18
|
+
# progress over the dataset. Users can read these but shouldn't alter them. They
|
|
19
|
+
# start with "_" to indicate that they are "private".
|
|
20
|
+
# Index into the stream of all records (globally unique). Starts with 0.
|
|
21
|
+
INDEX = "_index"
|
|
22
|
+
# Key of the record. If DATASET_INDEX is present it's the key in the dataset.
|
|
23
|
+
# Starts with 0.
|
|
24
|
+
RECORD_KEY = "_record_key"
|
|
25
|
+
# Index of the dataset from which to take the record. Only present when mixing.
|
|
26
|
+
# Starts with 0.
|
|
27
|
+
DATASET_INDEX = "_dataset_index"
|
|
28
|
+
# Epoch for the record. When mixing datasets this is the epoch over the dataset,
|
|
29
|
+
# not the mixture. Starts with 1.
|
|
30
|
+
EPOCH = "_epoch"
|
|
31
|
+
# Random seed for stateless random operations. This is unique per record
|
|
32
|
+
# and changes every epoch.
|
|
33
|
+
SEED = "_seed"
|
|
34
|
+
# Serialized record.
|
|
35
|
+
RECORD = "_record"
|
|
36
|
+
|
|
37
|
+
META_FEATURES = frozenset(
|
|
38
|
+
[INDEX, RECORD_KEY, DATASET_INDEX, EPOCH, SEED, RECORD]
|
|
39
|
+
)
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# Copyright 2022 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Type definition and methods for handling random numbers.
|
|
15
|
+
|
|
16
|
+
Grain uses counter-based pseudorandom number generators (PRNGs) for fast and
|
|
17
|
+
reproducible random numbers.
|
|
18
|
+
This means that there is no global random seed or a global stateful PRNGs
|
|
19
|
+
object. Instead, random seeds are always passed explicitly and users should
|
|
20
|
+
understand the rules outlined below (this applies both to the internals of
|
|
21
|
+
Grain as well as user defined preprocessing and extensions).
|
|
22
|
+
|
|
23
|
+
## Motivation
|
|
24
|
+
We want Grain input pipelines to be reproducible while still being highly
|
|
25
|
+
parallelizable. This very much overlaps with the design choices for JAX's PRNG
|
|
26
|
+
([design notes](github.com/google/jax/blob/main/docs/design_notes/prng.md)) and
|
|
27
|
+
their choice for counter-based PRNGs like ThreeFry and Philox.
|
|
28
|
+
NumPy, JAX and TF all contain implementations of ThreeFry/Philox.
|
|
29
|
+
|
|
30
|
+
## General Rules
|
|
31
|
+
If you are not familiar with `jax.random`, we recommend reading
|
|
32
|
+
[the JAX
|
|
33
|
+
documentation](https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#random-numbers)
|
|
34
|
+
or
|
|
35
|
+
[this
|
|
36
|
+
Colab](https://colab.research.google.com/github/google/jax/blob/main/docs/jax-101/05-random-numbers.ipynb).
|
|
37
|
+
to understand the concepts of counter-based PRNGs. You should be comfortable
|
|
38
|
+
with splitting your random seeds.
|
|
39
|
+
|
|
40
|
+
After that, just follow these rules:
|
|
41
|
+
- Never use the `random` module from the Python standard library.
|
|
42
|
+
- Either use methods in `jax.random`, random methods in TensorFlow that start
|
|
43
|
+
with `stateless_` (e.g. `tf.random.stateless_uniform()`) or non-global
|
|
44
|
+
NumPy random generators.
|
|
45
|
+
- Do not use the global RNG in `np.random`. Instead create a new generator
|
|
46
|
+
with appropiate bit generator: `np.random.default_rng(seed)` or
|
|
47
|
+
`np.random.Generator(np.random.Philox(seed))` and call random functions on it.
|
|
48
|
+
The generator (or more accurately the underlying bit generator) is stateful
|
|
49
|
+
and should not be used across between threads.
|
|
50
|
+
- If you want to use a new random seed each time your program starts, you can
|
|
51
|
+
use `int.from_bytes(os.urandom(4), sys.byteorder)` and log the value.
|
|
52
|
+
- Do not re-use a random seed. Instead, split the random seed using
|
|
53
|
+
`jax.random.split()` or `tf.random.experimental.stateless_split()`
|
|
54
|
+
|
|
55
|
+
## JAX
|
|
56
|
+
JAX supports multiple RNG implementations. Since Grain operations run on CPU,
|
|
57
|
+
we recommend the default (ThreeFry). It's stable across JAX/XLA versions.
|
|
58
|
+
Internally, ThreeFry uses a uint32 type vector of shape [2] as state.
|
|
59
|
+
|
|
60
|
+
## NumPy
|
|
61
|
+
NumPy contains several suitable RNG implementations (aka bit generators). We
|
|
62
|
+
recommend using `np.random.Philox` which is very similar to the ThreeFry
|
|
63
|
+
implementation in JAX and TF. However `PCG64` (default as of writting) and
|
|
64
|
+
`PCG64DXSM` are also suitable for input pipelines. Just don't use the global
|
|
65
|
+
RNG and always construct your own `np.random.Generator`.
|
|
66
|
+
|
|
67
|
+
## TensorFlow
|
|
68
|
+
TensorFlow implements multiple counter-based RNGs (ThreeFry and Philox) but
|
|
69
|
+
doesn't provide any guidance on which one to use. We recommend to not set the
|
|
70
|
+
algorithm explicitly and let TF select the RNG (on CPUs it seems to select
|
|
71
|
+
Philox). You can derive a random seed for stateless random functions using
|
|
72
|
+
`tf.random.Generator.from_seed(my_integer).make_seeds(1)[:, 0]`.
|
|
73
|
+
Splitting seeds can be done using:
|
|
74
|
+
```
|
|
75
|
+
seed, seed_for_my_op = tf.unstack(tf.random.experimental.stateless_split(seed))
|
|
76
|
+
```
|
|
77
|
+
"""
|
|
78
|
+
import os
|
|
79
|
+
import sys
|
|
80
|
+
from typing import Optional, Sequence, Union
|
|
81
|
+
|
|
82
|
+
from absl import logging
|
|
83
|
+
import jax
|
|
84
|
+
import jax.numpy as jnp
|
|
85
|
+
import numpy as np
|
|
86
|
+
|
|
87
|
+
# As of 2022-09 JAX uses arrays of dtype uint32 and shape [2] for RNG keys.
|
|
88
|
+
# However when custom RNGs are enabled by default keys are wrapped in a KeyArray
|
|
89
|
+
# object and can have different internal state. This might become the default
|
|
90
|
+
# in the future.
|
|
91
|
+
RNGKey = jax.random.PRNGKeyArray
|
|
92
|
+
RNGKeyLike = Union[RNGKey, int, Sequence[int], np.ndarray]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def as_rng_key(seed: RNGKeyLike) -> RNGKey:
|
|
96
|
+
"""Returns an RNGKey from the input.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
seed: Rndom seed as single integer or tuple/array of 2 integers.
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
RNGKey which is currently a NumPy array of dtype uint32 and shape [2].
|
|
103
|
+
"""
|
|
104
|
+
if isinstance(seed, jax.random.PRNGKeyArray):
|
|
105
|
+
return seed
|
|
106
|
+
if isinstance(seed, (int, jnp.integer)):
|
|
107
|
+
return jax.random.PRNGKey(seed)
|
|
108
|
+
if isinstance(seed, jnp.ndarray):
|
|
109
|
+
# Assume that this is already a KeyArray.
|
|
110
|
+
return seed
|
|
111
|
+
if len(seed) != 2:
|
|
112
|
+
raise ValueError(
|
|
113
|
+
"Random seed must be a single integer or a "
|
|
114
|
+
f"Tuple[int, int] but got {seed!r}"
|
|
115
|
+
)
|
|
116
|
+
return jax.random.PRNGKey(sum(seed))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def make_rng_key(seed: Optional[RNGKeyLike]) -> RNGKey:
|
|
120
|
+
logging.error(
|
|
121
|
+
"Using deprecated method make_rng_key(). Please always pass "
|
|
122
|
+
"None or a single integer as random seed to Grain methods."
|
|
123
|
+
)
|
|
124
|
+
if seed is None:
|
|
125
|
+
seed = int.from_bytes(os.urandom(4), sys.byteorder)
|
|
126
|
+
return as_rng_key(seed)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# Copyright 2022 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Unit tests for the random module."""
|
|
15
|
+
import contextlib
|
|
16
|
+
import itertools
|
|
17
|
+
|
|
18
|
+
from absl.testing import parameterized
|
|
19
|
+
from grain._src.core import grain_random
|
|
20
|
+
import jax
|
|
21
|
+
import numpy as np
|
|
22
|
+
import tensorflow as tf
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class RandomTest(tf.test.TestCase, parameterized.TestCase):
|
|
26
|
+
"""Tests for the random module."""
|
|
27
|
+
|
|
28
|
+
@parameterized.parameters(
|
|
29
|
+
{"seed": 585},
|
|
30
|
+
{"seed": (5424, 95849)},
|
|
31
|
+
{"seed": np.asarray([5424, 95849])},
|
|
32
|
+
{"seed": np.int32(95849)},
|
|
33
|
+
{"seed": np.int64(5424)},
|
|
34
|
+
)
|
|
35
|
+
def test_rng_for_jax(self, seed):
|
|
36
|
+
seed = grain_random.as_rng_key(seed)
|
|
37
|
+
# Verify we can split the seed.
|
|
38
|
+
seed, _ = jax.random.split(seed)
|
|
39
|
+
# Verify we can generate a random number.
|
|
40
|
+
jax.random.uniform(seed, [])
|
|
41
|
+
|
|
42
|
+
@parameterized.parameters(
|
|
43
|
+
itertools.product(
|
|
44
|
+
[585, (5424, 95849), np.asarray([5424, 95849])],
|
|
45
|
+
[
|
|
46
|
+
# No context.
|
|
47
|
+
(None, None),
|
|
48
|
+
# Each separately.
|
|
49
|
+
("custom", None),
|
|
50
|
+
("threefry2x32", None),
|
|
51
|
+
("rbg", None),
|
|
52
|
+
# implementation frist, custom afterwards.
|
|
53
|
+
("threefry2x32", "custom"),
|
|
54
|
+
("rbg", "custom"),
|
|
55
|
+
# custom first, implementation afterwards.
|
|
56
|
+
("custom", "threefry2x32"),
|
|
57
|
+
("custom", "rbg"),
|
|
58
|
+
],
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
def test_rng_for_jax_custom(self, seed, contexts):
|
|
62
|
+
def get_ctx(name):
|
|
63
|
+
if name is None:
|
|
64
|
+
return contextlib.nullcontext()
|
|
65
|
+
if name == "threefry2x32":
|
|
66
|
+
return jax.default_prng_impl("threefry2x32")
|
|
67
|
+
if name == "rbg":
|
|
68
|
+
return jax.default_prng_impl("rbg")
|
|
69
|
+
if name == "custom":
|
|
70
|
+
return jax.enable_custom_prng()
|
|
71
|
+
assert False
|
|
72
|
+
|
|
73
|
+
with get_ctx(contexts[0]):
|
|
74
|
+
with get_ctx(contexts[1]):
|
|
75
|
+
seed = grain_random.as_rng_key(seed)
|
|
76
|
+
# Verify we can split the seed.
|
|
77
|
+
seed, _ = jax.random.split(seed)
|
|
78
|
+
# Verify we can generate a random number.
|
|
79
|
+
jax.random.uniform(seed, [])
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
if __name__ == "__main__":
|
|
83
|
+
tf.test.main()
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Provides a methods to run functions in parallel using a thread pool."""
|
|
2
|
+
from collections.abc import Mapping, Sequence
|
|
3
|
+
from typing import Any, Callable, TypeVar
|
|
4
|
+
|
|
5
|
+
from concurrent import futures
|
|
6
|
+
|
|
7
|
+
T = TypeVar("T")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def run_in_parallel(
|
|
11
|
+
function: Callable[..., T],
|
|
12
|
+
list_of_kwargs_to_function: Sequence[Mapping[str, Any]],
|
|
13
|
+
num_workers: int,
|
|
14
|
+
thread_name_prefix: str = "parallel_",
|
|
15
|
+
) -> list[T]:
|
|
16
|
+
"""Run a function on a list of kwargs in parallel with ThreadPoolExecutor.
|
|
17
|
+
|
|
18
|
+
Works best when there is IO boundedness, not when there is CPU boundedness, as
|
|
19
|
+
the threads used are bound by the GIL.
|
|
20
|
+
|
|
21
|
+
Propagates first exception to the calling thread. If cancel_futures=True,
|
|
22
|
+
then stop as many of the ongoing work units as possible.
|
|
23
|
+
|
|
24
|
+
Example usage:
|
|
25
|
+
def io_bound_function(p):
|
|
26
|
+
get_contents_from_cns(p)
|
|
27
|
+
|
|
28
|
+
run_in_parallel(
|
|
29
|
+
function=io_bound_function,
|
|
30
|
+
list_of_kwargs_to_function=[{"p": p} for p in long_list_of_paths],
|
|
31
|
+
num_workers=3)
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
function: a function.
|
|
35
|
+
list_of_kwargs_to_function: A list of dicts mapping from string to argument
|
|
36
|
+
value. These will be passed into `function` as kwargs.
|
|
37
|
+
num_workers: int.
|
|
38
|
+
thread_name_prefix: The thread name prefix string. Processes are run in
|
|
39
|
+
threads, and each thread is named. This parameter allows the user to
|
|
40
|
+
control the prefix for that thread name.
|
|
41
|
+
|
|
42
|
+
Returns:
|
|
43
|
+
list of return values from function, in the same order as the arguments in
|
|
44
|
+
list_of_kwargs_to_function.
|
|
45
|
+
"""
|
|
46
|
+
if num_workers < 1:
|
|
47
|
+
raise ValueError(
|
|
48
|
+
"Number of workers must be greater than 0. Was {}".format(num_workers)
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
thread_name = thread_name_prefix + getattr(function, "__name__", "unknown")
|
|
52
|
+
with futures.ThreadPoolExecutor(
|
|
53
|
+
num_workers, thread_name_prefix=thread_name
|
|
54
|
+
) as executor:
|
|
55
|
+
fs = []
|
|
56
|
+
|
|
57
|
+
for kwargs in list_of_kwargs_to_function:
|
|
58
|
+
f = executor.submit(function, **kwargs)
|
|
59
|
+
fs.append(f)
|
|
60
|
+
|
|
61
|
+
futures_as_completed = futures.as_completed(fs)
|
|
62
|
+
|
|
63
|
+
for completed in futures_as_completed:
|
|
64
|
+
if completed.exception():
|
|
65
|
+
# Cancel all remaining futures, if possible.
|
|
66
|
+
for remaining_future in fs:
|
|
67
|
+
remaining_future.cancel()
|
|
68
|
+
|
|
69
|
+
# Propagate exception to main thread.
|
|
70
|
+
raise completed.exception()
|
|
71
|
+
|
|
72
|
+
return [f.result() for f in fs]
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Tests for parallel."""
|
|
2
|
+
import threading
|
|
3
|
+
|
|
4
|
+
from absl.testing import absltest
|
|
5
|
+
from absl.testing import parameterized
|
|
6
|
+
from grain._src.core import parallel
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def ReturnThreadName():
|
|
10
|
+
return threading.current_thread().name
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def Identity(i):
|
|
14
|
+
return i
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def FnThatAlwaysFails(arg):
|
|
18
|
+
del arg
|
|
19
|
+
raise ValueError("I always fail")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def FnThatFailsOnOddInputs(i):
|
|
23
|
+
if i % 2 == 1:
|
|
24
|
+
raise ValueError("Failed on an odd input")
|
|
25
|
+
return i
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ParallelTest(parameterized.TestCase):
|
|
29
|
+
|
|
30
|
+
@parameterized.named_parameters(
|
|
31
|
+
dict(
|
|
32
|
+
testcase_name=" empty list of args",
|
|
33
|
+
num_workers=1,
|
|
34
|
+
input_dict_list=[],
|
|
35
|
+
expected=[],
|
|
36
|
+
),
|
|
37
|
+
dict(
|
|
38
|
+
testcase_name=" one worker, nonempty list",
|
|
39
|
+
num_workers=1,
|
|
40
|
+
input_dict_list=[dict(i=k) for k in range(1, 10)],
|
|
41
|
+
expected=list(range(1, 10)),
|
|
42
|
+
),
|
|
43
|
+
dict(
|
|
44
|
+
testcase_name=" fewer workers than jobs, nonempty list",
|
|
45
|
+
num_workers=3,
|
|
46
|
+
input_dict_list=[dict(i=k) for k in range(1, 10)],
|
|
47
|
+
expected=list(range(1, 10)),
|
|
48
|
+
),
|
|
49
|
+
dict(
|
|
50
|
+
testcase_name=" more workers than jobs, nonempty list",
|
|
51
|
+
num_workers=20,
|
|
52
|
+
input_dict_list=[dict(i=k) for k in range(1, 10)],
|
|
53
|
+
expected=list(range(1, 10)),
|
|
54
|
+
),
|
|
55
|
+
)
|
|
56
|
+
def testRunInParallel(self, input_dict_list, num_workers: int, expected):
|
|
57
|
+
actual = parallel.run_in_parallel(Identity, input_dict_list, num_workers)
|
|
58
|
+
self.assertEqual(actual, expected)
|
|
59
|
+
|
|
60
|
+
def testRunInParallelOnAlwaysFailingFn(self):
|
|
61
|
+
with self.assertRaisesRegex(ValueError, "I always fail"):
|
|
62
|
+
parallel.run_in_parallel(FnThatAlwaysFails, [dict(arg="hi")], 10)
|
|
63
|
+
|
|
64
|
+
@parameterized.named_parameters(
|
|
65
|
+
dict(
|
|
66
|
+
testcase_name=" one failing input, one worker",
|
|
67
|
+
num_workers=1,
|
|
68
|
+
input_dict_list=[{"i": 1}],
|
|
69
|
+
),
|
|
70
|
+
dict(
|
|
71
|
+
testcase_name=" one failing input, many workers",
|
|
72
|
+
num_workers=5,
|
|
73
|
+
input_dict_list=[{"i": 1}],
|
|
74
|
+
),
|
|
75
|
+
dict(
|
|
76
|
+
testcase_name=" one failing input, one succeeding input",
|
|
77
|
+
num_workers=5,
|
|
78
|
+
input_dict_list=[{"i": 1}, {"i": 2}],
|
|
79
|
+
),
|
|
80
|
+
dict(
|
|
81
|
+
testcase_name=" two failing inputs, one succeeding input",
|
|
82
|
+
num_workers=5,
|
|
83
|
+
input_dict_list=[{"i": 1}, {"i": 2}, {"i": 3}],
|
|
84
|
+
),
|
|
85
|
+
)
|
|
86
|
+
def testRunInParallelFailsIfSomeFnCallsFail(
|
|
87
|
+
self, input_dict_list, num_workers: int
|
|
88
|
+
):
|
|
89
|
+
with self.assertRaisesRegex(ValueError, "Failed on an odd input"):
|
|
90
|
+
parallel.run_in_parallel(
|
|
91
|
+
FnThatFailsOnOddInputs, input_dict_list, num_workers
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
def testRunInParallelForThreadNamePrefix(self):
|
|
95
|
+
input_kwarg_list = [{}]
|
|
96
|
+
thread_names = parallel.run_in_parallel(
|
|
97
|
+
ReturnThreadName, input_kwarg_list, 5, thread_name_prefix="Customized-"
|
|
98
|
+
)
|
|
99
|
+
self.assertStartsWith(thread_names[0], "Customized-")
|
|
100
|
+
|
|
101
|
+
def testRunInParallelForDefaultThreadNamePrefix(self):
|
|
102
|
+
input_kwarg_list = [{}]
|
|
103
|
+
thread_names = parallel.run_in_parallel(
|
|
104
|
+
ReturnThreadName, input_kwarg_list, 5
|
|
105
|
+
)
|
|
106
|
+
self.assertStartsWith(thread_names[0], "parallel_")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
if __name__ == "__main__":
|
|
110
|
+
absltest.main()
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Copyright 2022 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Classes for handling sharding of data sources arcoss machines/VMs."""
|
|
15
|
+
import dataclasses
|
|
16
|
+
|
|
17
|
+
from absl import logging
|
|
18
|
+
import jax
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclasses.dataclass(frozen=True)
|
|
22
|
+
class ShardOptions:
|
|
23
|
+
"""Dataclass to hold options for sharding a data source.
|
|
24
|
+
|
|
25
|
+
Attributes:
|
|
26
|
+
shard_index: The index of the shard to use in this process. Must be in [0,
|
|
27
|
+
shard_count - 1].
|
|
28
|
+
shard_count: The total number of shards.
|
|
29
|
+
drop_remainder: If True shard() will create even splits and drop the
|
|
30
|
+
remainder examples (all shards will have the same number of examples). If
|
|
31
|
+
False will distribute the remainder N over the first N shards.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
shard_index: int
|
|
35
|
+
shard_count: int
|
|
36
|
+
drop_remainder: bool = False
|
|
37
|
+
|
|
38
|
+
def __post_init__(self):
|
|
39
|
+
if self.shard_count <= 0:
|
|
40
|
+
raise ValueError(
|
|
41
|
+
"Number of shards must be a positive integer but got "
|
|
42
|
+
f"{self.shard_count}."
|
|
43
|
+
)
|
|
44
|
+
if self.shard_index < 0 or self.shard_index >= self.shard_count:
|
|
45
|
+
raise ValueError(
|
|
46
|
+
"Shard shard_index must be in [0, shard_count - 1], shard_count was "
|
|
47
|
+
f"{self.shard_count} and shard_index was {self.shard_index}."
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class NoSharding(ShardOptions):
|
|
52
|
+
"""Doesn't shard data. Each process will load all data."""
|
|
53
|
+
|
|
54
|
+
def __init__(self):
|
|
55
|
+
super().__init__(shard_index=0, shard_count=1, drop_remainder=False)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ShardByJaxProcess(ShardOptions):
|
|
59
|
+
"""Shards the data across JAX processes."""
|
|
60
|
+
|
|
61
|
+
def __init__(self, drop_remainder: bool = False):
|
|
62
|
+
super().__init__(
|
|
63
|
+
shard_index=jax.process_index(),
|
|
64
|
+
shard_count=jax.process_count(),
|
|
65
|
+
drop_remainder=drop_remainder,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def even_split(num_examples: int, options: ShardOptions) -> tuple[int, int]:
|
|
70
|
+
"""Returns the interval for the shard when sharding `num_examples` evenly.
|
|
71
|
+
|
|
72
|
+
This splits the interval [0, num_examples - 1] into `shard_count` intervals
|
|
73
|
+
and returns the `shard_index`'s interval. If `drop_remainder` is True all
|
|
74
|
+
intervals will have the same size.
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
num_examples: Number of examples to shard.
|
|
78
|
+
options: Options for sharding the data in this process.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
Tuple with the start and end of the interval. The start is the first
|
|
82
|
+
example that should be included in this interval and end - 1 is the last
|
|
83
|
+
example to be include in the shard.
|
|
84
|
+
"""
|
|
85
|
+
examples_per_shard = num_examples // options.shard_count
|
|
86
|
+
shard_start = examples_per_shard * options.shard_index
|
|
87
|
+
shard_end = examples_per_shard * (options.shard_index + 1)
|
|
88
|
+
|
|
89
|
+
# Handle remaining examples.
|
|
90
|
+
num_unused_examples = num_examples % options.shard_count
|
|
91
|
+
|
|
92
|
+
if num_unused_examples > 0:
|
|
93
|
+
if options.drop_remainder:
|
|
94
|
+
logging.warning(
|
|
95
|
+
"Dropping %d examples of %d examples (shard %d).",
|
|
96
|
+
num_unused_examples,
|
|
97
|
+
num_examples,
|
|
98
|
+
options.shard_count,
|
|
99
|
+
)
|
|
100
|
+
else:
|
|
101
|
+
shard_start += min(options.shard_index, num_unused_examples)
|
|
102
|
+
shard_end += min(options.shard_index + 1, num_unused_examples)
|
|
103
|
+
return shard_start, shard_end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Tests for sharding."""
|
|
2
|
+
from absl.testing import absltest
|
|
3
|
+
from absl.testing import parameterized
|
|
4
|
+
from grain._src.core import sharding
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ShardingTest(parameterized.TestCase):
|
|
8
|
+
|
|
9
|
+
@parameterized.parameters(
|
|
10
|
+
# num_examples, shard_index, shard_count, drop_remainder, expected_output.
|
|
11
|
+
(9, 0, 1, True, (0, 9)),
|
|
12
|
+
(9, 0, 2, True, (0, 4)),
|
|
13
|
+
(9, 1, 2, True, (4, 8)), # Last example gets dropped.
|
|
14
|
+
(9, 0, 3, True, (0, 3)),
|
|
15
|
+
(9, 1, 3, True, (3, 6)),
|
|
16
|
+
(9, 2, 3, True, (6, 9)),
|
|
17
|
+
(9, 0, 1, False, (0, 9)),
|
|
18
|
+
(9, 0, 2, False, (0, 5)), # First shard gets an extra example.
|
|
19
|
+
(9, 1, 2, False, (5, 9)),
|
|
20
|
+
(8, 0, 3, False, (0, 3)), # First 2 shards get 1 example each.
|
|
21
|
+
(8, 1, 3, False, (3, 6)),
|
|
22
|
+
(8, 2, 3, False, (6, 8)),
|
|
23
|
+
)
|
|
24
|
+
def test_sharding(
|
|
25
|
+
self,
|
|
26
|
+
num_examples: int,
|
|
27
|
+
shard_index: int,
|
|
28
|
+
shard_count: int,
|
|
29
|
+
drop_remainder,
|
|
30
|
+
expected_output: tuple[int, int],
|
|
31
|
+
):
|
|
32
|
+
shard_options = sharding.ShardOptions(
|
|
33
|
+
shard_index=shard_index,
|
|
34
|
+
shard_count=shard_count,
|
|
35
|
+
drop_remainder=drop_remainder,
|
|
36
|
+
)
|
|
37
|
+
actual_output = sharding.even_split(num_examples, shard_options)
|
|
38
|
+
self.assertEqual(actual_output, expected_output)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
if __name__ == '__main__':
|
|
42
|
+
absltest.main()
|