grain-nightly 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of grain-nightly might be problematic. Click here for more details.

Files changed (65) hide show
  1. grain/_src/core/config.py +109 -0
  2. grain/_src/core/constants.py +39 -0
  3. grain/_src/core/exceptions.py +5 -0
  4. grain/_src/core/grain_random.py +126 -0
  5. grain/_src/core/grain_random_test.py +83 -0
  6. grain/_src/core/parallel.py +72 -0
  7. grain/_src/core/parallel_test.py +110 -0
  8. grain/_src/core/sharding.py +103 -0
  9. grain/_src/core/sharding_test.py +42 -0
  10. grain/_src/core/transforms.py +112 -0
  11. grain/_src/core/usage_logging.py +18 -0
  12. grain/_src/python/checkpoint_handlers.py +54 -0
  13. grain/_src/python/data_loader.py +679 -0
  14. grain/_src/python/data_loader_test.py +578 -0
  15. grain/_src/python/data_sources.py +242 -0
  16. grain/_src/python/data_sources_test.py +184 -0
  17. grain/_src/python/experimental/example_packing/packing.py +204 -0
  18. grain/_src/python/experimental/example_packing/packing_test.py +490 -0
  19. grain/_src/python/experimental/index_shuffle/libindex_shuffle.so +0 -0
  20. grain/_src/python/experimental/index_shuffle/python/index_shuffle_module.so +0 -0
  21. grain/_src/python/experimental/index_shuffle/python/index_shuffle_test.py +42 -0
  22. grain/_src/python/grain_pool.py +373 -0
  23. grain/_src/python/grain_pool_test.py +153 -0
  24. grain/_src/python/lazy_dataset/data_sources.py +49 -0
  25. grain/_src/python/lazy_dataset/data_sources_test.py +66 -0
  26. grain/_src/python/lazy_dataset/grain_pool_v2.py +334 -0
  27. grain/_src/python/lazy_dataset/grain_pool_v2_test.py +233 -0
  28. grain/_src/python/lazy_dataset/lazy_dataset.py +288 -0
  29. grain/_src/python/lazy_dataset/lazy_dataset_test.py +116 -0
  30. grain/_src/python/lazy_dataset/transformations/batch.py +136 -0
  31. grain/_src/python/lazy_dataset/transformations/batch_test.py +76 -0
  32. grain/_src/python/lazy_dataset/transformations/filter.py +120 -0
  33. grain/_src/python/lazy_dataset/transformations/filter_test.py +125 -0
  34. grain/_src/python/lazy_dataset/transformations/flatmap.py +62 -0
  35. grain/_src/python/lazy_dataset/transformations/flatmap_test.py +127 -0
  36. grain/_src/python/lazy_dataset/transformations/map.py +214 -0
  37. grain/_src/python/lazy_dataset/transformations/map_test.py +134 -0
  38. grain/_src/python/lazy_dataset/transformations/mix.py +148 -0
  39. grain/_src/python/lazy_dataset/transformations/mix_test.py +75 -0
  40. grain/_src/python/lazy_dataset/transformations/ragged_batch.py +51 -0
  41. grain/_src/python/lazy_dataset/transformations/ragged_batch_test.py +106 -0
  42. grain/_src/python/lazy_dataset/transformations/shuffle.py +47 -0
  43. grain/_src/python/lazy_dataset/transformations/shuffle_test.py +63 -0
  44. grain/_src/python/lazy_dataset/transformations/slice.py +38 -0
  45. grain/_src/python/lazy_dataset/transformations/slice_test.py +39 -0
  46. grain/_src/python/load.py +65 -0
  47. grain/_src/python/load_test.py +74 -0
  48. grain/_src/python/multiprocessing_common.py +73 -0
  49. grain/_src/python/multiprocessing_common_test.py +94 -0
  50. grain/_src/python/operations.py +205 -0
  51. grain/_src/python/operations_test.py +393 -0
  52. grain/_src/python/options.py +60 -0
  53. grain/_src/python/record.py +53 -0
  54. grain/_src/python/record_test.py +41 -0
  55. grain/_src/python/samplers.py +156 -0
  56. grain/_src/python/samplers_test.py +657 -0
  57. grain/core.py +22 -0
  58. grain/python.py +63 -0
  59. grain/python_experimental.py +23 -0
  60. grain/python_lazy_dataset.py +45 -0
  61. grain_nightly-0.0.1.dist-info/LICENSE +201 -0
  62. grain_nightly-0.0.1.dist-info/METADATA +228 -0
  63. grain_nightly-0.0.1.dist-info/RECORD +65 -0
  64. grain_nightly-0.0.1.dist-info/WHEEL +5 -0
  65. grain_nightly-0.0.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,109 @@
1
+ # Copyright 2022 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Handle Grain config options.
15
+
16
+ Config options can be set via flags starting with '--grain_' or by calling
17
+ `grain.config.update(name, value)`.
18
+ """
19
+ from typing import Any
20
+
21
+ from absl import flags
22
+
23
+ # Performance optimisations. Consider most of these experimental. We might
24
+ # remove them once we are confident that the default values work well for
25
+ # everyone.
26
+ _INTERLEAVED_SHUFFLE = flags.DEFINE_bool(
27
+ "grain_tf_interleaved_shuffle",
28
+ False,
29
+ (
30
+ "If enabled replace the global shuffle with an approximation that "
31
+ "interleaves small blocks. This can increase throughput of the input "
32
+ "pipeline. See index_dataset._interleaved_shuffle() for details."
33
+ ),
34
+ )
35
+ _INTERLEAVED_SHUFFLE_BLOCK_SIZE = flags.DEFINE_integer(
36
+ "grain_tf_interleaved_shuffle_block_size",
37
+ 10,
38
+ "Block size in index_dataset._interleaved_shuffle().",
39
+ )
40
+ _INTERLEAVED_SHUFFLE_NUM_BLOCKS_PER_PARTITION = flags.DEFINE_integer(
41
+ "grain_tf_interleaved_shuffle_parallel_blocks",
42
+ 10,
43
+ "Number of blocks per partition in index_dataset._interleaved_shuffle().",
44
+ )
45
+
46
+ _LOOKUP_BATCH_SIZE = flags.DEFINE_integer(
47
+ "grain_tf_lookup_batch_size",
48
+ 100,
49
+ (
50
+ "Number of keys to batch into a single lookup call."
51
+ " --grain_tf_num_parallel_calls controls on how many lookups to perform"
52
+ " in parallel."
53
+ ),
54
+ )
55
+ _LOOKUP_NUM_PARALLEL_CALLS = flags.DEFINE_integer(
56
+ "grain_tf_lookup_num_parallel_calls",
57
+ -1,
58
+ (
59
+ "Number of parallel lookup calls. Each lookup will read"
60
+ " --grain_tf_lookup_batch_size records. This must be a positive integer"
61
+ " > 0 or -1. If -1 Grain will use tf.data.AUTOTUNE to tune the value"
62
+ " automatically."
63
+ ),
64
+ )
65
+ _LOOKUP_FAST_WARMUP = flags.DEFINE_bool(
66
+ "grain_tf_lookup_fast_warmup",
67
+ False,
68
+ (
69
+ "Deprecated. If True will split up the first batch lookup into smaller"
70
+ " chunks. This can help to provide initial elements fast while allowing"
71
+ " when using a large tf_lookup_batch_size (which is usually better for"
72
+ " throughput)."
73
+ ),
74
+ )
75
+
76
+ _GRAIN_FLAGS = (
77
+ _INTERLEAVED_SHUFFLE,
78
+ _INTERLEAVED_SHUFFLE_BLOCK_SIZE,
79
+ _INTERLEAVED_SHUFFLE_NUM_BLOCKS_PER_PARTITION,
80
+ _LOOKUP_BATCH_SIZE,
81
+ _LOOKUP_NUM_PARALLEL_CALLS,
82
+ _LOOKUP_FAST_WARMUP,
83
+ )
84
+
85
+
86
+ class Config:
87
+ """Class for holding current Grain configuration."""
88
+
89
+ # Loosen the static type checking requirements.
90
+ _HAS_DYNAMIC_ATTRIBUTES = True
91
+
92
+ def __getattr__(self, name: str) -> Any:
93
+ flag_name = f"grain_{name}"
94
+ if any(f.name == flag_name for f in _GRAIN_FLAGS):
95
+ return getattr(flags.FLAGS, flag_name)
96
+ raise ValueError(f"Unrecognized config option: {name}")
97
+
98
+ def __setattr__(self, name: str, value: Any):
99
+ raise ValueError("Please use update().")
100
+
101
+ def update(self, name: str, value: Any):
102
+ flag_name = f"grain_{name}"
103
+ if any(f.name == flag_name for f in _GRAIN_FLAGS):
104
+ setattr(flags.FLAGS, flag_name, value)
105
+ return
106
+ raise ValueError(f"Unrecognized config option: {name}")
107
+
108
+
109
+ config = Config()
@@ -0,0 +1,39 @@
1
+ # Copyright 2022 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Shared constants for various Grain APIs."""
15
+
16
+ # Below are names of meta features used by index_dataset (and pipelines building
17
+ # on top of it). These features are generated on the fly and help to track
18
+ # progress over the dataset. Users can read these but shouldn't alter them. They
19
+ # start with "_" to indicate that they are "private".
20
+ # Index into the stream of all records (globally unique). Starts with 0.
21
+ INDEX = "_index"
22
+ # Key of the record. If DATASET_INDEX is present it's the key in the dataset.
23
+ # Starts with 0.
24
+ RECORD_KEY = "_record_key"
25
+ # Index of the dataset from which to take the record. Only present when mixing.
26
+ # Starts with 0.
27
+ DATASET_INDEX = "_dataset_index"
28
+ # Epoch for the record. When mixing datasets this is the epoch over the dataset,
29
+ # not the mixture. Starts with 1.
30
+ EPOCH = "_epoch"
31
+ # Random seed for stateless random operations. This is unique per record
32
+ # and changes every epoch.
33
+ SEED = "_seed"
34
+ # Serialized record.
35
+ RECORD = "_record"
36
+
37
+ META_FEATURES = frozenset(
38
+ [INDEX, RECORD_KEY, DATASET_INDEX, EPOCH, SEED, RECORD]
39
+ )
@@ -0,0 +1,5 @@
1
+ """Custom exception for PyGrain-specific internal errors."""
2
+
3
+
4
+ class PyGrainInternalError(Exception):
5
+ pass
@@ -0,0 +1,126 @@
1
+ # Copyright 2022 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Type definition and methods for handling random numbers.
15
+
16
+ Grain uses counter-based pseudorandom number generators (PRNGs) for fast and
17
+ reproducible random numbers.
18
+ This means that there is no global random seed or a global stateful PRNGs
19
+ object. Instead, random seeds are always passed explicitly and users should
20
+ understand the rules outlined below (this applies both to the internals of
21
+ Grain as well as user defined preprocessing and extensions).
22
+
23
+ ## Motivation
24
+ We want Grain input pipelines to be reproducible while still being highly
25
+ parallelizable. This very much overlaps with the design choices for JAX's PRNG
26
+ ([design notes](github.com/google/jax/blob/main/docs/design_notes/prng.md)) and
27
+ their choice for counter-based PRNGs like ThreeFry and Philox.
28
+ NumPy, JAX and TF all contain implementations of ThreeFry/Philox.
29
+
30
+ ## General Rules
31
+ If you are not familiar with `jax.random`, we recommend reading
32
+ [the JAX
33
+ documentation](https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#random-numbers)
34
+ or
35
+ [this
36
+ Colab](https://colab.research.google.com/github/google/jax/blob/main/docs/jax-101/05-random-numbers.ipynb).
37
+ to understand the concepts of counter-based PRNGs. You should be comfortable
38
+ with splitting your random seeds.
39
+
40
+ After that, just follow these rules:
41
+ - Never use the `random` module from the Python standard library.
42
+ - Either use methods in `jax.random`, random methods in TensorFlow that start
43
+ with `stateless_` (e.g. `tf.random.stateless_uniform()`) or non-global
44
+ NumPy random generators.
45
+ - Do not use the global RNG in `np.random`. Instead create a new generator
46
+ with appropiate bit generator: `np.random.default_rng(seed)` or
47
+ `np.random.Generator(np.random.Philox(seed))` and call random functions on it.
48
+ The generator (or more accurately the underlying bit generator) is stateful
49
+ and should not be used across between threads.
50
+ - If you want to use a new random seed each time your program starts, you can
51
+ use `int.from_bytes(os.urandom(4), sys.byteorder)` and log the value.
52
+ - Do not re-use a random seed. Instead, split the random seed using
53
+ `jax.random.split()` or `tf.random.experimental.stateless_split()`
54
+
55
+ ## JAX
56
+ JAX supports multiple RNG implementations. Since Grain operations run on CPU,
57
+ we recommend the default (ThreeFry). It's stable across JAX/XLA versions.
58
+ Internally, ThreeFry uses a uint32 type vector of shape [2] as state.
59
+
60
+ ## NumPy
61
+ NumPy contains several suitable RNG implementations (aka bit generators). We
62
+ recommend using `np.random.Philox` which is very similar to the ThreeFry
63
+ implementation in JAX and TF. However `PCG64` (default as of writting) and
64
+ `PCG64DXSM` are also suitable for input pipelines. Just don't use the global
65
+ RNG and always construct your own `np.random.Generator`.
66
+
67
+ ## TensorFlow
68
+ TensorFlow implements multiple counter-based RNGs (ThreeFry and Philox) but
69
+ doesn't provide any guidance on which one to use. We recommend to not set the
70
+ algorithm explicitly and let TF select the RNG (on CPUs it seems to select
71
+ Philox). You can derive a random seed for stateless random functions using
72
+ `tf.random.Generator.from_seed(my_integer).make_seeds(1)[:, 0]`.
73
+ Splitting seeds can be done using:
74
+ ```
75
+ seed, seed_for_my_op = tf.unstack(tf.random.experimental.stateless_split(seed))
76
+ ```
77
+ """
78
+ import os
79
+ import sys
80
+ from typing import Optional, Sequence, Union
81
+
82
+ from absl import logging
83
+ import jax
84
+ import jax.numpy as jnp
85
+ import numpy as np
86
+
87
+ # As of 2022-09 JAX uses arrays of dtype uint32 and shape [2] for RNG keys.
88
+ # However when custom RNGs are enabled by default keys are wrapped in a KeyArray
89
+ # object and can have different internal state. This might become the default
90
+ # in the future.
91
+ RNGKey = jax.random.PRNGKeyArray
92
+ RNGKeyLike = Union[RNGKey, int, Sequence[int], np.ndarray]
93
+
94
+
95
+ def as_rng_key(seed: RNGKeyLike) -> RNGKey:
96
+ """Returns an RNGKey from the input.
97
+
98
+ Args:
99
+ seed: Rndom seed as single integer or tuple/array of 2 integers.
100
+
101
+ Returns:
102
+ RNGKey which is currently a NumPy array of dtype uint32 and shape [2].
103
+ """
104
+ if isinstance(seed, jax.random.PRNGKeyArray):
105
+ return seed
106
+ if isinstance(seed, (int, jnp.integer)):
107
+ return jax.random.PRNGKey(seed)
108
+ if isinstance(seed, jnp.ndarray):
109
+ # Assume that this is already a KeyArray.
110
+ return seed
111
+ if len(seed) != 2:
112
+ raise ValueError(
113
+ "Random seed must be a single integer or a "
114
+ f"Tuple[int, int] but got {seed!r}"
115
+ )
116
+ return jax.random.PRNGKey(sum(seed))
117
+
118
+
119
+ def make_rng_key(seed: Optional[RNGKeyLike]) -> RNGKey:
120
+ logging.error(
121
+ "Using deprecated method make_rng_key(). Please always pass "
122
+ "None or a single integer as random seed to Grain methods."
123
+ )
124
+ if seed is None:
125
+ seed = int.from_bytes(os.urandom(4), sys.byteorder)
126
+ return as_rng_key(seed)
@@ -0,0 +1,83 @@
1
+ # Copyright 2022 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Unit tests for the random module."""
15
+ import contextlib
16
+ import itertools
17
+
18
+ from absl.testing import parameterized
19
+ from grain._src.core import grain_random
20
+ import jax
21
+ import numpy as np
22
+ import tensorflow as tf
23
+
24
+
25
+ class RandomTest(tf.test.TestCase, parameterized.TestCase):
26
+ """Tests for the random module."""
27
+
28
+ @parameterized.parameters(
29
+ {"seed": 585},
30
+ {"seed": (5424, 95849)},
31
+ {"seed": np.asarray([5424, 95849])},
32
+ {"seed": np.int32(95849)},
33
+ {"seed": np.int64(5424)},
34
+ )
35
+ def test_rng_for_jax(self, seed):
36
+ seed = grain_random.as_rng_key(seed)
37
+ # Verify we can split the seed.
38
+ seed, _ = jax.random.split(seed)
39
+ # Verify we can generate a random number.
40
+ jax.random.uniform(seed, [])
41
+
42
+ @parameterized.parameters(
43
+ itertools.product(
44
+ [585, (5424, 95849), np.asarray([5424, 95849])],
45
+ [
46
+ # No context.
47
+ (None, None),
48
+ # Each separately.
49
+ ("custom", None),
50
+ ("threefry2x32", None),
51
+ ("rbg", None),
52
+ # implementation frist, custom afterwards.
53
+ ("threefry2x32", "custom"),
54
+ ("rbg", "custom"),
55
+ # custom first, implementation afterwards.
56
+ ("custom", "threefry2x32"),
57
+ ("custom", "rbg"),
58
+ ],
59
+ )
60
+ )
61
+ def test_rng_for_jax_custom(self, seed, contexts):
62
+ def get_ctx(name):
63
+ if name is None:
64
+ return contextlib.nullcontext()
65
+ if name == "threefry2x32":
66
+ return jax.default_prng_impl("threefry2x32")
67
+ if name == "rbg":
68
+ return jax.default_prng_impl("rbg")
69
+ if name == "custom":
70
+ return jax.enable_custom_prng()
71
+ assert False
72
+
73
+ with get_ctx(contexts[0]):
74
+ with get_ctx(contexts[1]):
75
+ seed = grain_random.as_rng_key(seed)
76
+ # Verify we can split the seed.
77
+ seed, _ = jax.random.split(seed)
78
+ # Verify we can generate a random number.
79
+ jax.random.uniform(seed, [])
80
+
81
+
82
+ if __name__ == "__main__":
83
+ tf.test.main()
@@ -0,0 +1,72 @@
1
+ """Provides a methods to run functions in parallel using a thread pool."""
2
+ from collections.abc import Mapping, Sequence
3
+ from typing import Any, Callable, TypeVar
4
+
5
+ from concurrent import futures
6
+
7
+ T = TypeVar("T")
8
+
9
+
10
+ def run_in_parallel(
11
+ function: Callable[..., T],
12
+ list_of_kwargs_to_function: Sequence[Mapping[str, Any]],
13
+ num_workers: int,
14
+ thread_name_prefix: str = "parallel_",
15
+ ) -> list[T]:
16
+ """Run a function on a list of kwargs in parallel with ThreadPoolExecutor.
17
+
18
+ Works best when there is IO boundedness, not when there is CPU boundedness, as
19
+ the threads used are bound by the GIL.
20
+
21
+ Propagates first exception to the calling thread. If cancel_futures=True,
22
+ then stop as many of the ongoing work units as possible.
23
+
24
+ Example usage:
25
+ def io_bound_function(p):
26
+ get_contents_from_cns(p)
27
+
28
+ run_in_parallel(
29
+ function=io_bound_function,
30
+ list_of_kwargs_to_function=[{"p": p} for p in long_list_of_paths],
31
+ num_workers=3)
32
+
33
+ Args:
34
+ function: a function.
35
+ list_of_kwargs_to_function: A list of dicts mapping from string to argument
36
+ value. These will be passed into `function` as kwargs.
37
+ num_workers: int.
38
+ thread_name_prefix: The thread name prefix string. Processes are run in
39
+ threads, and each thread is named. This parameter allows the user to
40
+ control the prefix for that thread name.
41
+
42
+ Returns:
43
+ list of return values from function, in the same order as the arguments in
44
+ list_of_kwargs_to_function.
45
+ """
46
+ if num_workers < 1:
47
+ raise ValueError(
48
+ "Number of workers must be greater than 0. Was {}".format(num_workers)
49
+ )
50
+
51
+ thread_name = thread_name_prefix + getattr(function, "__name__", "unknown")
52
+ with futures.ThreadPoolExecutor(
53
+ num_workers, thread_name_prefix=thread_name
54
+ ) as executor:
55
+ fs = []
56
+
57
+ for kwargs in list_of_kwargs_to_function:
58
+ f = executor.submit(function, **kwargs)
59
+ fs.append(f)
60
+
61
+ futures_as_completed = futures.as_completed(fs)
62
+
63
+ for completed in futures_as_completed:
64
+ if completed.exception():
65
+ # Cancel all remaining futures, if possible.
66
+ for remaining_future in fs:
67
+ remaining_future.cancel()
68
+
69
+ # Propagate exception to main thread.
70
+ raise completed.exception()
71
+
72
+ return [f.result() for f in fs]
@@ -0,0 +1,110 @@
1
+ """Tests for parallel."""
2
+ import threading
3
+
4
+ from absl.testing import absltest
5
+ from absl.testing import parameterized
6
+ from grain._src.core import parallel
7
+
8
+
9
+ def ReturnThreadName():
10
+ return threading.current_thread().name
11
+
12
+
13
+ def Identity(i):
14
+ return i
15
+
16
+
17
+ def FnThatAlwaysFails(arg):
18
+ del arg
19
+ raise ValueError("I always fail")
20
+
21
+
22
+ def FnThatFailsOnOddInputs(i):
23
+ if i % 2 == 1:
24
+ raise ValueError("Failed on an odd input")
25
+ return i
26
+
27
+
28
+ class ParallelTest(parameterized.TestCase):
29
+
30
+ @parameterized.named_parameters(
31
+ dict(
32
+ testcase_name=" empty list of args",
33
+ num_workers=1,
34
+ input_dict_list=[],
35
+ expected=[],
36
+ ),
37
+ dict(
38
+ testcase_name=" one worker, nonempty list",
39
+ num_workers=1,
40
+ input_dict_list=[dict(i=k) for k in range(1, 10)],
41
+ expected=list(range(1, 10)),
42
+ ),
43
+ dict(
44
+ testcase_name=" fewer workers than jobs, nonempty list",
45
+ num_workers=3,
46
+ input_dict_list=[dict(i=k) for k in range(1, 10)],
47
+ expected=list(range(1, 10)),
48
+ ),
49
+ dict(
50
+ testcase_name=" more workers than jobs, nonempty list",
51
+ num_workers=20,
52
+ input_dict_list=[dict(i=k) for k in range(1, 10)],
53
+ expected=list(range(1, 10)),
54
+ ),
55
+ )
56
+ def testRunInParallel(self, input_dict_list, num_workers: int, expected):
57
+ actual = parallel.run_in_parallel(Identity, input_dict_list, num_workers)
58
+ self.assertEqual(actual, expected)
59
+
60
+ def testRunInParallelOnAlwaysFailingFn(self):
61
+ with self.assertRaisesRegex(ValueError, "I always fail"):
62
+ parallel.run_in_parallel(FnThatAlwaysFails, [dict(arg="hi")], 10)
63
+
64
+ @parameterized.named_parameters(
65
+ dict(
66
+ testcase_name=" one failing input, one worker",
67
+ num_workers=1,
68
+ input_dict_list=[{"i": 1}],
69
+ ),
70
+ dict(
71
+ testcase_name=" one failing input, many workers",
72
+ num_workers=5,
73
+ input_dict_list=[{"i": 1}],
74
+ ),
75
+ dict(
76
+ testcase_name=" one failing input, one succeeding input",
77
+ num_workers=5,
78
+ input_dict_list=[{"i": 1}, {"i": 2}],
79
+ ),
80
+ dict(
81
+ testcase_name=" two failing inputs, one succeeding input",
82
+ num_workers=5,
83
+ input_dict_list=[{"i": 1}, {"i": 2}, {"i": 3}],
84
+ ),
85
+ )
86
+ def testRunInParallelFailsIfSomeFnCallsFail(
87
+ self, input_dict_list, num_workers: int
88
+ ):
89
+ with self.assertRaisesRegex(ValueError, "Failed on an odd input"):
90
+ parallel.run_in_parallel(
91
+ FnThatFailsOnOddInputs, input_dict_list, num_workers
92
+ )
93
+
94
+ def testRunInParallelForThreadNamePrefix(self):
95
+ input_kwarg_list = [{}]
96
+ thread_names = parallel.run_in_parallel(
97
+ ReturnThreadName, input_kwarg_list, 5, thread_name_prefix="Customized-"
98
+ )
99
+ self.assertStartsWith(thread_names[0], "Customized-")
100
+
101
+ def testRunInParallelForDefaultThreadNamePrefix(self):
102
+ input_kwarg_list = [{}]
103
+ thread_names = parallel.run_in_parallel(
104
+ ReturnThreadName, input_kwarg_list, 5
105
+ )
106
+ self.assertStartsWith(thread_names[0], "parallel_")
107
+
108
+
109
+ if __name__ == "__main__":
110
+ absltest.main()
@@ -0,0 +1,103 @@
1
+ # Copyright 2022 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Classes for handling sharding of data sources arcoss machines/VMs."""
15
+ import dataclasses
16
+
17
+ from absl import logging
18
+ import jax
19
+
20
+
21
+ @dataclasses.dataclass(frozen=True)
22
+ class ShardOptions:
23
+ """Dataclass to hold options for sharding a data source.
24
+
25
+ Attributes:
26
+ shard_index: The index of the shard to use in this process. Must be in [0,
27
+ shard_count - 1].
28
+ shard_count: The total number of shards.
29
+ drop_remainder: If True shard() will create even splits and drop the
30
+ remainder examples (all shards will have the same number of examples). If
31
+ False will distribute the remainder N over the first N shards.
32
+ """
33
+
34
+ shard_index: int
35
+ shard_count: int
36
+ drop_remainder: bool = False
37
+
38
+ def __post_init__(self):
39
+ if self.shard_count <= 0:
40
+ raise ValueError(
41
+ "Number of shards must be a positive integer but got "
42
+ f"{self.shard_count}."
43
+ )
44
+ if self.shard_index < 0 or self.shard_index >= self.shard_count:
45
+ raise ValueError(
46
+ "Shard shard_index must be in [0, shard_count - 1], shard_count was "
47
+ f"{self.shard_count} and shard_index was {self.shard_index}."
48
+ )
49
+
50
+
51
+ class NoSharding(ShardOptions):
52
+ """Doesn't shard data. Each process will load all data."""
53
+
54
+ def __init__(self):
55
+ super().__init__(shard_index=0, shard_count=1, drop_remainder=False)
56
+
57
+
58
+ class ShardByJaxProcess(ShardOptions):
59
+ """Shards the data across JAX processes."""
60
+
61
+ def __init__(self, drop_remainder: bool = False):
62
+ super().__init__(
63
+ shard_index=jax.process_index(),
64
+ shard_count=jax.process_count(),
65
+ drop_remainder=drop_remainder,
66
+ )
67
+
68
+
69
+ def even_split(num_examples: int, options: ShardOptions) -> tuple[int, int]:
70
+ """Returns the interval for the shard when sharding `num_examples` evenly.
71
+
72
+ This splits the interval [0, num_examples - 1] into `shard_count` intervals
73
+ and returns the `shard_index`'s interval. If `drop_remainder` is True all
74
+ intervals will have the same size.
75
+
76
+ Args:
77
+ num_examples: Number of examples to shard.
78
+ options: Options for sharding the data in this process.
79
+
80
+ Returns:
81
+ Tuple with the start and end of the interval. The start is the first
82
+ example that should be included in this interval and end - 1 is the last
83
+ example to be include in the shard.
84
+ """
85
+ examples_per_shard = num_examples // options.shard_count
86
+ shard_start = examples_per_shard * options.shard_index
87
+ shard_end = examples_per_shard * (options.shard_index + 1)
88
+
89
+ # Handle remaining examples.
90
+ num_unused_examples = num_examples % options.shard_count
91
+
92
+ if num_unused_examples > 0:
93
+ if options.drop_remainder:
94
+ logging.warning(
95
+ "Dropping %d examples of %d examples (shard %d).",
96
+ num_unused_examples,
97
+ num_examples,
98
+ options.shard_count,
99
+ )
100
+ else:
101
+ shard_start += min(options.shard_index, num_unused_examples)
102
+ shard_end += min(options.shard_index + 1, num_unused_examples)
103
+ return shard_start, shard_end
@@ -0,0 +1,42 @@
1
+ """Tests for sharding."""
2
+ from absl.testing import absltest
3
+ from absl.testing import parameterized
4
+ from grain._src.core import sharding
5
+
6
+
7
+ class ShardingTest(parameterized.TestCase):
8
+
9
+ @parameterized.parameters(
10
+ # num_examples, shard_index, shard_count, drop_remainder, expected_output.
11
+ (9, 0, 1, True, (0, 9)),
12
+ (9, 0, 2, True, (0, 4)),
13
+ (9, 1, 2, True, (4, 8)), # Last example gets dropped.
14
+ (9, 0, 3, True, (0, 3)),
15
+ (9, 1, 3, True, (3, 6)),
16
+ (9, 2, 3, True, (6, 9)),
17
+ (9, 0, 1, False, (0, 9)),
18
+ (9, 0, 2, False, (0, 5)), # First shard gets an extra example.
19
+ (9, 1, 2, False, (5, 9)),
20
+ (8, 0, 3, False, (0, 3)), # First 2 shards get 1 example each.
21
+ (8, 1, 3, False, (3, 6)),
22
+ (8, 2, 3, False, (6, 8)),
23
+ )
24
+ def test_sharding(
25
+ self,
26
+ num_examples: int,
27
+ shard_index: int,
28
+ shard_count: int,
29
+ drop_remainder,
30
+ expected_output: tuple[int, int],
31
+ ):
32
+ shard_options = sharding.ShardOptions(
33
+ shard_index=shard_index,
34
+ shard_count=shard_count,
35
+ drop_remainder=drop_remainder,
36
+ )
37
+ actual_output = sharding.even_split(num_examples, shard_options)
38
+ self.assertEqual(actual_output, expected_output)
39
+
40
+
41
+ if __name__ == '__main__':
42
+ absltest.main()