tfds-nightly 4.9.8.dev202504250044__py3-none-any.whl → 4.9.8.dev202504270045__py3-none-any.whl

tensorflow_datasets/core/splits.py

@@ -282,9 +282,11 @@ class MultiSplitInfo(SplitInfo):
   This should only be used to read data and not when producing data.
   """
 
-  split_infos: list[SplitInfo] = dataclasses.field(default_factory=list)
+  split_infos: list[SplitInfo | SubSplitInfo] = dataclasses.field(
+      default_factory=list
+  )
 
-  def __init__(self, name: str, split_infos: list[SplitInfo]):
+  def __init__(self, name: str, split_infos: list[SplitInfo | SubSplitInfo]):
     if not split_infos:
       raise ValueError('Need to pass a non-empty list of SplitInfos')
     object.__setattr__(self, 'split_infos', split_infos)
@@ -315,6 +317,16 @@ class MultiSplitInfo(SplitInfo):
         f'split_infos={self.split_infos!r})'
     )
 
+  @property
+  def examples_in_shards(self) -> list[int]:
+    result = []
+    for split_info in self.split_infos:
+      if isinstance(split_info, (SubSplitInfo, MultiSplitInfo)):
+        result.extend(split_info.examples_in_shards)
+      else:
+        result.extend(split_info.shard_lengths)
+    return result
+
   @property
   def file_instructions(self) -> list[shard_utils.FileInstruction]:
     result = []
@@ -361,6 +373,10 @@ class SubSplitInfo:
   def shard_lengths(self) -> list[int]:
     return [f.take for f in self.file_instructions]
 
+  @property
+  def examples_in_shards(self) -> list[int]:
+    return [f.examples_in_shard for f in self.file_instructions]
+
   @property
   def num_examples(self) -> int:
     """Returns the number of example in the subsplit."""
@@ -526,7 +542,7 @@ def _make_absolute_instructions(
 
 def _file_instructions_for_split(
     instruction: _AbsoluteInstruction,
-    split_info: SplitInfo,
+    split_info: SplitInfo | SubSplitInfo,
 ) -> list[shard_utils.FileInstruction]:
   """Returns the file instructions from the given instruction applied to the given split info."""
   if not split_info.num_examples:
@@ -537,9 +553,7 @@ def _file_instructions_for_split(
     return []
   to = split_info.num_examples if instruction.to is None else instruction.to
   if isinstance(split_info, (SubSplitInfo, MultiSplitInfo)):
-    examples_in_shards = [
-        f.examples_in_shard for f in split_info.file_instructions
-    ]
+    examples_in_shards = split_info.examples_in_shards
   else:
     examples_in_shards = None
   return shard_utils.get_file_instructions(

tensorflow_datasets/core/splits_test.py

@@ -255,6 +255,43 @@ class SplitDictTest(testing.TestCase):
     self.assertEqual(file_instruction.take, 2)
     self.assertEqual(file_instruction.examples_in_shard, 10)
 
+  def test_multi_split_empty_shard(self):
+    split_info = splits.MultiSplitInfo(
+        name='train',
+        split_infos=[
+            splits.SplitInfo(
+                name='train',
+                shard_lengths=[5, 0, 5],
+                num_bytes=0,
+                filename_template=_filename_template(
+                    split='train', data_dir='/abc'
+                ),
+            ),
+        ],
+    )
+    split_dict = splits.SplitDict([split_info])
+    sub_split = split_dict['train[:90%]']
+    self.assertEqual(sub_split.name, 'train[:90%]')
+    self.assertEqual(sub_split.num_examples, 9)
+    self.assertEqual(sub_split.shard_lengths, [5, 4])
+    self.assertEqual(
+        sub_split.file_instructions,
+        [
+            shard_utils.FileInstruction(
+                filename='/abc/ds_name-train.tfrecord-00000-of-00003',
+                skip=0,
+                take=5,
+                examples_in_shard=5,
+            ),
+            shard_utils.FileInstruction(
+                filename='/abc/ds_name-train.tfrecord-00002-of-00003',
+                skip=0,
+                take=4,
+                examples_in_shard=5,
+            ),
+        ],
+    )
+
 
 class SplitsTest(testing.TestCase):
 

{tfds_nightly-4.9.8.dev202504250044.dist-info → tfds_nightly-4.9.8.dev202504270045.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfds-nightly
-Version: 4.9.8.dev202504250044
+Version: 4.9.8.dev202504270045
 Summary: tensorflow/datasets is a library of datasets ready to use with TensorFlow.
 Home-page: https://github.com/tensorflow/datasets
 Download-URL: https://github.com/tensorflow/datasets/tags

{tfds_nightly-4.9.8.dev202504250044.dist-info → tfds_nightly-4.9.8.dev202504270045.dist-info}/RECORD

@@ -102,8 +102,8 @@ tensorflow_datasets/core/shuffle.py,sha256=xMOaKql03o6sxcJp_KFG60N2tua2xE2xibQue
 tensorflow_datasets/core/shuffle_test.py,sha256=-xkd3kx5Ci2gGyKbMwdzGwhjFBv6LTKX1FPPn9eanGI,7123
 tensorflow_datasets/core/split_builder.py,sha256=HHOX3KatqnSL5EW-6vXzyDDKG4FOEyjGXM_MRuLNxHE,22190
 tensorflow_datasets/core/split_builder_test.py,sha256=0HsFONj8A09voGS_7zCAb9nd1PdTqT1u4MXR4aWAoS4,3223
-tensorflow_datasets/core/splits.py,sha256=LKCmJSMQxRKTo3IoOBB3dGaoOxxUpogXVyilGTrRcOU,28912
-tensorflow_datasets/core/splits_test.py,sha256=WtbAKIs7eOsGmLC3KxZ6DcSNSP4AE2KNBgP_nanK6JY,23820
+tensorflow_datasets/core/splits.py,sha256=K5M8fJeleCcJiHa_TsiNI0AQcMimkSSb0D73VOOfai0,29356
+tensorflow_datasets/core/splits_test.py,sha256=OD-ZNnpkPV8WL43IUywXdoAM2QuMV8htsqChw3Nqq9Q,24987
 tensorflow_datasets/core/subsplits_utils.py,sha256=ZxzeEVbUVlADSTigQbf4k9V4rdjjoRvAuBVXl1-BNrg,6127
 tensorflow_datasets/core/subsplits_utils_test.py,sha256=iC6f4Bfpnon_NL7TGeCwp3OW2P3jTIPIpWbCqxmuhX4,5159
 tensorflow_datasets/core/tf_compat.py,sha256=GY-LBj1V9LLNl2LtAh4dLcATisW_8VYTWHgTR31MXvU,1820
@@ -2460,10 +2460,10 @@ tensorflow_datasets/vision_language/wit/wit_test.py,sha256=DJ687VN9hAp6SLXnr_P12
 tensorflow_datasets/vision_language/wit_kaggle/__init__.py,sha256=38tJQ73VHz8vOJn-AyZh2we2YJucbSRIgmgcrsC6bQM,719
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle.py,sha256=k-7YD1SGr5bASfdR2_09rrqz-8cpWdIcBWWEXhCvzuk,16903
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle_test.py,sha256=vLlluBW77ASNVC4ix7t8idkSUBI6q1-B7zmRV_ICCQM,1778
-tfds_nightly-4.9.8.dev202504250044.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
-tfds_nightly-4.9.8.dev202504250044.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-tfds_nightly-4.9.8.dev202504250044.dist-info/METADATA,sha256=z0sHgmHnCUDPiFpIf9hdFefU9FdtlhyrD-0ksU7lurI,11879
-tfds_nightly-4.9.8.dev202504250044.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-tfds_nightly-4.9.8.dev202504250044.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
-tfds_nightly-4.9.8.dev202504250044.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
-tfds_nightly-4.9.8.dev202504250044.dist-info/RECORD,,
+tfds_nightly-4.9.8.dev202504270045.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
+tfds_nightly-4.9.8.dev202504270045.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+tfds_nightly-4.9.8.dev202504270045.dist-info/METADATA,sha256=zhywWcA5B-E_n0qBfUkiFbrRdksbC8Caca5mwbDsPr4,11879
+tfds_nightly-4.9.8.dev202504270045.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+tfds_nightly-4.9.8.dev202504270045.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
+tfds_nightly-4.9.8.dev202504270045.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
+tfds_nightly-4.9.8.dev202504270045.dist-info/RECORD,,