tfds-nightly 4.9.9.dev202507010046__py3-none-any.whl → 4.9.9.dev202507030044__py3-none-any.whl

tensorflow_datasets/core/naming.py

@@ -666,7 +666,7 @@ class ShardedFileTemplate:
     `/path/dataset_name-split.fileformat@num_shards` or
     `/path/dataset_name-split@num_shards.fileformat` depending on the format.
     If `num_shards` is not given, then it returns
-    `/path/dataset_name-split.fileformat*`.
+    `/path/dataset_name-split.fileformat-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]`.
 
     Args:
       num_shards: optional specification of the number of shards.
@@ -681,7 +681,7 @@ class ShardedFileTemplate:
     elif use_at_notation:
       replacement = '@*'
     else:
-      replacement = '*'
+      replacement = '-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]'
     return _replace_shard_pattern(os.fspath(a_filepath), replacement)
 
   def glob_pattern(self, num_shards: int | None = None) -> str:

tensorflow_datasets/core/naming_test.py

@@ -459,7 +459,7 @@ def test_sharded_file_template_shard_index():
   )
   assert (
       os.fspath(template.sharded_filepaths_pattern())
-      == '/my/path/data/mnist-train.tfrecord*'
+      == '/my/path/data/mnist-train.tfrecord-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]'
   )
   assert (
       os.fspath(template.sharded_filepaths_pattern(num_shards=100))
@@ -474,7 +474,10 @@ def test_glob_pattern():
       filetype_suffix='tfrecord',
       data_dir=epath.Path('/data'),
   )
-  assert '/data/ds-train.tfrecord*' == template.glob_pattern()
+  assert (
+      '/data/ds-train.tfrecord-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]'
+      == template.glob_pattern()
+  )
   assert '/data/ds-train.tfrecord-*-of-00042' == template.glob_pattern(
       num_shards=42
   )

tensorflow_datasets/core/writer.py

@@ -816,8 +816,9 @@ class NoShuffleBeamWriter:
     logging.info("Finalizing writer for %s", self._filename_template.split)
     # We don't know the number of shards, the length of each shard, nor the
     # total size, so we compute them here.
-    prefix = epath.Path(self._filename_template.filepath_prefix())
-    shards = self._filename_template.data_dir.glob(f"{prefix.name}*")
+    shards = self._filename_template.data_dir.glob(
+        self._filename_template.glob_pattern()
+    )
 
     def _get_length_and_size(shard: epath.Path) -> tuple[epath.Path, int, int]:
       length = self._file_adapter.num_examples(shard)

tensorflow_datasets/core/writer_test.py

@@ -592,39 +592,47 @@ class NoShuffleBeamWriterTest(parameterized.TestCase):
 
     with tempfile.TemporaryDirectory() as tmp_dir:
       tmp_dir = epath.Path(tmp_dir)
-      filename_template = naming.ShardedFileTemplate(
-          dataset_name='foo',
-          split='train',
-          filetype_suffix=file_format.file_suffix,
-          data_dir=tmp_dir,
-      )
-      writer = writer_lib.NoShuffleBeamWriter(
-          serializer=testing.DummySerializer('dummy specs'),
-          filename_template=filename_template,
-          file_format=file_format,
-      )
+
+      def get_writer(split):
+        filename_template = naming.ShardedFileTemplate(
+            dataset_name='foo',
+            split=split,
+            filetype_suffix=file_format.file_suffix,
+            data_dir=tmp_dir,
+        )
+        return writer_lib.NoShuffleBeamWriter(
+            serializer=testing.DummySerializer('dummy specs'),
+            filename_template=filename_template,
+            file_format=file_format,
+        )
+
       to_write = [(i, str(i).encode('utf-8')) for i in range(10)]
       # Here we need to disable type check as `beam.Create` is not capable of
       # inferring the type of the PCollection elements.
       options = beam.options.pipeline_options.PipelineOptions(
           pipeline_type_check=False
       )
-      with beam.Pipeline(options=options, runner=_get_runner()) as pipeline:
-
-        @beam.ptransform_fn
-        def _build_pcollection(pipeline):
-          pcollection = pipeline | 'Start' >> beam.Create(to_write)
-          return writer.write_from_pcollection(pcollection)
-
-        _ = pipeline | 'test' >> _build_pcollection()  # pylint: disable=no-value-for-parameter
-      shard_lengths, total_size = writer.finalize()
-      self.assertNotEmpty(shard_lengths)
-      self.assertEqual(sum(shard_lengths), 10)
-      self.assertGreater(total_size, 10)
+      writers = [get_writer(split) for split in ('train-b', 'train')]
+
+      for writer in writers:
+        with beam.Pipeline(options=options, runner=_get_runner()) as pipeline:
+
+          @beam.ptransform_fn
+          def _build_pcollection(pipeline, writer):
+            pcollection = pipeline | 'Start' >> beam.Create(to_write)
+            return writer.write_from_pcollection(pcollection)
+
+          _ = pipeline | 'test' >> _build_pcollection(writer)
+
       files = list(tmp_dir.iterdir())
-      self.assertGreaterEqual(len(files), 1)
+      self.assertGreaterEqual(len(files), 2)
       for f in files:
         self.assertIn(file_format.file_suffix, f.name)
+      for writer in writers:
+        shard_lengths, total_size = writer.finalize()
+        self.assertNotEmpty(shard_lengths)
+        self.assertEqual(sum(shard_lengths), 10)
+        self.assertGreater(total_size, 10)
 
 
 class CustomExampleWriter(writer_lib.ExampleWriter):

{tfds_nightly-4.9.9.dev202507010046.dist-info → tfds_nightly-4.9.9.dev202507030044.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfds-nightly
-Version: 4.9.9.dev202507010046
+Version: 4.9.9.dev202507030044
 Summary: tensorflow/datasets is a library of datasets ready to use with TensorFlow.
 Home-page: https://github.com/tensorflow/datasets
 Download-URL: https://github.com/tensorflow/datasets/tags

{tfds_nightly-4.9.9.dev202507010046.dist-info → tfds_nightly-4.9.9.dev202507030044.dist-info}/RECORD

@@ -88,8 +88,8 @@ tensorflow_datasets/core/lazy_imports_lib.py,sha256=Q-c3qGEZJDqviEQUiro2iBpMw7KA
 tensorflow_datasets/core/lazy_imports_lib_test.py,sha256=cbdamDUJIY5YORm6coyCMIralgsL_gCUfa2Dzdj6ZPY,1695
 tensorflow_datasets/core/load.py,sha256=1FQVnKwn8OVS_IgDbs9XN7aIVxQnyfrS0pI2X9dh77M,37765
 tensorflow_datasets/core/load_test.py,sha256=EEa8GuSIrEbn0RcGrWS3hmmatKBqBA3QOQWpQ1WjVgA,6490
-tensorflow_datasets/core/naming.py,sha256=IYz4U9_2lLqpeyJzHZCZ0mbL6hLxnBugsB-IhKduYBU,25506
-tensorflow_datasets/core/naming_test.py,sha256=cC6Cf3Urhpwf1Wtgt85zar8KsFU9VrZgV0ZqPCp5PE4,29986
+tensorflow_datasets/core/naming.py,sha256=B_P77QDA4lkG2FUl4PrzZR0U6qqae_fLxruGBw3ZSVc,25614
+tensorflow_datasets/core/naming_test.py,sha256=SwydgLjf2Mouow1yVZlc73sb8rp4522NhkTSEmg31vo,30112
 tensorflow_datasets/core/read_only_builder.py,sha256=R0QIqckUjl74G7oBj1uCRm_g9e0omstDMTbbwC25B88,22146
 tensorflow_datasets/core/read_only_builder_test.py,sha256=Nw2KQCHBdTW7210Um2K3SzfqAOJB1v1r2yJkzdFehWA,24174
 tensorflow_datasets/core/reader.py,sha256=s65FNOUDyAhd4OgHOSvE5lr4rnlUnOILjlVcRS6Qbhw,17345
@@ -112,8 +112,8 @@ tensorflow_datasets/core/units_test.py,sha256=rGR0rsP9M0BVCqv2OA1GZRH5csq8_gPYhI
 tensorflow_datasets/core/valid_tags.txt,sha256=HLn8CV1ORQZaAhLr-U-5MsYFrYBVHDgs4bKEu2nzlVw,20100
 tensorflow_datasets/core/visibility.py,sha256=43jHRRdg2xHRpAA2mUD1Yz-vOs5EVhx3xhB2RoIJBg8,3498
 tensorflow_datasets/core/visibility_test.py,sha256=h_UwIBfLgIkMSSSPoQmT0mNUUOH8jAdebA_DdWNSxdg,1350
-tensorflow_datasets/core/writer.py,sha256=JyVr7Zs5IYp-kHXp8LBs03enuMgD7-T5DN6uq0LbF4s,28909
-tensorflow_datasets/core/writer_test.py,sha256=CLx1tE2QM0sUkFz3hpIP8tZQcTpPDY9VklzuCEa4qCE,22531
+tensorflow_datasets/core/writer.py,sha256=T41xcagE1IhFqKNtoHR467SXqbOw7PrQR2nm7nXn5Yc,28877
+tensorflow_datasets/core/writer_test.py,sha256=j-lvS96jFmvBF0bd0mVR4EGBbxeFW7ucxxFXtC40wTo,22702
 tensorflow_datasets/core/community/__init__.py,sha256=bAU6d62u2i14gRw3xgAzkQS8kRcuRnJWqEVn_r0RXRs,1206
 tensorflow_datasets/core/community/cache.py,sha256=-dx3iEsgktu8OR42a64CFX64HtaXMHjXAfnYlc0H5BM,2130
 tensorflow_datasets/core/community/config.py,sha256=SiIgegGmxQjoM_8HmKFLdib-loTxpQpEwXXKQbTLJI0,4451
@@ -2461,10 +2461,10 @@ tensorflow_datasets/vision_language/wit/wit_test.py,sha256=PXS8DMNW-MDrT2p5oy4Ic
 tensorflow_datasets/vision_language/wit_kaggle/__init__.py,sha256=vGwSGeM8WE4Q-l0-eEE1sBojmk6YT0l1OO60AWa4Q40,719
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle.py,sha256=q-vX_FBzIwsFxL4sY9vuyQ3UQD2PLM4yhUR4U6l-qao,16903
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle_test.py,sha256=ZymHT1NkmD-pUnh3BmM3_g30c5afsWYnmqDD9dVyDSA,1778
-tfds_nightly-4.9.9.dev202507010046.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
-tfds_nightly-4.9.9.dev202507010046.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-tfds_nightly-4.9.9.dev202507010046.dist-info/METADATA,sha256=ldOcz84AQsdQXpXII94T_z57amwrq8Je-zIvnGynfX4,11963
-tfds_nightly-4.9.9.dev202507010046.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-tfds_nightly-4.9.9.dev202507010046.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
-tfds_nightly-4.9.9.dev202507010046.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
-tfds_nightly-4.9.9.dev202507010046.dist-info/RECORD,,
+tfds_nightly-4.9.9.dev202507030044.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
+tfds_nightly-4.9.9.dev202507030044.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+tfds_nightly-4.9.9.dev202507030044.dist-info/METADATA,sha256=mBYq6qCzA_MbWBx9ui0LQFkk3GFTUH0nWBZnNOafRxY,11963
+tfds_nightly-4.9.9.dev202507030044.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tfds_nightly-4.9.9.dev202507030044.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
+tfds_nightly-4.9.9.dev202507030044.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
+tfds_nightly-4.9.9.dev202507030044.dist-info/RECORD,,