tfds-nightly 4.9.9.dev202506040044__py3-none-any.whl → 4.9.9.dev202506060044__py3-none-any.whl

tensorflow_datasets/core/community/huggingface_wrapper.py

@@ -21,6 +21,7 @@ import builtins
 import contextlib
 import functools
 import glob
+from importlib import machinery
 import os
 import sys
 import types
@@ -248,6 +249,7 @@ class _MockedHFDatasets(types.ModuleType):
 
   def __init__(self):
     super().__init__('datasets')
+    self.__spec__ = machinery.ModuleSpec('datasets', None)
 
   # pylint: disable=invalid-name
 

tensorflow_datasets/core/constants.py

@@ -49,3 +49,28 @@ CHECKSUMS_FILENAME = 'checksums.tsv'
 # Filepath for mapping between TFDS datasets and PapersWithCode entries.
 PWC_FILENAME = 'tfds_to_pwc_links.json'
 PWC_LINKS_PATH = f'scripts/documentation/{PWC_FILENAME}'
+
+# Retry parameters. Delays are in seconds.
+TFDS_RETRY_TRIES = int(os.environ.get('TFDS_RETRY_TRIES', 3))
+TFDS_RETRY_INITIAL_DELAY = int(os.environ.get('TFDS_RETRY_INITIAL_DELAY', 1))
+# How much to multiply the delay by for each subsequent try
+TFDS_RETRY_DELAY_MULTIPLIER = int(
+    os.environ.get('TFDS_RETRY_DELAY_MULTIPLIER', 2)
+)
+# Random noise to add to the delay (random pick between 0 and noise).
+TFDS_RETRY_NOISE = float(os.environ.get('TFDS_RETRY_NOISE', 0.5))
+# If the error message contains any of these substrings, retry.
+TFDS_RETRY_MSG_SUBSTRINGS = os.environ.get(
+    'TFDS_RETRY_MSG_SUBSTRINGS',
+    (
+        'deadline_exceeded,'
+        '408 Request Timeout,'
+        '429 Too Many Requests,'
+        '500 Internal Server Error,'
+        '502 Bad Gateway,'
+        '503 Service Unavailable,'
+        '504 Gateway Timeout,'
+        '509 Bandwidth Limit Exceeded,'
+        '599 Gateway Error'
+    ),
+).split(',')

tensorflow_datasets/core/dataset_builder.py

@@ -64,6 +64,7 @@ with epy.lazy_imports():
   from tensorflow_datasets.core.utils import file_utils
   from tensorflow_datasets.core.utils import gcs_utils
   from tensorflow_datasets.core.utils import read_config as read_config_lib
+  from tensorflow_datasets.core.utils import retry
   from tensorflow_datasets.core.utils import type_utils
   # pylint: enable=g-import-not-at-top
 
@@ -290,7 +291,8 @@ class DatasetBuilder(registered.RegisteredDataset):
     # Compute the base directory (for download) and dataset/version directory.
     self._data_dir_root, self._data_dir = self._build_data_dir(data_dir)
     # If the dataset info is available, use it.
-    if dataset_info.dataset_info_path(self.data_path).exists():
+    dataset_info_path = dataset_info.dataset_info_path(self.data_path)
+    if retry.retry(dataset_info_path.exists):
       self.info.read_from_directory(self._data_dir)
     else:  # Use the code version (do not restore data)
       self.info.initialize_from_bucket()
@@ -466,8 +468,8 @@ class DatasetBuilder(registered.RegisteredDataset):
         # zipfile.Path does not have `.parts`. Additionally, `os.fspath`
         # will extract the file, so use `str`.
         "tensorflow_datasets" in str(new_path)
-        and legacy_path.exists()
-        and not new_path.exists()
+        and retry.retry(legacy_path.exists)
+        and not retry.retry(new_path.exists)
     ):
       return legacy_path
     else:
@@ -484,7 +486,7 @@ class DatasetBuilder(registered.RegisteredDataset):
     # Search for the url_info file.
     checksums_path = cls._checksums_path
     # If url_info file is found, load the urls
-    if checksums_path and checksums_path.exists():
+    if checksums_path and retry.retry(checksums_path.exists):
       return download.checksums.load_url_infos(checksums_path)
     else:
       return None
@@ -624,7 +626,7 @@ class DatasetBuilder(registered.RegisteredDataset):
 
     download_config = download_config or download.DownloadConfig()
     data_path = self.data_path
-    data_exists = data_path.exists()
+    data_exists = retry.retry(data_path.exists)
 
     # Saving nondeterministic_order in the DatasetInfo for documentation.
     if download_config.nondeterministic_order:
@@ -640,7 +642,7 @@ class DatasetBuilder(registered.RegisteredDataset):
             "Deleting pre-existing dataset %s (%s)", self.name, self.data_dir
         )
         data_path.rmtree()  # Delete pre-existing data.
-        data_exists = data_path.exists()
+        data_exists = retry.retry(data_path.exists)
       else:
         logging.info("Reusing dataset %s (%s)", self.name, self.data_dir)
         return
@@ -805,7 +807,7 @@ class DatasetBuilder(registered.RegisteredDataset):
   def _update_dataset_info(self) -> None:
     """Updates the `dataset_info.json` file in the dataset dir."""
     info_file = self.data_path / constants.DATASET_INFO_FILENAME
-    if not info_file.exists():
+    if not retry.retry(info_file.exists):
       raise AssertionError(f"To update {info_file}, it must already exist.")
     new_info = self.info
     new_info.read_from_directory(self.data_path)
@@ -1020,7 +1022,7 @@ class DatasetBuilder(registered.RegisteredDataset):
     self.assert_is_not_blocked()
 
     # pylint: enable=line-too-long
-    if not self.data_path.exists():
+    if not retry.retry(self.data_path.exists):
       raise AssertionError(
           "Dataset %s: could not find data in %s. Please make sure to call "
           "dataset_builder.download_and_prepare(), or pass download=True to "
@@ -1817,7 +1819,7 @@ class GeneratorBasedBuilder(FileReaderBuilder):
     """Returns the text in the given file and records the lineage."""
     filename = epath.Path(filename)
     self.info.add_file_data_source_access(filename)
-    return filename.read_text(encoding=encoding)
+    return retry.retry(filename.read_text, encoding=encoding)
 
   def read_tfrecord_as_dataset(
       self,
@@ -2057,9 +2059,9 @@ def _save_default_config_name(
 def load_default_config_name(builder_dir: epath.Path) -> str | None:
   """Load `builder_cls` metadata (common to all builder configs)."""
   config_path = builder_dir / ".config" / constants.METADATA_FILENAME
-  if not config_path.exists():
+  if not retry.retry(config_path.exists):
     return None
-  data = json.loads(config_path.read_text())
+  data = json.loads(retry.retry(config_path.read_text))
   return data.get("default_config_name")
 
 

tensorflow_datasets/core/dataset_info.py

@@ -61,6 +61,7 @@ with epy.lazy_imports():
   # pylint: disable=g-import-not-at-top
   from tensorflow_datasets.core.utils import file_utils
   from tensorflow_datasets.core.utils import gcs_utils
+  from tensorflow_datasets.core.utils import retry
 
   from google.protobuf import json_format
   # pylint: enable=g-import-not-at-top
@@ -1123,7 +1124,7 @@ def read_from_json(path: epath.PathLike) -> dataset_info_pb2.DatasetInfo:
     DatasetInfoFileError: If the dataset info file cannot be read.
   """
   try:
-    json_str = epath.Path(path).read_text()
+    json_str = retry.retry(epath.Path(path).read_text)
   except OSError as e:
     raise DatasetInfoFileError(
         f"Could not read dataset info from {path}"

tensorflow_datasets/core/features/feature.py

@@ -37,6 +37,7 @@ from tensorflow_datasets.core.proto import feature_pb2
 from tensorflow_datasets.core.utils import dtype_utils
 from tensorflow_datasets.core.utils import np_utils
 from tensorflow_datasets.core.utils import py_utils
+from tensorflow_datasets.core.utils import retry
 from tensorflow_datasets.core.utils import tf_utils
 from tensorflow_datasets.core.utils import type_utils
 from tensorflow_datasets.core.utils.lazy_imports_utils import tensorflow as tf
@@ -658,7 +659,7 @@ class FeatureConnector(object, metaclass=abc.ABCMeta):
     Returns:
       The reconstructed feature instance.
     """
-    content = json.loads(make_config_path(root_dir).read_text())
+    content = json.loads(retry.retry(make_config_path(root_dir).read_text))
     feature = FeatureConnector.from_json(content)
     feature.load_metadata(root_dir, feature_name=None)
     return feature

tensorflow_datasets/core/splits.py

@@ -36,6 +36,7 @@ from tensorflow_datasets.core import naming
 from tensorflow_datasets.core import proto as proto_lib
 from tensorflow_datasets.core import units
 from tensorflow_datasets.core import utils
+from tensorflow_datasets.core.utils import retry
 from tensorflow_datasets.core.utils import shard_utils
 
 from tensorflow_metadata.proto.v0 import statistics_pb2
@@ -149,7 +150,7 @@ class SplitInfo:
         pattern = filename_template.glob_pattern(num_shards=self.num_shards)
       else:
         pattern = filename_template.sharded_filepaths_pattern(num_shards=None)
-      return list(data_dir.glob(pattern))
+      return list(retry.retry(data_dir.glob, pattern))
     else:
       raise ValueError(f'Filename template for split {self.name} is empty.')
 

tensorflow_datasets/core/utils/retry.py

@@ -0,0 +1,54 @@
+# coding=utf-8
+# Copyright 2025 The TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""To add retry logic to operations suceptible to transient failures."""
+
+import random
+import time
+from typing import Callable, ParamSpec, TypeVar
+
+from absl import logging
+from tensorflow_datasets.core import constants
+
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+def retry(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T:
+  """Returns a decorator that retries the function."""
+  # We purposely don't use flags, as this code might be run before flags are
+  # parsed.
+  tries = constants.TFDS_RETRY_TRIES
+  delay = constants.TFDS_RETRY_INITIAL_DELAY
+  multiplier = constants.TFDS_RETRY_DELAY_MULTIPLIER
+  noise = constants.TFDS_RETRY_NOISE
+  msg_substrings = constants.TFDS_RETRY_MSG_SUBSTRINGS
+  for trial in range(1, tries + 1):
+    try:
+      return func(*args, **kwargs)
+    except BaseException as err:  # pylint: disable=broad-except
+      if trial >= tries:
+        raise err
+      msg = str(err)
+      for msg_substring in msg_substrings:
+        if msg_substring in msg:
+          break
+      else:
+        raise err
+      delay = delay + random.uniform(0, noise)
+      logging.warning("%s, retrying in %s seconds...", msg, delay)
+      time.sleep(delay)
+      delay *= multiplier

{tfds_nightly-4.9.9.dev202506040044.dist-info → tfds_nightly-4.9.9.dev202506060044.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfds-nightly
-Version: 4.9.9.dev202506040044
+Version: 4.9.9.dev202506060044
 Summary: tensorflow/datasets is a library of datasets ready to use with TensorFlow.
 Home-page: https://github.com/tensorflow/datasets
 Download-URL: https://github.com/tensorflow/datasets/tags

{tfds_nightly-4.9.9.dev202506040044.dist-info → tfds_nightly-4.9.9.dev202506060044.dist-info}/RECORD

@@ -59,15 +59,15 @@ tensorflow_datasets/core/as_dataframe.py,sha256=3-2ScAo2G6wwYWbz_w3Crb4QyBwcuIYh
 tensorflow_datasets/core/as_dataframe_test.py,sha256=cGgk3f9j87dDRA2EXedlYb11NpOLdew0dA_O0ZG-PLQ,2048
 tensorflow_datasets/core/beam_utils.py,sha256=0X2lE9ILqLWZe5Idg58-G5XtgITXEAxqVodDtCDo9Ro,5109
 tensorflow_datasets/core/beam_utils_test.py,sha256=5ZhntgG658uT5pF4kw_U5Epm8lu0tdg4cI-0viMItzg,2852
-tensorflow_datasets/core/constants.py,sha256=O6uQ-jNiup0PUq-7itcLDgkszNbZb31hkvuX_POPx9c,1841
-tensorflow_datasets/core/dataset_builder.py,sha256=eqBUVv9jNboGGU3flEcT9iATOA_k-QqdRUkEGEu7DIk,80387
+tensorflow_datasets/core/constants.py,sha256=eAzxhRpS1vx5LMedCmes2nfSU7BL831yi1J8JZkhtXY,2769
+tensorflow_datasets/core/dataset_builder.py,sha256=4l3ArfMSUs0llpo0e59LMg6qZIuoIoQhz8EOK5ThYxk,80603
 tensorflow_datasets/core/dataset_builder_beam_test.py,sha256=d7UsYNsAIY4FcANAERLcVMDcajIpAi0uMfrnQoe4yv4,8386
 tensorflow_datasets/core/dataset_builder_notfdv_test.py,sha256=eIWlOZijQfopdze85EkbcPY1I8lFmEBnedcoUoOAnRQ,1346
 tensorflow_datasets/core/dataset_builder_read_test.py,sha256=QxodggixId7zmknhnC0hAYMcuLehWUODVPo7JvjYw9k,4571
 tensorflow_datasets/core/dataset_builder_test.py,sha256=Q2GlADkgd0hzHHPLdMalW1tyc00QPdPNzaLxR5oU4HQ,49257
 tensorflow_datasets/core/dataset_collection_builder.py,sha256=9tvIWFL4gLFaDFnU_ioKaGGzPQshmACS64vagjTVzkE,7224
 tensorflow_datasets/core/dataset_collection_builder_test.py,sha256=m6H9Kxb-RHUiBEZXd6kyheoixlogJ3OwzixHM2jP6WA,3664
-tensorflow_datasets/core/dataset_info.py,sha256=fUBghpY4DtKWlhVA7OOVvbuQKhaWPhHZ2jXUJMPMQm8,52563
+tensorflow_datasets/core/dataset_info.py,sha256=NHerzNwv9L_ZUE1vvd9O9pS06Si7rCXGYaiTiUF1PJY,52625
 tensorflow_datasets/core/dataset_info_test.py,sha256=T9wnE8XMf1Cl_5IX6fx55AHp7q40mTTQG9H_dyBlTLM,31522
 tensorflow_datasets/core/dataset_metadata.py,sha256=ffmBbfrwsiBqMfwrbzm2EcpmmzmEhIUJNuTkz3oZW3U,3091
 tensorflow_datasets/core/dataset_metadata_test.py,sha256=gbY2Yv1Q8_euuZ0DGaZu3g-0KteqoYoTwrBgyf-uFbc,1822
@@ -102,7 +102,7 @@ tensorflow_datasets/core/shuffle.py,sha256=nBS0pysbTRP77LY6wnxwJmAyKc3aS1lxksaFO
 tensorflow_datasets/core/shuffle_test.py,sha256=Hx2v9QXt3Ul8Q6_VsWfsKgtNHFQwGtx7U_1AYufXkHg,7123
 tensorflow_datasets/core/split_builder.py,sha256=cpz-YowMhmiZZVp7eQPNrh23KvE0-Ef5gme1kCkDjD0,22190
 tensorflow_datasets/core/split_builder_test.py,sha256=kBUVUnQQB_c82AhgjhK3hoYfiAqLt7tDFTzsvZRGQCw,3223
-tensorflow_datasets/core/splits.py,sha256=Bgrh-ncQs0COz2z8fa1bIOnzgCUvsPpPpXQXveCBTCc,29356
+tensorflow_datasets/core/splits.py,sha256=O3jK4Dalp4tEPeZ9AHbkpW1UkJ6uv5m4YRu2x_ZZTJ4,29418
 tensorflow_datasets/core/splits_test.py,sha256=KrM82r0YsJRTGfpYUCkBxiGDC7BjZFcTvJ-Hbo6HwF0,24987
 tensorflow_datasets/core/subsplits_utils.py,sha256=BPHVPAvHlqt4d3HUr4J2Znn8G63pXLPQ29TBi484MOE,6127
 tensorflow_datasets/core/subsplits_utils_test.py,sha256=TIRLtfaf2n38pByhpqYTXEEvs8hrWe2eXk9RFdBMrFQ,5159
@@ -120,7 +120,7 @@ tensorflow_datasets/core/community/config.py,sha256=SiIgegGmxQjoM_8HmKFLdib-loTx
 tensorflow_datasets/core/community/config_test.py,sha256=8lV4RFQsM7rXxJE6AKhE3jEDS-V_YMiySGDZ_e3MNJU,950
 tensorflow_datasets/core/community/dataset_sources.py,sha256=LMUPN3cFVgGK0oGSOT7TzJQ-wyKcZkXtr7OykjfAeAA,2642
 tensorflow_datasets/core/community/dataset_sources_test.py,sha256=RuiDJWRn8KmNAxu4cQTgKecUogdxAFXxXStlECyuvAU,1857
-tensorflow_datasets/core/community/huggingface_wrapper.py,sha256=8tfQzo6VgCDFNKUmaTU6I6NWAySqVfmcr-zihLAacCo,9465
+tensorflow_datasets/core/community/huggingface_wrapper.py,sha256=bKipX4vRV_bBd_y7QLyUqWs5sS3SsTCe5_siQqtcqyU,9556
 tensorflow_datasets/core/community/huggingface_wrapper_test.py,sha256=lxaIWhQte7f5EVOiOVtm0yh-zTmB62jHEOvjN0H_3Q0,1869
 tensorflow_datasets/core/community/load.py,sha256=H1JYz5IU2EMY9RLZJ6-5aBUlNY-tkmu8IDtgDl9j_R8,2461
 tensorflow_datasets/core/community/load_test.py,sha256=AFYSm2qtXDulnZTujPd89DhMmTfAPs6K61bBbrPKdU4,1696
@@ -191,7 +191,7 @@ tensorflow_datasets/core/features/class_label_feature.py,sha256=d8fbBDCWs6UjnGgk
 tensorflow_datasets/core/features/class_label_feature_test.py,sha256=RuUmDTbf9VLTKKQRCjyutLM54EIoNEkVe-lQkvdC3yw,6215
 tensorflow_datasets/core/features/dataset_feature.py,sha256=R-TyDL6-yspS1bWInWBiDYNWXeUm9VAXqPGQz8zUMPA,7403
 tensorflow_datasets/core/features/dataset_feature_test.py,sha256=np40I8R9zHfwvB2njk0r4pm0QdOFN5FmMcLjqqP9cvQ,19725
-tensorflow_datasets/core/features/feature.py,sha256=2EO3ypXS_-urY4Gf_Tf8H_kOpRBUQTqYdSewTbO-03c,40184
+tensorflow_datasets/core/features/feature.py,sha256=Ih8T_4xY9tPtap3fHeD7Y8ADFjyUM1eJ5pCQVD5GGCE,40244
 tensorflow_datasets/core/features/feature_test.py,sha256=FBrRjxzsgdn7o7YEvv5Kd09pBPu9-VyOyXJDL2d8_So,6712
 tensorflow_datasets/core/features/features_dict.py,sha256=qQfmpfPDhKfmUh_38DGkFtn8ANvze9ziNfZwoqqWFwk,10071
 tensorflow_datasets/core/features/features_test.py,sha256=i3k_r8BJIC4oJ1dxSRIJP4LJU3Y0279EOBXmY4S0iGI,14041
@@ -269,6 +269,7 @@ tensorflow_datasets/core/utils/py_utils_test.py,sha256=gGrcypIyFI7Ej0nWREfEqkrhE
 tensorflow_datasets/core/utils/read_config.py,sha256=tRx4ab2I1phU-eSfin_CUNtO7YsCA8n9dqNAX-PzyaE,6601
 tensorflow_datasets/core/utils/resource_utils.py,sha256=T-gvhO9JdUh-_hx08WSbwjUXtWGj78902hVur-kirRM,1949
 tensorflow_datasets/core/utils/resource_utils_test.py,sha256=AT-uwz_739qUSPtWlijl39Ks2RhVOuse4FJD9gsY__c,956
+tensorflow_datasets/core/utils/retry.py,sha256=tjw2MHPWQ-uk3ldKu1Z7skUkT-X6GULbrhpfOPC6O2Q,1812
 tensorflow_datasets/core/utils/shard_utils.py,sha256=BgR4S2SNDKiz8SlOdfdWm0FeRcmCRHOIM4v9FHSZH0g,9208
 tensorflow_datasets/core/utils/shard_utils_test.py,sha256=DJBDVEwy72JRAWBC9yT4U1nYuqwYqgfeiYfgcKuqWFA,8129
 tensorflow_datasets/core/utils/tf_utils.py,sha256=NUPVvsL7bFfhOYRKFdoZdZZ5vx5PUWWYUc3FTXJO1x4,8450
@@ -2460,10 +2461,10 @@ tensorflow_datasets/vision_language/wit/wit_test.py,sha256=PXS8DMNW-MDrT2p5oy4Ic
 tensorflow_datasets/vision_language/wit_kaggle/__init__.py,sha256=vGwSGeM8WE4Q-l0-eEE1sBojmk6YT0l1OO60AWa4Q40,719
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle.py,sha256=q-vX_FBzIwsFxL4sY9vuyQ3UQD2PLM4yhUR4U6l-qao,16903
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle_test.py,sha256=ZymHT1NkmD-pUnh3BmM3_g30c5afsWYnmqDD9dVyDSA,1778
-tfds_nightly-4.9.9.dev202506040044.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
-tfds_nightly-4.9.9.dev202506040044.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-tfds_nightly-4.9.9.dev202506040044.dist-info/METADATA,sha256=u2UDO83W52k1gG7xoqEX7jhLRUyYaXxn0IkTcoo8BLU,11963
-tfds_nightly-4.9.9.dev202506040044.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-tfds_nightly-4.9.9.dev202506040044.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
-tfds_nightly-4.9.9.dev202506040044.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
-tfds_nightly-4.9.9.dev202506040044.dist-info/RECORD,,
+tfds_nightly-4.9.9.dev202506060044.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
+tfds_nightly-4.9.9.dev202506060044.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+tfds_nightly-4.9.9.dev202506060044.dist-info/METADATA,sha256=_tdjfWsO6MPjOVRF_u0gJjfL0fYkK2VlUEgC59W1Fw8,11963
+tfds_nightly-4.9.9.dev202506060044.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tfds_nightly-4.9.9.dev202506060044.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
+tfds_nightly-4.9.9.dev202506060044.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
+tfds_nightly-4.9.9.dev202506060044.dist-info/RECORD,,