tfds-nightly 4.9.8.dev202505200044__py3-none-any.whl → 4.9.8.dev202505220044__py3-none-any.whl

tensorflow_datasets/core/dataset_builders/croissant_builder.py

@@ -270,6 +270,9 @@ class CroissantBuilder(
         the values should be the values to filter by. If a record matches all
         the filters, it will be included in the dataset.
       **kwargs: kwargs to pass to GeneratorBasedBuilder directly.
+
+    Raises:
+      ValueError: If no record sets are found in the Croissant JSON-LD.
     """
     if mapping is None:
       mapping = {}
@@ -279,11 +282,17 @@ class CroissantBuilder(
     self.name = croissant_utils.get_tfds_dataset_name(dataset)
     self.metadata = dataset.metadata
 
-    # In TFDS, version is a mandatory attribute, while in Croissant it is only a
-    # recommended attribute. If the version is unspecified in Croissant, we set
-    # it to `1.0.0` in TFDS.
+    # The dataset version is determined using the following precedence:
+    # * overwrite_version (if provided).
+    # * The version from Croissant metadata (self.metadata.version),
+    # automatically converting major.minor formats to major.minor.0 (e.g., "1.2"
+    # becomes "1.2.0"). See croissant_utils.get_croissant_version for details.
+    # * Defaults to '1.0.0' if no version is specified (version is optional in
+    # Croissant, but mandatory in TFDS).
     self.VERSION = version_lib.Version(  # pylint: disable=invalid-name
-        overwrite_version or self.metadata.version or '1.0.0'
+        overwrite_version
+        or croissant_utils.get_croissant_version(self.metadata.version)
+        or '1.0.0'
     )
     self.RELEASE_NOTES = {}  # pylint: disable=invalid-name
 
@@ -293,6 +302,12 @@ class CroissantBuilder(
         conversion_utils.to_tfds_name(record_set_id)
         for record_set_id in record_set_ids
     ]
+    if not config_names:
+      raise ValueError(
+          'No record sets found in the Croissant JSON-LD. At least one record'
+          ' set is required to be able to download and prepare the dataset.'
+      )
+
     self.BUILDER_CONFIGS: list[dataset_builder.BuilderConfig] = [  # pylint: disable=invalid-name
         dataset_builder.BuilderConfig(name=config_name)
         for config_name in config_names

tensorflow_datasets/core/dataset_builders/croissant_builder_test.py

@@ -249,6 +249,17 @@ def test_sequence_feature_datatype_converter():
   assert isinstance(actual_feature.feature, text_feature.Text)
 
 
+def test_version_converter(tmp_path):
+  with testing.dummy_croissant_file(version="1.0") as croissant_file:
+    builder = croissant_builder.CroissantBuilder(
+        jsonld=croissant_file,
+        file_format=FileFormat.ARRAY_RECORD,
+        disable_shuffling=True,
+        data_dir=tmp_path,
+    )
+    assert builder.version == "1.0.0"
+
+
 @pytest.fixture(name="crs_builder")
 def mock_croissant_dataset_builder(tmp_path, request):
   dataset_name = request.param["dataset_name"]

tensorflow_datasets/core/utils/croissant_utils.py

@@ -18,6 +18,7 @@
 from __future__ import annotations
 
 import dataclasses
+import re
 import typing
 
 from tensorflow_datasets.core.utils import conversion_utils
@@ -28,6 +29,7 @@ if typing.TYPE_CHECKING:
   import mlcroissant as mlc
 
 _HUGGINGFACE_URL_PREFIX = "https://huggingface.co/datasets/"
+_VERSION_REGEX_WITHOUT_PATCH = re.compile(r"^(?P<major>\d+)\.(?P<minor>\d+)$")
 
 
 @dataclasses.dataclass(frozen=True)
@@ -40,6 +42,27 @@ class SplitReference:
   reference_field: mlc.Field
 
 
+def get_croissant_version(version: str | None) -> str | None:
+  """Returns the possibly corrected Croissant version in TFDS format.
+
+  TFDS expects versions to follow the Semantic versioning 2.0.0 syntax, but
+  Croissant is more lax and accepts also {major.minor}. To avoid raising errors
+  in these cases, we add a `0` as a patch version to the Croissant-provided
+  version.
+
+  Args:
+    version: The Croissant version.
+
+  Returns:
+    The Croissant version in TFDS format.
+  """
+  if not version:
+    return None
+  if _VERSION_REGEX_WITHOUT_PATCH.match(version):
+    return f"{version}.0"
+  return version
+
+
 def get_dataset_name(dataset: mlc.Dataset) -> str:
   """Returns dataset name of the given MLcroissant dataset."""
   if (url := dataset.metadata.url) and url.startswith(_HUGGINGFACE_URL_PREFIX):

tensorflow_datasets/core/utils/croissant_utils_test.py

@@ -36,6 +36,22 @@ def test_get_tfds_dataset_name(croissant_name, croissant_url, tfds_name):
   assert croissant_utils.get_tfds_dataset_name(dataset) == tfds_name
 
 
+@pytest.mark.parametrize(
+    'croissant_version,tfds_version',
+    [
+        ('1.0', '1.0.0'),
+        ('1.2', '1.2.0'),
+        ('1.2.3', '1.2.3'),
+        ('1.2.3.4', '1.2.3.4'),
+        (None, None),
+    ],
+)
+def test_get_croissant_version(croissant_version, tfds_version):
+  assert (
+      croissant_utils.get_croissant_version(croissant_version) == tfds_version
+  )
+
+
 def test_get_record_set_ids():
   metadata = mlc.Metadata(
       name='dummy_dataset',

tensorflow_datasets/testing/test_utils.py

@@ -724,6 +724,7 @@ def dummy_croissant_file(
     raw_data_filename: epath.PathLike = 'raw_data.jsonl',
     croissant_filename: epath.PathLike = 'croissant.json',
     split_names: Sequence[str] | None = None,
+    version: str = '1.2.0',
 ) -> Iterator[epath.Path]:
   """Yields temporary path to a dummy Croissant file.
 
@@ -746,6 +747,7 @@ def dummy_croissant_file(
       If None, the function will create a split record set with the default
       split names `train` and `test`. If `split_names` is defined, the `split`
       key in the entries must match one of the split names.
+    version: The version of the dataset. Defaults to `1.2.0`.
   """
   if entries is None:
     entries = [
@@ -874,7 +876,7 @@ def dummy_croissant_file(
         url='https://dummy_url',
         distribution=distribution,
         record_sets=record_sets,
-        version='1.2.0',
+        version=version,
         license='Public',
     )
     # Write Croissant JSON-LD to tempdir.

{tfds_nightly-4.9.8.dev202505200044.dist-info → tfds_nightly-4.9.8.dev202505220044.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfds-nightly
-Version: 4.9.8.dev202505200044
+Version: 4.9.8.dev202505220044
 Summary: tensorflow/datasets is a library of datasets ready to use with TensorFlow.
 Home-page: https://github.com/tensorflow/datasets
 Download-URL: https://github.com/tensorflow/datasets/tags
@@ -52,7 +52,7 @@ Requires-Dist: jupyter; extra == "tests-all"
 Requires-Dist: pytest; extra == "tests-all"
 Requires-Dist: pytest-shard; extra == "tests-all"
 Requires-Dist: pytest-xdist; extra == "tests-all"
-Requires-Dist: apache-beam; extra == "tests-all"
+Requires-Dist: apache-beam<2.65.0; extra == "tests-all"
 Requires-Dist: conllu; extra == "tests-all"
 Requires-Dist: mlcroissant>=1.0.9; extra == "tests-all"
 Requires-Dist: pandas; extra == "tests-all"
@@ -60,15 +60,15 @@ Requires-Dist: pydub; extra == "tests-all"
 Requires-Dist: pyyaml; extra == "tests-all"
 Requires-Dist: tensorflow-io[tensorflow]; python_version < "3.12" and extra == "tests-all"
 Requires-Dist: scipy; extra == "tests-all"
-Requires-Dist: apache-beam; extra == "tests-all"
+Requires-Dist: apache-beam<2.65.0; extra == "tests-all"
 Requires-Dist: gcsfs; extra == "tests-all"
 Requires-Dist: zarr<3.0.0; extra == "tests-all"
-Requires-Dist: apache-beam; extra == "tests-all"
+Requires-Dist: apache-beam<2.65.0; extra == "tests-all"
 Requires-Dist: gcld3; extra == "tests-all"
 Requires-Dist: langdetect; extra == "tests-all"
 Requires-Dist: nltk==3.8.1; extra == "tests-all"
 Requires-Dist: tldextract; extra == "tests-all"
-Requires-Dist: apache-beam; extra == "tests-all"
+Requires-Dist: apache-beam<2.65.0; extra == "tests-all"
 Requires-Dist: matplotlib; extra == "tests-all"
 Requires-Dist: Pillow; extra == "tests-all"
 Requires-Dist: pydub; extra == "tests-all"
@@ -96,8 +96,8 @@ Requires-Dist: scipy; extra == "tests-all"
 Requires-Dist: scipy; extra == "tests-all"
 Requires-Dist: pandas; extra == "tests-all"
 Requires-Dist: Pillow; extra == "tests-all"
-Requires-Dist: apache-beam; extra == "tests-all"
-Requires-Dist: apache-beam; extra == "tests-all"
+Requires-Dist: apache-beam<2.65.0; extra == "tests-all"
+Requires-Dist: apache-beam<2.65.0; extra == "tests-all"
 Requires-Dist: mwparserfromhell; extra == "tests-all"
 Requires-Dist: mwxml; extra == "tests-all"
 Requires-Dist: bs4; extra == "tests-all"
@@ -110,7 +110,7 @@ Requires-Dist: jupyter; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-shard; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: apache-beam; extra == "dev"
+Requires-Dist: apache-beam<2.65.0; extra == "dev"
 Requires-Dist: conllu; extra == "dev"
 Requires-Dist: mlcroissant>=1.0.9; extra == "dev"
 Requires-Dist: pandas; extra == "dev"
@@ -175,18 +175,18 @@ Requires-Dist: datasets; extra == "huggingface"
 Provides-Extra: aflw2k3d
 Requires-Dist: scipy; extra == "aflw2k3d"
 Provides-Extra: beir
-Requires-Dist: apache-beam; extra == "beir"
+Requires-Dist: apache-beam<2.65.0; extra == "beir"
 Provides-Extra: ble-wind-field
 Requires-Dist: gcsfs; extra == "ble-wind-field"
 Requires-Dist: zarr<3.0.0; extra == "ble-wind-field"
 Provides-Extra: c4
-Requires-Dist: apache-beam; extra == "c4"
+Requires-Dist: apache-beam<2.65.0; extra == "c4"
 Requires-Dist: gcld3; extra == "c4"
 Requires-Dist: langdetect; extra == "c4"
 Requires-Dist: nltk==3.8.1; extra == "c4"
 Requires-Dist: tldextract; extra == "c4"
 Provides-Extra: c4-wsrs
-Requires-Dist: apache-beam; extra == "c4-wsrs"
+Requires-Dist: apache-beam<2.65.0; extra == "c4-wsrs"
 Provides-Extra: cats-vs-dogs
 Requires-Dist: matplotlib; extra == "cats-vs-dogs"
 Provides-Extra: colorectal-histology
@@ -242,9 +242,9 @@ Requires-Dist: pandas; extra == "wake-vision"
 Provides-Extra: wider-face
 Requires-Dist: Pillow; extra == "wider-face"
 Provides-Extra: wiki-dialog
-Requires-Dist: apache-beam; extra == "wiki-dialog"
+Requires-Dist: apache-beam<2.65.0; extra == "wiki-dialog"
 Provides-Extra: wikipedia
-Requires-Dist: apache-beam; extra == "wikipedia"
+Requires-Dist: apache-beam<2.65.0; extra == "wikipedia"
 Requires-Dist: mwparserfromhell; extra == "wikipedia"
 Requires-Dist: mwxml; extra == "wikipedia"
 Provides-Extra: wsc273

{tfds_nightly-4.9.8.dev202505200044.dist-info → tfds_nightly-4.9.8.dev202505220044.dist-info}/RECORD

@@ -141,8 +141,8 @@ tensorflow_datasets/core/data_sources/python_test.py,sha256=Rg_ui2fA75532ma134VU
 tensorflow_datasets/core/dataset_builders/__init__.py,sha256=KULxN1WzBvUwTw3K3SHFM-sE4wbm9UC6cX0tOm0jEZg,1945
 tensorflow_datasets/core/dataset_builders/adhoc_builder.py,sha256=fS6OxRtI3DKdcsniJagTHYfopuoFs55cpW0_gPbhnkY,9216
 tensorflow_datasets/core/dataset_builders/adhoc_builder_test.py,sha256=PAQXfp_qGs4KaAzYbTMm8y87HvnCUca035Jrafjq3zw,13096
-tensorflow_datasets/core/dataset_builders/croissant_builder.py,sha256=g3LZEeyozAdjgKd6mlAOdBa6C6VBEeu1vnikWUYXxzc,15287
-tensorflow_datasets/core/dataset_builders/croissant_builder_test.py,sha256=S_dVDe_fUt-fIRTh8e7beJtLbUMcc5_OBIQ-Ji1OpUg,10199
+tensorflow_datasets/core/dataset_builders/croissant_builder.py,sha256=1_-GgOu_5ONlInvjZZFGGR44mcxlp_ZdVX9ttC83Z-s,15913
+tensorflow_datasets/core/dataset_builders/croissant_builder_test.py,sha256=Jc1EZI357oS1Vfszgy8E9uLMNoFh-kzQjcAz0g1OTN8,10538
 tensorflow_datasets/core/dataset_builders/huggingface_dataset_builder.py,sha256=si5OoTPeftUyPj9w5OuYKjwfx1ByZghBo6vfEKQQGhA,18313
 tensorflow_datasets/core/dataset_builders/huggingface_dataset_builder_test.py,sha256=UCvcr6DRwcWLu7x3o_PDyLJBI3r84ylfYMCO3qhD-8w,4389
 tensorflow_datasets/core/dataset_builders/view_builder.py,sha256=HhLOqlkWwOSvr_8wzwwTRapgXY8yUg_bbBGasTb3cQw,11907
@@ -245,8 +245,8 @@ tensorflow_datasets/core/utils/bool_utils_test.py,sha256=4x3vqQvc8PtC-ptrUpu5TWx
 tensorflow_datasets/core/utils/colormap.csv,sha256=DDayUU9R19cxhcG3fj4cFwhI46W20U7ofBG0kToUHOw,2732
 tensorflow_datasets/core/utils/conversion_utils.py,sha256=f_ZW71wx_dyg5LJlmuaU_TtiApfC2IqsG6fsNR8SOQc,6340
 tensorflow_datasets/core/utils/conversion_utils_test.py,sha256=Ipjqju_81kweBgqOYjoHErRxBWSbskQhSs6HTxRtlso,5043
-tensorflow_datasets/core/utils/croissant_utils.py,sha256=-hPQQjNQ0f4zG7qKBG38ak4OfYZCZsJUiPkuMYb5ISw,4482
-tensorflow_datasets/core/utils/croissant_utils_test.py,sha256=R5lQnwTXml9hSOrHwTav7CvAZQvAP3EaRxQJJ3MXOlk,4123
+tensorflow_datasets/core/utils/croissant_utils.py,sha256=WVmrKqmjYf1u6BOQWMs0yGic85aVxDOZtIPbcDsAL1o,5181
+tensorflow_datasets/core/utils/croissant_utils_test.py,sha256=QbfAv-Ktrib4ARsR7fIoMXTsezfK1UAsgOQWTocjtaA,4497
 tensorflow_datasets/core/utils/docs.py,sha256=e8sua3eiqkDsPJUbYrgp_FUwnnHwMyMzTMNWKMHn464,1549
 tensorflow_datasets/core/utils/dtype_utils.py,sha256=E3Zwe_Hv6BiN_qv8VI4I1oHFd3MVgs2NCcimXMoahxI,3246
 tensorflow_datasets/core/utils/dtype_utils_test.py,sha256=LAE2idzXz0BiNHELm3rmCdhwKIstkWj8ZRmztslcLi0,3259
@@ -2124,7 +2124,7 @@ tensorflow_datasets/testing/mocking.py,sha256=52uOXhIEZM-Aet6fRAW13REVSmq7mWT_kA
 tensorflow_datasets/testing/mocking_test.py,sha256=OZQAAzOQLOEwGJaNZhEYg0Dn0Vi9a8xKswjbSWBeYao,13796
 tensorflow_datasets/testing/test_case.py,sha256=5BHRT99KJWx7Zzn8fkgsq09h2lkrPONyWEZdt_S9aGY,2554
 tensorflow_datasets/testing/test_case_in_context.py,sha256=Yf4ehrvtCnxR4HKN1c8Qi6prURWfLuxvwQieKfnMD8g,1872
-tensorflow_datasets/testing/test_utils.py,sha256=UlbaYTVRITXWyztCwbGcw5vezEo1McXnGFDw53W0Zwo,26646
+tensorflow_datasets/testing/test_utils.py,sha256=FouQSpUAuE-ZmiA0QHy_VuQjdabrT_gi7F2odjB4m04,26736
 tensorflow_datasets/testing/test_utils_test.py,sha256=KKKlkXRtp-FidZ_B1taONT-sS46nyzTNcUDdtGS0FMo,9576
 tensorflow_datasets/testing/version_test.py,sha256=PsD-paPb4DkrMvOdskmRvqyXXDp9hOQFDPYoQ9k2osA,2791
 tensorflow_datasets/text/__init__.py,sha256=IvEJrcFmicw-ptk5p3aZkJnXOnpCW61u-k2NBgNgQX8,5319
@@ -2460,10 +2460,10 @@ tensorflow_datasets/vision_language/wit/wit_test.py,sha256=DJ687VN9hAp6SLXnr_P12
 tensorflow_datasets/vision_language/wit_kaggle/__init__.py,sha256=38tJQ73VHz8vOJn-AyZh2we2YJucbSRIgmgcrsC6bQM,719
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle.py,sha256=k-7YD1SGr5bASfdR2_09rrqz-8cpWdIcBWWEXhCvzuk,16903
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle_test.py,sha256=vLlluBW77ASNVC4ix7t8idkSUBI6q1-B7zmRV_ICCQM,1778
-tfds_nightly-4.9.8.dev202505200044.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
-tfds_nightly-4.9.8.dev202505200044.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-tfds_nightly-4.9.8.dev202505200044.dist-info/METADATA,sha256=_W0gTuzS5sXNQIVzrwiSY16_J96yk3DHQuxCgKGhauQ,11879
-tfds_nightly-4.9.8.dev202505200044.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-tfds_nightly-4.9.8.dev202505200044.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
-tfds_nightly-4.9.8.dev202505200044.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
-tfds_nightly-4.9.8.dev202505200044.dist-info/RECORD,,
+tfds_nightly-4.9.8.dev202505220044.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
+tfds_nightly-4.9.8.dev202505220044.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+tfds_nightly-4.9.8.dev202505220044.dist-info/METADATA,sha256=hG0sC5hrzN2O7iYLzt9_ZLcwzz2Dv3xT6IKvGjnhReM,11963
+tfds_nightly-4.9.8.dev202505220044.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+tfds_nightly-4.9.8.dev202505220044.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
+tfds_nightly-4.9.8.dev202505220044.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
+tfds_nightly-4.9.8.dev202505220044.dist-info/RECORD,,

{tfds_nightly-4.9.8.dev202505200044.dist-info → tfds_nightly-4.9.8.dev202505220044.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.1)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any