tfds-nightly 4.9.9.dev202508250044__py3-none-any.whl → 4.9.9.dev202508270044__py3-none-any.whl

tensorflow_datasets/core/utils/croissant_utils.py

@@ -63,16 +63,49 @@ def get_croissant_version(version: str | None) -> str | None:
   return version
 
 
-def get_dataset_name(dataset: mlc.Dataset) -> str:
-  """Returns dataset name of the given MLcroissant dataset."""
+def get_dataset_name(dataset: mlc.Dataset, language: str | None = None) -> str:
+  """Returns dataset name of the given MLcroissant dataset.
+
+  Args:
+    dataset: The MLcroissant dataset.
+    language: For datasets with multiple names in different languages, this
+      argument specifies the language to use.
+  """
   if (url := dataset.metadata.url) and url.startswith(_HUGGINGFACE_URL_PREFIX):
     return url.removeprefix(_HUGGINGFACE_URL_PREFIX)
-  return dataset.metadata.name
+  name = dataset.metadata.name
+  if isinstance(name, dict):
+    if language is None:
+      # Try a heuristic language, e.g., 'en'.
+      if "en" in name:
+        return name["en"]
+      # Otherwise, take the first language in the dict.
+      try:
+        first_lang = next(iter(name))
+        return name[first_lang]
+      except StopIteration as exc:
+        raise ValueError("Dataset name dictionary is empty.") from exc
+    elif language not in dataset.metadata.name:
+      raise ValueError(
+          f"Language {language} not found in dataset names {name}."
+      )
+    else:
+      return name[language]
+  # At this point, name is not a dict anymore.
+  return typing.cast(str, name)
+
+
+def get_tfds_dataset_name(
+    dataset: mlc.Dataset, language: str | None = None
+) -> str:
+  """Returns TFDS compatible dataset name of the given MLcroissant dataset.
 
-
-def get_tfds_dataset_name(dataset: mlc.Dataset) -> str:
-  """Returns TFDS compatible dataset name of the given MLcroissant dataset."""
-  dataset_name = get_dataset_name(dataset)
+  Args:
+    dataset: The MLcroissant dataset.
+    language: For datasets with multiple names in different languages, this
+      argument specifies the language to use.
+  """
+  dataset_name = get_dataset_name(dataset, language=language)
   return conversion_utils.to_tfds_name(dataset_name)
 
 

tensorflow_datasets/core/utils/croissant_utils_test.py

@@ -33,7 +33,83 @@ from tensorflow_datasets.core.utils import croissant_utils
 def test_get_tfds_dataset_name(croissant_name, croissant_url, tfds_name):
   metadata = mlc.Metadata(name=croissant_name, url=croissant_url)
   dataset = mlc.Dataset.from_metadata(metadata)
-  assert croissant_utils.get_tfds_dataset_name(dataset) == tfds_name
+  assert (
+      croissant_utils.get_tfds_dataset_name(dataset) == tfds_name
+  ), f'Expected TFDS name: {tfds_name}'
+
+
+@pytest.mark.parametrize(
+    'croissant_name,language,expected_name',
+    [
+        ({'en': 'English Name', 'fr': 'Nom Français'}, None, 'English Name'),
+        (
+            {'de': 'Deutscher Name', 'fr': 'Nom Français'},
+            None,
+            'Deutscher Name',
+        ),
+        ({'en': 'English Name', 'fr': 'Nom Français'}, 'fr', 'Nom Français'),
+        ('Simple Name', None, 'Simple Name'),
+    ],
+)
+def test_get_dataset_name(croissant_name, language, expected_name):
+  ctx = mlc.Context(conforms_to='http://mlcommons.org/croissant/1.1')
+  metadata = mlc.Metadata(name=croissant_name, ctx=ctx, url=None)
+  dataset = mlc.Dataset.from_metadata(metadata)
+  assert (
+      croissant_utils.get_dataset_name(dataset, language=language)
+      == expected_name
+  )
+
+
+def test_get_dataset_name_url_precedence():
+  ctx = mlc.Context(conforms_to='http://mlcommons.org/croissant/1.1')
+  # Test that URL prefix removal works and takes precedence over name.
+  metadata = mlc.Metadata(
+      name='Should Be Ignored',
+      ctx=ctx,
+      url='https://huggingface.co/datasets/user/dataset_name',
+  )
+  dataset = mlc.Dataset.from_metadata(metadata)
+  assert croissant_utils.get_dataset_name(dataset) == 'user/dataset_name'
+
+  # Test that URL precedence also works when the name is a dict.
+  metadata_dict_name = mlc.Metadata(
+      name={'en': 'Should Be Ignored'},
+      ctx=ctx,
+      url='https://huggingface.co/datasets/another/other_dataset',
+  )
+  dataset_dict_name = mlc.Dataset.from_metadata(metadata_dict_name)
+  assert (
+      croissant_utils.get_dataset_name(dataset_dict_name)
+      == 'another/other_dataset'
+  )
+
+  # Test that non-HuggingFace URLs don't cause name to be ignored.
+  metadata_other_url = mlc.Metadata(
+      name='Not Ignored',
+      ctx=ctx,
+      url='https://example.com/dataset',
+  )
+  dataset_other_url = mlc.Dataset.from_metadata(metadata_other_url)
+  assert croissant_utils.get_dataset_name(dataset_other_url) == 'Not Ignored'
+
+
+def test_get_dataset_multilingual_name_with_language_not_found():
+  ctx = mlc.Context(conforms_to='http://mlcommons.org/croissant/1.1')
+  metadata_lang_not_found = mlc.Metadata(
+      name={'en': 'English Name', 'fr': 'Nom Français'}, ctx=ctx, url=None
+  )
+  dataset_lang_not_found = mlc.Dataset.from_metadata(metadata_lang_not_found)
+  with pytest.raises(ValueError, match='Language de not found'):
+    croissant_utils.get_dataset_name(dataset_lang_not_found, language='de')
+
+
+def test_get_dataset_multilingual_name_with_empty_dict():
+  ctx = mlc.Context(conforms_to='http://mlcommons.org/croissant/1.1')
+  metadata_empty_dict = mlc.Metadata(name={}, ctx=ctx, url=None)
+  dataset_empty_dict = mlc.Dataset.from_metadata(metadata_empty_dict)
+  with pytest.raises(ValueError, match='Dataset name dictionary is empty'):
+    croissant_utils.get_dataset_name(dataset_empty_dict, language=None)
 
 
 @pytest.mark.parametrize(

{tfds_nightly-4.9.9.dev202508250044.dist-info → tfds_nightly-4.9.9.dev202508270044.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfds-nightly
-Version: 4.9.9.dev202508250044
+Version: 4.9.9.dev202508270044
 Summary: tensorflow/datasets is a library of datasets ready to use with TensorFlow.
 Home-page: https://github.com/tensorflow/datasets
 Download-URL: https://github.com/tensorflow/datasets/tags

{tfds_nightly-4.9.9.dev202508250044.dist-info → tfds_nightly-4.9.9.dev202508270044.dist-info}/RECORD

@@ -245,8 +245,8 @@ tensorflow_datasets/core/utils/bool_utils_test.py,sha256=rwFRcYV0wBknvYODjeTgRDq
 tensorflow_datasets/core/utils/colormap.csv,sha256=DDayUU9R19cxhcG3fj4cFwhI46W20U7ofBG0kToUHOw,2732
 tensorflow_datasets/core/utils/conversion_utils.py,sha256=V8kFmJu38op7-8ufZvEn0fLOH8FMkjQebQ1NstIMRYo,6747
 tensorflow_datasets/core/utils/conversion_utils_test.py,sha256=rP_nbzQWzmZc_GXp3Y6TirwIGJqiQbF-JtY3B1tOuN0,5346
-tensorflow_datasets/core/utils/croissant_utils.py,sha256=9C8sScaEqSRsThqpQQc48GDNR1KFmDkS8hmKIvfZCB0,5181
-tensorflow_datasets/core/utils/croissant_utils_test.py,sha256=UdkAVYDTPm1L0zmMESScurV_IMA5K3qAKmL_umeMJZI,4497
+tensorflow_datasets/core/utils/croissant_utils.py,sha256=9-_j86KKKkfxgg0aAM1zxlqCdkaC-0p9XzdWjSLmOwk,6265
+tensorflow_datasets/core/utils/croissant_utils_test.py,sha256=ssIEsdd7BwEPFLwktHu_LXrruWHffVJDme44KqFXglw,7407
 tensorflow_datasets/core/utils/docs.py,sha256=nRE4d8wxYZav8AcT3dkiY0yplAJBx1hygWxkeKj_V7I,1412
 tensorflow_datasets/core/utils/dtype_utils.py,sha256=LvDe1hbgQem57RiqXjG9U5Roj8-1KkBMmSYTtgctx2U,3246
 tensorflow_datasets/core/utils/dtype_utils_test.py,sha256=-Qe2fQzDO5sjS36ZL-dY9w0tNrJXokIoSRFEQCv5dQA,3259
@@ -2471,10 +2471,10 @@ tensorflow_datasets/vision_language/wit/wit_test.py,sha256=PXS8DMNW-MDrT2p5oy4Ic
 tensorflow_datasets/vision_language/wit_kaggle/__init__.py,sha256=vGwSGeM8WE4Q-l0-eEE1sBojmk6YT0l1OO60AWa4Q40,719
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle.py,sha256=q-vX_FBzIwsFxL4sY9vuyQ3UQD2PLM4yhUR4U6l-qao,16903
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle_test.py,sha256=ZymHT1NkmD-pUnh3BmM3_g30c5afsWYnmqDD9dVyDSA,1778
-tfds_nightly-4.9.9.dev202508250044.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
-tfds_nightly-4.9.9.dev202508250044.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-tfds_nightly-4.9.9.dev202508250044.dist-info/METADATA,sha256=aKrw-Y2V5ZIWaZ4d1VO-2hg6rXBtBn8rihFkdtDjkWk,11291
-tfds_nightly-4.9.9.dev202508250044.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-tfds_nightly-4.9.9.dev202508250044.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
-tfds_nightly-4.9.9.dev202508250044.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
-tfds_nightly-4.9.9.dev202508250044.dist-info/RECORD,,
+tfds_nightly-4.9.9.dev202508270044.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
+tfds_nightly-4.9.9.dev202508270044.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+tfds_nightly-4.9.9.dev202508270044.dist-info/METADATA,sha256=tGKucOVOtUXVaj5IMH587tEOr7bsbmyYfwRmsuQYkMU,11291
+tfds_nightly-4.9.9.dev202508270044.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tfds_nightly-4.9.9.dev202508270044.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
+tfds_nightly-4.9.9.dev202508270044.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
+tfds_nightly-4.9.9.dev202508270044.dist-info/RECORD,,