tfds-nightly 4.9.9.dev202508280044__py3-none-any.whl → 4.9.9.dev202508290044__py3-none-any.whl

tensorflow_datasets/core/dataset_builders/croissant_builder.py

@@ -85,6 +85,7 @@ def array_datatype_converter(
     feature: type_utils.TfdsDType | feature_lib.FeatureConnector | None,
     field: mlc.Field,
     dtype_mapping: Mapping[type_utils.TfdsDType, type_utils.TfdsDType],
+    language: str | None = None,
 ):
   """Includes the given feature in a sequence or tensor feature.
 
@@ -97,6 +98,10 @@ def array_datatype_converter(
     field: The mlc.Field object.
     dtype_mapping: A mapping of dtypes to the corresponding dtypes that will be
       used in TFDS.
+    language: For Croissant jsonld which include multi-lingual descriptions, the
+      language code to use to extract the description to be used in TFDS. If
+      None, it will extract the description in English or the first available
+      language in the dictionary.
 
   Returns:
     A sequence or tensor feature including the inner feature.
@@ -108,7 +113,7 @@ def array_datatype_converter(
     field_dtype = field.data_type
 
   description = croissant_utils.extract_localized_string(
-      field.description, field_name='description'
+      field.description, language=language, field_name='description'
   )
 
   if len(field.array_shape_tuple) == 1:
@@ -129,6 +134,7 @@ def datatype_converter(
     field: mlc.Field,
     int_dtype: type_utils.TfdsDType = np.int64,
     float_dtype: type_utils.TfdsDType = np.float32,
+    language: str | None = None,
 ):
   """Converts a Croissant field to a TFDS-compatible feature.
 
@@ -137,6 +143,10 @@ def datatype_converter(
     int_dtype: The dtype to use for TFDS integer features. Defaults to np.int64.
     float_dtype: The dtype to use for TFDS float features. Defaults to
       np.float32.
+    language: For Croissant jsonld which include multi-lingual descriptions, the
+      language code to use to extract the description to be used in TFDS. If
+      None, it will extract the description in English or the first available
+      language in the dictionary.
 
   Returns:
     Converted datatype for TFDS, or None when a Field does not specify a type.
@@ -156,7 +166,7 @@ def datatype_converter(
 
   field_data_type = field.data_type
   description = croissant_utils.extract_localized_string(
-      field.description, field_name='description'
+      field.description, language=language, field_name='description'
   )
 
   if not field_data_type:
@@ -165,7 +175,10 @@ def datatype_converter(
       feature = features_dict.FeaturesDict(
           {
               subfield.id: datatype_converter(
-                  subfield, int_dtype=int_dtype, float_dtype=float_dtype
+                  subfield,
+                  int_dtype=int_dtype,
+                  float_dtype=float_dtype,
+                  language=language,
               )
               for subfield in field.sub_fields
           },
@@ -215,6 +228,7 @@ def datatype_converter(
         feature=feature,
         field=field,
         dtype_mapping=dtype_mapping,
+        language=language,
     )
   # If the field is repeated, we return a sequence feature. `field.repeated` is
   # deprecated starting from Croissant 1.1, but we still support it for

{tfds_nightly-4.9.9.dev202508280044.dist-info → tfds_nightly-4.9.9.dev202508290044.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfds-nightly
-Version: 4.9.9.dev202508280044
+Version: 4.9.9.dev202508290044
 Summary: tensorflow/datasets is a library of datasets ready to use with TensorFlow.
 Home-page: https://github.com/tensorflow/datasets
 Download-URL: https://github.com/tensorflow/datasets/tags

{tfds_nightly-4.9.9.dev202508280044.dist-info → tfds_nightly-4.9.9.dev202508290044.dist-info}/RECORD

@@ -141,7 +141,7 @@ tensorflow_datasets/core/data_sources/python_test.py,sha256=O3yqMPx40JlHN0uFfZPN
 tensorflow_datasets/core/dataset_builders/__init__.py,sha256=StTA3euephqDZdpTzJQgfWNqB5inZosrAhaWg2BOeio,1945
 tensorflow_datasets/core/dataset_builders/adhoc_builder.py,sha256=1a-5hVjf9t24SD9fWzDDuKoOrA-Vmydf5QxvU7ap-sI,9263
 tensorflow_datasets/core/dataset_builders/adhoc_builder_test.py,sha256=yhRwrznK78MvHeWGRggnMTiyx_SlR1z30iD5VU3Gweo,13096
-tensorflow_datasets/core/dataset_builders/croissant_builder.py,sha256=nmRIRZZGJjXtJgcvlTOsNKbqsAjosjn_M_zOu86uc04,17253
+tensorflow_datasets/core/dataset_builders/croissant_builder.py,sha256=EoK93nCth72yF_Qxjecahu_gLIo58ci2GEz3XkKjanc,17998
 tensorflow_datasets/core/dataset_builders/croissant_builder_test.py,sha256=ordMGrhNh-S1MjfY0QO8HUnsangqJCQCo3wCVBvMToA,16220
 tensorflow_datasets/core/dataset_builders/huggingface_dataset_builder.py,sha256=Loq3qeGk1Ias-d2oT_dK47BRNgTA4LKJchNGh7aA4a0,18313
 tensorflow_datasets/core/dataset_builders/huggingface_dataset_builder_test.py,sha256=6N3DLsry9LhDqhpleaoXrrhaGiLJMBgUlwDnAji-1fI,4389
@@ -2471,10 +2471,10 @@ tensorflow_datasets/vision_language/wit/wit_test.py,sha256=PXS8DMNW-MDrT2p5oy4Ic
 tensorflow_datasets/vision_language/wit_kaggle/__init__.py,sha256=vGwSGeM8WE4Q-l0-eEE1sBojmk6YT0l1OO60AWa4Q40,719
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle.py,sha256=q-vX_FBzIwsFxL4sY9vuyQ3UQD2PLM4yhUR4U6l-qao,16903
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle_test.py,sha256=ZymHT1NkmD-pUnh3BmM3_g30c5afsWYnmqDD9dVyDSA,1778
-tfds_nightly-4.9.9.dev202508280044.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
-tfds_nightly-4.9.9.dev202508280044.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-tfds_nightly-4.9.9.dev202508280044.dist-info/METADATA,sha256=NYDGgvfbautnlGWNWg2_8pOq9eblXgaPjGU-2__DPco,11291
-tfds_nightly-4.9.9.dev202508280044.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-tfds_nightly-4.9.9.dev202508280044.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
-tfds_nightly-4.9.9.dev202508280044.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
-tfds_nightly-4.9.9.dev202508280044.dist-info/RECORD,,
+tfds_nightly-4.9.9.dev202508290044.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
+tfds_nightly-4.9.9.dev202508290044.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+tfds_nightly-4.9.9.dev202508290044.dist-info/METADATA,sha256=nkC_JlCaoq6KaOyqYSu_lwR9AZuWyUJPGTXRfs07lD4,11291
+tfds_nightly-4.9.9.dev202508290044.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tfds_nightly-4.9.9.dev202508290044.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
+tfds_nightly-4.9.9.dev202508290044.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
+tfds_nightly-4.9.9.dev202508290044.dist-info/RECORD,,