tfds-nightly 4.9.9.dev202507210045__py3-none-any.whl → 4.9.9.dev202507230045__py3-none-any.whl

tensorflow_datasets/core/dataset_builders/croissant_builder.py

@@ -37,6 +37,7 @@ print(ds['default'][0])
 from __future__ import annotations
 
 from collections.abc import Mapping, Sequence
+import datetime
 import json
 from typing import Any
 
@@ -133,10 +134,11 @@ def datatype_converter(
       np.float32.
 
   Returns:
-    Converted datatype for TFDS.
+    Converted datatype for TFDS, or None when a Field does not specify a type.
 
   Raises:
-    NotImplementedError
+    NotImplementedError when the feature is not supported yet, or ValueError
+    when a Field is malformed.
   """
   if field.is_enumeration:
     raise NotImplementedError('Not implemented yet.')
@@ -150,7 +152,7 @@ def datatype_converter(
   field_data_type = field.data_type
 
   if not field_data_type:
-    # Fields with sub fields are of type None
+    # Fields with sub fields are of type None.
     if field.sub_fields:
       feature = features_dict.FeaturesDict(
           {
@@ -169,8 +171,9 @@ def datatype_converter(
     feature = dtype_mapping[field_data_type]
   elif enp.lazy.is_np_dtype(field_data_type):
     feature = field_data_type
-  # We return a text feature for mlc.DataType.DATE features.
-  elif field_data_type == pd.Timestamp:
+  # We return a text feature for date-time features (mlc.DataType.DATE,
+  # mlc.DataType.DATETIME, and mlc.DataType.TIME).
+  elif field_data_type == pd.Timestamp or field_data_type == datetime.time:
     feature = text_feature.Text(doc=field.description)
   elif field_data_type == mlc.DataType.IMAGE_OBJECT:
     feature = image_feature.Image(doc=field.description)
@@ -193,7 +196,9 @@ def datatype_converter(
         doc=field.description, sample_rate=field.source.sampling_rate
     )
   else:
-    raise ValueError(f'Unknown data type: {field_data_type}.')
+    raise ValueError(
+        f'Unknown data type: {field_data_type} for field {field.id}.'
+    )
 
   if feature and field.is_array:
     feature = array_datatype_converter(

tensorflow_datasets/core/dataset_builders/croissant_builder_test.py

@@ -165,6 +165,18 @@ def test_bbox_datatype_converter_with_invalid_format():
             text_feature.Text,
             None,
         ),
+        (
+            mlc.Field(
+                data_types=mlc.DataType.DATETIME, description="DateTime feature"
+            ),
+            text_feature.Text,
+            None,
+        ),
+        (
+            mlc.Field(data_types=mlc.DataType.TIME, description="Time feature"),
+            text_feature.Text,
+            None,
+        ),
         (
             mlc.Field(
                 data_types=mlc.DataType.IMAGE_OBJECT,
@@ -218,6 +230,13 @@ def test_complex_datatype_converter(field, feature_type, subfield_types):
       )
 
 
+def test_datatype_converter_none():
+  field = mlc.Field(
+      name="my_field", id="my_field", description="Field with empty data type."
+  )
+  assert croissant_builder.datatype_converter(field) is None
+
+
 def test_multidimensional_datatype_converter():
   field = mlc.Field(
       data_types=mlc.DataType.TEXT,

tensorflow_datasets/core/features/video_feature.py

@@ -93,7 +93,7 @@ class Video(sequence_feature.Sequence):
 
   def __init__(
       self,
-      shape: Sequence[Optional[int]],
+      shape: Sequence[Optional[int]] | None = None,
       encoding_format: str = 'png',
       ffmpeg_extra_args: Sequence[str] = (),
       use_colormap: bool = False,
@@ -103,8 +103,8 @@ class Video(sequence_feature.Sequence):
     """Initializes the connector.
 
     Args:
-      shape: tuple of ints, the shape of the video (num_frames, height, width,
-        channels), where channels is 1 or 3.
+      shape: The shape of the video (num_frames, height, width, channels), where
+        channels is 1 or 3.
       encoding_format: The video is stored as a sequence of encoded images. You
         can use any encoding format supported by image_feature.Feature.
       ffmpeg_extra_args: A sequence of additional args to be passed to the
@@ -121,19 +121,22 @@ class Video(sequence_feature.Sequence):
       ValueError: If the shape is invalid
     """
     dtype = tf.dtypes.as_dtype(dtype)
-    shape = tuple(shape)
-    if len(shape) != 4:
-      raise ValueError('Video shape should be of rank 4')
+    frame_shape = None
+    if shape:
+      shape = tuple(shape)
+      if len(shape) != 4:
+        raise ValueError('Video shape should be of rank 4')
+      frame_shape = shape[1:]
     self._encoding_format = encoding_format
     self._extra_ffmpeg_args = list(ffmpeg_extra_args or [])
     super(Video, self).__init__(
         image_feature.Image(
-            shape=shape[1:],
+            shape=frame_shape,
             dtype=dtype,
             encoding_format=encoding_format,
             use_colormap=use_colormap,
         ),
-        length=shape[0],
+        length=shape[0] if shape else None,
     )
 
   def _ffmpeg_decode(self, path_or_fobj):

tensorflow_datasets/core/features/video_feature_test.py

@@ -48,6 +48,22 @@ class VideoFeatureTest(testing.FeatureExpectationsTestCase):
         test_attributes=dict(_encoding_format='png', _extra_ffmpeg_args=[]),
     )
 
+  def test_video_with_none_shape(self):
+    np_video = np.random.randint(256, size=(128, 64, 64, 3), dtype=np.uint8)
+
+    self.assertFeature(
+        feature=features.Video(shape=None),
+        shape=(None, None, None, 3),
+        dtype=tf.uint8,
+        tests=[
+            testing.FeatureExpectationItem(
+                value=np_video,
+                expected=np_video,
+            ),
+        ],
+        test_attributes=dict(_encoding_format='png', _extra_ffmpeg_args=[]),
+    )
+
   def test_video_concatenated_frames(self):
     video_shape = (None, 400, 640, 3)
     lsun_examples_path = os.path.join(self._test_data_path, 'lsun_examples')
@@ -119,6 +135,5 @@ class VideoFeatureTest(testing.FeatureExpectationsTestCase):
           ],
       )
 
-
 if __name__ == '__main__':
   testing.test_main()

tensorflow_datasets/core/subsplits_utils.py

@@ -119,7 +119,7 @@ def even_splits(
       not evenly divisible by `n`. If `False`, examples are distributed evenly
       across subsplits, starting by the first. For example, if there is 11
       examples with `n=3`, splits will contain `[4, 4, 3]` examples
-      respectivelly.
+      respectively.
 
   Returns:
     The list of subsplits. Those splits can be combined together (with
@@ -169,7 +169,7 @@ def split_for_jax_process(
       not evenly divisible by `n`. If `False`, examples are distributed evenly
       across subsplits, starting by the first. For example, if there is 11
       examples with `n=3`, splits will contain `[4, 4, 3]` examples
-      respectivelly.
+      respectively.
 
   Returns:
     subsplit: The sub-split of the given `split` for the current

tensorflow_datasets/core/utils/huggingface_utils.py

@@ -119,6 +119,8 @@ def convert_hf_features(hf_features) -> feature_lib.FeatureConnector:
           sample_rate=hf_features.sampling_rate,
           dtype=np.int32,
       )
+    case hf_datasets.Video():
+      return feature_lib.Video()
 
   raise TypeError(f'Type {type(hf_features)} is not supported.')
 

tensorflow_datasets/testing/dataset_builder_testing.py

@@ -191,7 +191,7 @@ class DatasetBuilderTestCase(
     # The `dl_manager.download` and `dl_manager.download_and_extract` are
     # patched to record the urls in `_download_urls`.
     # Calling `dl_manager.download_checksums` stop the url
-    # registration (as checksums are stored remotelly)
+    # registration (as checksums are stored remotely)
     # `_test_checksums` validates the recorded urls.
     self._download_urls = set()
     self._stop_record_download = False
@@ -291,7 +291,7 @@ class DatasetBuilderTestCase(
   def _add_url(self, url_or_urls):
     if self._stop_record_download:
       # Stop record the checksums if dl_manager.download_checksums has been
-      # called (as checksums may be stored remotelly)
+      # called (as checksums may be stored remotely).
       return
     if isinstance(url_or_urls, download.resource.Resource):
       self._download_urls.add(url_or_urls.url)

tensorflow_datasets/testing/test_utils.py

@@ -147,7 +147,7 @@ class MockFs(object):
       with self._mock() as m:
         yield m
       self._tmp_dir = None
-      # TODO(epot): recursivelly record all
+      # TODO(epot): recursively record all.
 
   def _to_tmp(self, p, *, with_state: bool = False):
     """Normalize the path by returning `tmp_path / p`."""

{tfds_nightly-4.9.9.dev202507210045.dist-info → tfds_nightly-4.9.9.dev202507230045.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfds-nightly
-Version: 4.9.9.dev202507210045
+Version: 4.9.9.dev202507230045
 Summary: tensorflow/datasets is a library of datasets ready to use with TensorFlow.
 Home-page: https://github.com/tensorflow/datasets
 Download-URL: https://github.com/tensorflow/datasets/tags

{tfds_nightly-4.9.9.dev202507210045.dist-info → tfds_nightly-4.9.9.dev202507230045.dist-info}/RECORD

@@ -104,7 +104,7 @@ tensorflow_datasets/core/split_builder.py,sha256=cpz-YowMhmiZZVp7eQPNrh23KvE0-Ef
 tensorflow_datasets/core/split_builder_test.py,sha256=kBUVUnQQB_c82AhgjhK3hoYfiAqLt7tDFTzsvZRGQCw,3223
 tensorflow_datasets/core/splits.py,sha256=O3jK4Dalp4tEPeZ9AHbkpW1UkJ6uv5m4YRu2x_ZZTJ4,29418
 tensorflow_datasets/core/splits_test.py,sha256=KrM82r0YsJRTGfpYUCkBxiGDC7BjZFcTvJ-Hbo6HwF0,24987
-tensorflow_datasets/core/subsplits_utils.py,sha256=BPHVPAvHlqt4d3HUr4J2Znn8G63pXLPQ29TBi484MOE,6127
+tensorflow_datasets/core/subsplits_utils.py,sha256=6mVCr-QNZfNgX0Ka_htsqmr-JgFXJXJ7IFfl1ytCQio,6125
 tensorflow_datasets/core/subsplits_utils_test.py,sha256=TIRLtfaf2n38pByhpqYTXEEvs8hrWe2eXk9RFdBMrFQ,5159
 tensorflow_datasets/core/tf_compat.py,sha256=qdZUtaO9FsZUds7Wf0w0MoRydPPRsuZ0_8ebRJg19gg,1820
 tensorflow_datasets/core/units.py,sha256=m3ht8oM8wr6oTU3tCbKOj1yaPyXn1MCu7dUjzw0LrPY,1975
@@ -141,8 +141,8 @@ tensorflow_datasets/core/data_sources/python_test.py,sha256=O3yqMPx40JlHN0uFfZPN
 tensorflow_datasets/core/dataset_builders/__init__.py,sha256=StTA3euephqDZdpTzJQgfWNqB5inZosrAhaWg2BOeio,1945
 tensorflow_datasets/core/dataset_builders/adhoc_builder.py,sha256=QVE8wWGPOgILPTC27Q28QZ3KIi5N64OGOfKpTq4W4_0,9216
 tensorflow_datasets/core/dataset_builders/adhoc_builder_test.py,sha256=yhRwrznK78MvHeWGRggnMTiyx_SlR1z30iD5VU3Gweo,13096
-tensorflow_datasets/core/dataset_builders/croissant_builder.py,sha256=Ef9fSTxFIvUKf_zfCqL3JyUje6f9buIQXvT1iuHgB20,16596
-tensorflow_datasets/core/dataset_builders/croissant_builder_test.py,sha256=1KPrGPFYHQvo33TwnG5LmPpdlyRTudPfni4ipFsj0ao,11607
+tensorflow_datasets/core/dataset_builders/croissant_builder.py,sha256=CCiXOgcr5VJYaQlSf_ss_712BtrDuP6QCyP0K4UgKFs,16876
+tensorflow_datasets/core/dataset_builders/croissant_builder_test.py,sha256=O5j9pUKpgtZKpFQYPAYKQ7DMHXVtuaF_7lwjZZxFRzc,12151
 tensorflow_datasets/core/dataset_builders/huggingface_dataset_builder.py,sha256=Loq3qeGk1Ias-d2oT_dK47BRNgTA4LKJchNGh7aA4a0,18313
 tensorflow_datasets/core/dataset_builders/huggingface_dataset_builder_test.py,sha256=6N3DLsry9LhDqhpleaoXrrhaGiLJMBgUlwDnAji-1fI,4389
 tensorflow_datasets/core/dataset_builders/view_builder.py,sha256=eaCtjN5Vg4rK8JD3auA4PhF9mjH5HvQ9dslDX8LbwyM,11907
@@ -213,8 +213,8 @@ tensorflow_datasets/core/features/top_level_feature.py,sha256=JeOnaBUqp-xFLuPxUt
 tensorflow_datasets/core/features/top_level_feature_test.py,sha256=JutGHU-08tg5KWiB3mIB6Q3a80CvS5_F6jG0bfAYXWM,3628
 tensorflow_datasets/core/features/translation_feature.py,sha256=Qmx39XwMJy18u9eoZlT3Spc0VT0qtqsTHahWoETLZZo,8284
 tensorflow_datasets/core/features/translation_feature_test.py,sha256=iK8ckwApuMu13BS1-vkny-m_NV6uNTz6ky5gbZEfxoc,6060
-tensorflow_datasets/core/features/video_feature.py,sha256=K4lld2N-a9TWrwYss8gEBGnkiraKqKyM81Z4chUj8qU,7720
-tensorflow_datasets/core/features/video_feature_test.py,sha256=qUFu8quELWr15r_VaMZGKoFPb2ueZkNmVJHBcgR5fkc,3772
+tensorflow_datasets/core/features/video_feature.py,sha256=o4tuH4HxkG_sYCYUx8R-LVHcuIgFcfCNe8n0cE5MISk,7813
+tensorflow_datasets/core/features/video_feature_test.py,sha256=k0qpYvZIC_-xXTG9EN1Mo2d9RA77CcvJYl9c0U3oXcQ,4255
 tensorflow_datasets/core/folder_dataset/__init__.py,sha256=Pn2mSU-CPxC89lvywHAD-XrhQj0mvAaqZogpekjr-bs,1515
 tensorflow_datasets/core/folder_dataset/compute_split_utils.py,sha256=Ob_ZaqfS00zViAtRhHK_ff7R8eJAtYDDh6XjQGXdcP4,13515
 tensorflow_datasets/core/folder_dataset/compute_split_utils_test.py,sha256=XBo4UC1IydAPuIP1SY2psrEVeEr3y0KPmCEje6yQWhs,3784
@@ -256,7 +256,7 @@ tensorflow_datasets/core/utils/file_utils.py,sha256=vL-ulAVClrvkA71DvEvdGR2EdNmO
 tensorflow_datasets/core/utils/file_utils_test.py,sha256=SCw_XFRhyxGCFEVjt9pOdupsoULPdi8iT38JBrnUuDM,13708
 tensorflow_datasets/core/utils/gcs_utils.py,sha256=8mBOgEepkah1Rw36F6DNIVhLzfXbR8iS8KMLQUM5sPk,5154
 tensorflow_datasets/core/utils/gcs_utils_test.py,sha256=Ig8S37AvFG2g7kNjYxqgmqNKlLPeXt31XD7RY4UzsDg,2578
-tensorflow_datasets/core/utils/huggingface_utils.py,sha256=J-TZyisMkjqsFWw6MAA0NWot4KPDmfWe_7t1R9wjQv4,5262
+tensorflow_datasets/core/utils/huggingface_utils.py,sha256=NeYaUoO3vIFH8M0hZ8k4w7AchFZJIGsuV1XwKJVttfw,5325
 tensorflow_datasets/core/utils/huggingface_utils_test.py,sha256=wYKY5vh5q4ImpkvDjZWwcTbH1s2YORKpsklA-9Qwfxs,4792
 tensorflow_datasets/core/utils/image_utils.py,sha256=5xHKJO8wsPGZpuFoBsvwaXp_-pnrtwXvyLBSK7itAm4,5939
 tensorflow_datasets/core/utils/image_utils_test.py,sha256=6QLpWwveq4Jtw0nLxG4S-VGpVsI9qwr6bJm0Vgunbu0,3127
@@ -2122,7 +2122,7 @@ tensorflow_datasets/summarization/media_sum/media_sum.py,sha256=CIhR_cfQb1aEfu9B
 tensorflow_datasets/summarization/summscreen/__init__.py,sha256=ADxohrpUPJjug4r2kGCCJEWZzVD4s2S0smqLfjkc8YY,718
 tensorflow_datasets/summarization/summscreen/summscreen.py,sha256=DfwGr3vsRhOC62ODJ1Sp7-v219bPjJ93KK043YReV7I,884
 tensorflow_datasets/testing/__init__.py,sha256=aSwY_kciK-EZXp1D_JRkuuCJwtbFljGZ72c9YNB6yfE,6049
-tensorflow_datasets/testing/dataset_builder_testing.py,sha256=Ers73TcGgVjjLWvpfuKmr34QSBl6QB3Z9qvFPzSHjSE,25094
+tensorflow_datasets/testing/dataset_builder_testing.py,sha256=ziE2twrc1-LQExGp4g5Nbq9hlbFow3VdX8RTC83R6bM,25093
 tensorflow_datasets/testing/dataset_builder_testing_test.py,sha256=Nf7Ykg5bY5o9ZatQKrRJhr-qGTtNKle4aZph4rt72i4,1283
 tensorflow_datasets/testing/dataset_collection_builder_testing.py,sha256=tUv2l53rc9GEo4sWvM9OP9r-Ze54dcDakeLQBMS7yos,4825
 tensorflow_datasets/testing/dataset_collection_builder_testing_test.py,sha256=Dw5tACaDjVt9CZi0V84tMAh2JJexrRwWF1N3DID1Mbs,1155
@@ -2132,7 +2132,7 @@ tensorflow_datasets/testing/mocking.py,sha256=4mIq0ngxfs3w0hFlosGOSTp-mAQVfBfoFw
 tensorflow_datasets/testing/mocking_test.py,sha256=9DMkxcQw_dZTKULNHiKv91e0VcBsUTa6FIhUOLvJKls,13796
 tensorflow_datasets/testing/test_case.py,sha256=_H_M3pp6Vp3dbtPyVy5Um7X8S4V4EKPLrao1mbS2IdU,2554
 tensorflow_datasets/testing/test_case_in_context.py,sha256=7YrdTI_rqR01Q-ToVqewIm1OKDwvxIidPhaffYmjP1E,1872
-tensorflow_datasets/testing/test_utils.py,sha256=wFWG9jryWFA3NnNBrEva4u6HWRu9yfQbXQdYpHkDNKw,26736
+tensorflow_datasets/testing/test_utils.py,sha256=sQTTXa8YHPXml514vayxiu_E6qHFQ_1Maizy3OR0J8Y,26736
 tensorflow_datasets/testing/test_utils_test.py,sha256=nL2niozCO5Gh4cWPWbDW5_w3w-mHRYZEQmmfej2fpjY,9576
 tensorflow_datasets/testing/version_test.py,sha256=fNMSX1FSNs_66MHcRGAWzoPZWJ-sAvmc-rceKXGK-uM,2791
 tensorflow_datasets/text/__init__.py,sha256=_PtJTw2LQqgxFNVeBCEXrLGF2qg5NNOiXTW9oKZR_ZA,5319
@@ -2468,10 +2468,10 @@ tensorflow_datasets/vision_language/wit/wit_test.py,sha256=PXS8DMNW-MDrT2p5oy4Ic
 tensorflow_datasets/vision_language/wit_kaggle/__init__.py,sha256=vGwSGeM8WE4Q-l0-eEE1sBojmk6YT0l1OO60AWa4Q40,719
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle.py,sha256=q-vX_FBzIwsFxL4sY9vuyQ3UQD2PLM4yhUR4U6l-qao,16903
 tensorflow_datasets/vision_language/wit_kaggle/wit_kaggle_test.py,sha256=ZymHT1NkmD-pUnh3BmM3_g30c5afsWYnmqDD9dVyDSA,1778
-tfds_nightly-4.9.9.dev202507210045.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
-tfds_nightly-4.9.9.dev202507210045.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-tfds_nightly-4.9.9.dev202507210045.dist-info/METADATA,sha256=DYsSCdhyzArKSDLHHfJR1cUcV7MJpg9I231kc68ETak,11694
-tfds_nightly-4.9.9.dev202507210045.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-tfds_nightly-4.9.9.dev202507210045.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
-tfds_nightly-4.9.9.dev202507210045.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
-tfds_nightly-4.9.9.dev202507210045.dist-info/RECORD,,
+tfds_nightly-4.9.9.dev202507230045.dist-info/licenses/AUTHORS,sha256=nvBG4WwfgjuOu1oZkuQKw9kg7X6rve679ObS-YDDmXg,309
+tfds_nightly-4.9.9.dev202507230045.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+tfds_nightly-4.9.9.dev202507230045.dist-info/METADATA,sha256=fha1BBcJdcuOuYN_oHawuzQx_EsXdW6fWiqn1eDa3OI,11694
+tfds_nightly-4.9.9.dev202507230045.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tfds_nightly-4.9.9.dev202507230045.dist-info/entry_points.txt,sha256=eHEL7nF5y1uCY2FgkuYIdE062epJXlAQTSdq89px4p4,73
+tfds_nightly-4.9.9.dev202507230045.dist-info/top_level.txt,sha256=bAevmk9209s_oxVZVlN6hSDIVS423qrMQvmcWSvW4do,20
+tfds_nightly-4.9.9.dev202507230045.dist-info/RECORD,,