datasets 5.0.1__tar.gz → 5.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. {datasets-5.0.1/src/datasets.egg-info → datasets-5.2.0}/PKG-INFO +19 -10
  2. {datasets-5.0.1 → datasets-5.2.0}/README.md +1 -1
  3. {datasets-5.0.1 → datasets-5.2.0}/setup.py +17 -9
  4. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/__init__.py +1 -1
  5. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/arrow_dataset.py +143 -52
  6. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/arrow_writer.py +50 -7
  7. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/builder.py +14 -7
  8. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/combine.py +2 -0
  9. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/config.py +2 -0
  10. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/data_files.py +2 -0
  11. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/dataset_dict.py +16 -28
  12. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/distributed.py +18 -6
  13. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/download/download_config.py +2 -0
  14. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/features/__init__.py +4 -0
  15. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/features/audio.py +3 -0
  16. datasets-5.2.0/src/datasets/features/bio_sequence.py +303 -0
  17. datasets-5.2.0/src/datasets/features/bio_structure.py +244 -0
  18. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/features/features.py +85 -4
  19. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/features/image.py +3 -0
  20. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/features/mesh.py +7 -4
  21. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/features/nifti.py +5 -2
  22. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/features/pdf.py +4 -5
  23. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/features/video.py +3 -0
  24. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/formatting/np_formatter.py +4 -1
  25. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/info.py +4 -1
  26. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/io/csv.py +3 -0
  27. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/io/json.py +15 -0
  28. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/io/parquet.py +6 -12
  29. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/iterable_dataset.py +562 -152
  30. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/load.py +11 -40
  31. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/__init__.py +40 -0
  32. datasets-5.2.0/src/datasets/packaged_modules/fasta/fasta.py +289 -0
  33. datasets-5.2.0/src/datasets/packaged_modules/fastq/fastq.py +313 -0
  34. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/folder_based_builder/folder_based_builder.py +10 -5
  35. datasets-5.2.0/src/datasets/packaged_modules/genbank/genbank.py +735 -0
  36. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/generator/generator.py +5 -0
  37. datasets-5.2.0/src/datasets/packaged_modules/harbor/harbor.py +224 -0
  38. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/hdf5/hdf5.py +41 -15
  39. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/json/json.py +27 -1
  40. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/meshfolder/meshfolder.py +2 -0
  41. datasets-5.2.0/src/datasets/packaged_modules/mmcif/mmcif.py +68 -0
  42. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/niftifolder/niftifolder.py +1 -0
  43. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/pandas/pandas.py +8 -6
  44. datasets-5.2.0/src/datasets/packaged_modules/pdb/pdb.py +68 -0
  45. datasets-5.2.0/src/datasets/packaged_modules/text/__init__.py +0 -0
  46. datasets-5.2.0/src/datasets/packaged_modules/tsfile/__init__.py +0 -0
  47. datasets-5.2.0/src/datasets/packaged_modules/videofolder/__init__.py +0 -0
  48. datasets-5.2.0/src/datasets/packaged_modules/vortex/__init__.py +0 -0
  49. datasets-5.2.0/src/datasets/packaged_modules/vortex/vortex.py +362 -0
  50. datasets-5.2.0/src/datasets/packaged_modules/webdataset/__init__.py +0 -0
  51. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/webdataset/webdataset.py +4 -0
  52. datasets-5.2.0/src/datasets/packaged_modules/xml/__init__.py +0 -0
  53. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/splits.py +10 -0
  54. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/table.py +9 -2
  55. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/extract.py +20 -4
  56. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/file_utils.py +38 -24
  57. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/json.py +32 -6
  58. datasets-5.2.0/src/datasets/utils/resources/__init__.py +0 -0
  59. {datasets-5.0.1 → datasets-5.2.0/src/datasets.egg-info}/PKG-INFO +19 -10
  60. {datasets-5.0.1 → datasets-5.2.0}/src/datasets.egg-info/SOURCES.txt +16 -0
  61. {datasets-5.0.1 → datasets-5.2.0}/src/datasets.egg-info/requires.txt +30 -8
  62. {datasets-5.0.1 → datasets-5.2.0}/tests/test_arrow_dataset.py +222 -2
  63. {datasets-5.0.1 → datasets-5.2.0}/tests/test_arrow_writer.py +63 -1
  64. datasets-5.2.0/tests/test_distributed.py +272 -0
  65. {datasets-5.0.1 → datasets-5.2.0}/tests/test_extract.py +28 -2
  66. {datasets-5.0.1 → datasets-5.2.0}/tests/test_file_utils.py +81 -0
  67. {datasets-5.0.1 → datasets-5.2.0}/tests/test_iterable_dataset.py +1097 -40
  68. {datasets-5.0.1 → datasets-5.2.0}/tests/test_load.py +37 -1
  69. {datasets-5.0.1 → datasets-5.2.0}/tests/test_offline_util.py +3 -5
  70. {datasets-5.0.1 → datasets-5.2.0}/tests/test_search.py +18 -1
  71. {datasets-5.0.1 → datasets-5.2.0}/tests/test_table.py +19 -1
  72. {datasets-5.0.1 → datasets-5.2.0}/tests/test_upstream_hub.py +0 -8
  73. datasets-5.0.1/tests/test_distributed.py +0 -139
  74. {datasets-5.0.1 → datasets-5.2.0}/AUTHORS +0 -0
  75. {datasets-5.0.1 → datasets-5.2.0}/LICENSE +0 -0
  76. {datasets-5.0.1 → datasets-5.2.0}/pyproject.toml +0 -0
  77. {datasets-5.0.1 → datasets-5.2.0}/setup.cfg +0 -0
  78. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/arrow_reader.py +0 -0
  79. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/commands/__init__.py +0 -0
  80. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/commands/datasets_cli.py +0 -0
  81. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/commands/delete_from_hub.py +0 -0
  82. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/commands/env.py +0 -0
  83. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/commands/test.py +0 -0
  84. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/download/__init__.py +0 -0
  85. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/download/download_manager.py +0 -0
  86. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/download/streaming_download_manager.py +0 -0
  87. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/exceptions.py +0 -0
  88. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/features/_torchcodec.py +0 -0
  89. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/features/translation.py +0 -0
  90. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/filesystems/__init__.py +0 -0
  91. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/filesystems/compression.py +0 -0
  92. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/fingerprint.py +0 -0
  93. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/formatting/__init__.py +0 -0
  94. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/formatting/formatting.py +0 -0
  95. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/formatting/jax_formatter.py +0 -0
  96. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/formatting/polars_formatter.py +0 -0
  97. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/formatting/tf_formatter.py +0 -0
  98. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/formatting/torch_formatter.py +0 -0
  99. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/hub.py +0 -0
  100. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/inspect.py +0 -0
  101. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/io/__init__.py +0 -0
  102. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/io/abc.py +0 -0
  103. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/io/generator.py +0 -0
  104. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/io/spark.py +0 -0
  105. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/io/sql.py +0 -0
  106. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/io/text.py +0 -0
  107. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/naming.py +0 -0
  108. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/arrow/__init__.py +0 -0
  109. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/arrow/arrow.py +0 -0
  110. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/audiofolder/__init__.py +0 -0
  111. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/audiofolder/audiofolder.py +0 -0
  112. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/cache/__init__.py +0 -0
  113. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/cache/cache.py +0 -0
  114. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/conll/__init__.py +0 -0
  115. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/conll/conll.py +0 -0
  116. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/csv/__init__.py +0 -0
  117. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/csv/csv.py +0 -0
  118. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/eval/__init__.py +0 -0
  119. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/eval/eval.py +0 -0
  120. {datasets-5.0.1/src/datasets/packaged_modules/folder_based_builder → datasets-5.2.0/src/datasets/packaged_modules/fasta}/__init__.py +0 -0
  121. {datasets-5.0.1/src/datasets/packaged_modules/generator → datasets-5.2.0/src/datasets/packaged_modules/fastq}/__init__.py +0 -0
  122. {datasets-5.0.1/src/datasets/packaged_modules/hdf5 → datasets-5.2.0/src/datasets/packaged_modules/folder_based_builder}/__init__.py +0 -0
  123. {datasets-5.0.1/src/datasets/packaged_modules/iceberg → datasets-5.2.0/src/datasets/packaged_modules/genbank}/__init__.py +0 -0
  124. {datasets-5.0.1/src/datasets/packaged_modules/imagefolder → datasets-5.2.0/src/datasets/packaged_modules/generator}/__init__.py +0 -0
  125. {datasets-5.0.1/src/datasets/packaged_modules/json → datasets-5.2.0/src/datasets/packaged_modules/harbor}/__init__.py +0 -0
  126. {datasets-5.0.1/src/datasets/packaged_modules/lance → datasets-5.2.0/src/datasets/packaged_modules/hdf5}/__init__.py +0 -0
  127. {datasets-5.0.1/src/datasets/packaged_modules/meshfolder → datasets-5.2.0/src/datasets/packaged_modules/iceberg}/__init__.py +0 -0
  128. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/iceberg/iceberg.py +0 -0
  129. {datasets-5.0.1/src/datasets/packaged_modules/niftifolder → datasets-5.2.0/src/datasets/packaged_modules/imagefolder}/__init__.py +0 -0
  130. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/imagefolder/imagefolder.py +0 -0
  131. {datasets-5.0.1/src/datasets/packaged_modules/pandas → datasets-5.2.0/src/datasets/packaged_modules/json}/__init__.py +0 -0
  132. {datasets-5.0.1/src/datasets/packaged_modules/parquet → datasets-5.2.0/src/datasets/packaged_modules/lance}/__init__.py +0 -0
  133. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/lance/lance.py +0 -0
  134. {datasets-5.0.1/src/datasets/packaged_modules/pdffolder → datasets-5.2.0/src/datasets/packaged_modules/meshfolder}/__init__.py +0 -0
  135. {datasets-5.0.1/src/datasets/packaged_modules/spark → datasets-5.2.0/src/datasets/packaged_modules/mmcif}/__init__.py +0 -0
  136. {datasets-5.0.1/src/datasets/packaged_modules/sql → datasets-5.2.0/src/datasets/packaged_modules/niftifolder}/__init__.py +0 -0
  137. {datasets-5.0.1/src/datasets/packaged_modules/text → datasets-5.2.0/src/datasets/packaged_modules/pandas}/__init__.py +0 -0
  138. {datasets-5.0.1/src/datasets/packaged_modules/tsfile → datasets-5.2.0/src/datasets/packaged_modules/parquet}/__init__.py +0 -0
  139. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/parquet/parquet.py +0 -0
  140. {datasets-5.0.1/src/datasets/packaged_modules/videofolder → datasets-5.2.0/src/datasets/packaged_modules/pdb}/__init__.py +0 -0
  141. {datasets-5.0.1/src/datasets/packaged_modules/webdataset → datasets-5.2.0/src/datasets/packaged_modules/pdffolder}/__init__.py +0 -0
  142. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/pdffolder/pdffolder.py +0 -0
  143. {datasets-5.0.1/src/datasets/packaged_modules/xml → datasets-5.2.0/src/datasets/packaged_modules/spark}/__init__.py +0 -0
  144. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/spark/spark.py +0 -0
  145. {datasets-5.0.1/src/datasets/utils/resources → datasets-5.2.0/src/datasets/packaged_modules/sql}/__init__.py +0 -0
  146. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/sql/sql.py +0 -0
  147. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/text/text.py +0 -0
  148. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/tsfile/tsfile.py +0 -0
  149. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/videofolder/videofolder.py +0 -0
  150. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/webdataset/_tenbin.py +0 -0
  151. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/packaged_modules/xml/xml.py +0 -0
  152. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/parallel/__init__.py +0 -0
  153. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/parallel/parallel.py +0 -0
  154. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/search.py +0 -0
  155. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/streaming.py +0 -0
  156. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/__init__.py +0 -0
  157. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/_dataset_viewer.py +0 -0
  158. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/_dill.py +0 -0
  159. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/_filelock.py +0 -0
  160. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/deprecation_utils.py +0 -0
  161. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/doc_utils.py +0 -0
  162. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/experimental.py +0 -0
  163. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/filelock.py +0 -0
  164. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/hub.py +0 -0
  165. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/info_utils.py +0 -0
  166. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/logging.py +0 -0
  167. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/metadata.py +0 -0
  168. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/patching.py +0 -0
  169. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/py_utils.py +0 -0
  170. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/resources/creators.json +0 -0
  171. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/resources/languages.json +0 -0
  172. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/resources/multilingualities.json +0 -0
  173. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/resources/readme_structure.yaml +0 -0
  174. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/resources/size_categories.json +0 -0
  175. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/sharding.py +0 -0
  176. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/stratify.py +0 -0
  177. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/tf_utils.py +0 -0
  178. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/tqdm.py +0 -0
  179. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/track.py +0 -0
  180. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/typing.py +0 -0
  181. {datasets-5.0.1 → datasets-5.2.0}/src/datasets/utils/version.py +0 -0
  182. {datasets-5.0.1 → datasets-5.2.0}/src/datasets.egg-info/dependency_links.txt +0 -0
  183. {datasets-5.0.1 → datasets-5.2.0}/src/datasets.egg-info/entry_points.txt +0 -0
  184. {datasets-5.0.1 → datasets-5.2.0}/src/datasets.egg-info/not-zip-safe +0 -0
  185. {datasets-5.0.1 → datasets-5.2.0}/src/datasets.egg-info/top_level.txt +0 -0
  186. {datasets-5.0.1 → datasets-5.2.0}/tests/test_arrow_reader.py +0 -0
  187. {datasets-5.0.1 → datasets-5.2.0}/tests/test_buckets.py +0 -0
  188. {datasets-5.0.1 → datasets-5.2.0}/tests/test_builder.py +0 -0
  189. {datasets-5.0.1 → datasets-5.2.0}/tests/test_data_files.py +0 -0
  190. {datasets-5.0.1 → datasets-5.2.0}/tests/test_dataset_dict.py +0 -0
  191. {datasets-5.0.1 → datasets-5.2.0}/tests/test_dataset_list.py +0 -0
  192. {datasets-5.0.1 → datasets-5.2.0}/tests/test_download_manager.py +0 -0
  193. {datasets-5.0.1 → datasets-5.2.0}/tests/test_exceptions.py +0 -0
  194. {datasets-5.0.1 → datasets-5.2.0}/tests/test_experimental.py +0 -0
  195. {datasets-5.0.1 → datasets-5.2.0}/tests/test_filelock.py +0 -0
  196. {datasets-5.0.1 → datasets-5.2.0}/tests/test_filesystem.py +0 -0
  197. {datasets-5.0.1 → datasets-5.2.0}/tests/test_fingerprint.py +0 -0
  198. {datasets-5.0.1 → datasets-5.2.0}/tests/test_fingerprint_tokenizer_stability.py +0 -0
  199. {datasets-5.0.1 → datasets-5.2.0}/tests/test_formatting.py +0 -0
  200. {datasets-5.0.1 → datasets-5.2.0}/tests/test_hub.py +0 -0
  201. {datasets-5.0.1 → datasets-5.2.0}/tests/test_info.py +0 -0
  202. {datasets-5.0.1 → datasets-5.2.0}/tests/test_info_utils.py +0 -0
  203. {datasets-5.0.1 → datasets-5.2.0}/tests/test_inspect.py +0 -0
  204. {datasets-5.0.1 → datasets-5.2.0}/tests/test_metadata_util.py +0 -0
  205. {datasets-5.0.1 → datasets-5.2.0}/tests/test_parallel.py +0 -0
  206. {datasets-5.0.1 → datasets-5.2.0}/tests/test_patching.py +0 -0
  207. {datasets-5.0.1 → datasets-5.2.0}/tests/test_py_utils.py +0 -0
  208. {datasets-5.0.1 → datasets-5.2.0}/tests/test_sharding_utils.py +0 -0
  209. {datasets-5.0.1 → datasets-5.2.0}/tests/test_splits.py +0 -0
  210. {datasets-5.0.1 → datasets-5.2.0}/tests/test_streaming_download_manager.py +0 -0
  211. {datasets-5.0.1 → datasets-5.2.0}/tests/test_tqdm.py +0 -0
  212. {datasets-5.0.1 → datasets-5.2.0}/tests/test_version.py +0 -0
{datasets-5.0.1/src/datasets.egg-info → datasets-5.2.0}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datasets
3
- Version: 5.0.1
3
+ Version: 5.2.0
4
4
  Summary: HuggingFace community-driven open-source library of datasets
5
5
  Home-page: https://github.com/huggingface/datasets
6
6
  Download-URL: https://github.com/huggingface/datasets/tags
@@ -27,16 +27,14 @@ License-File: LICENSE
27
27
  License-File: AUTHORS
28
28
  Requires-Dist: filelock
29
29
  Requires-Dist: numpy>=1.17
30
- Requires-Dist: pyarrow>=21.0.0
30
+ Requires-Dist: pyarrow>=24.0.0
31
31
  Requires-Dist: dill<0.4.2,>=0.3.0
32
32
  Requires-Dist: pandas
33
- Requires-Dist: requests>=2.32.2
34
- Requires-Dist: httpx<1.0.0
35
33
  Requires-Dist: tqdm>=4.66.3
36
34
  Requires-Dist: xxhash
37
35
  Requires-Dist: multiprocess<0.70.20
38
- Requires-Dist: fsspec[http]<=2026.6.0,>=2023.1.0
39
- Requires-Dist: huggingface-hub<2.0,>=0.25.0
36
+ Requires-Dist: fsspec<=2026.9.0,>=2023.1.0
37
+ Requires-Dist: huggingface-hub<3.0,>=1.31.0
40
38
  Requires-Dist: packaging
41
39
  Requires-Dist: pyyaml>=5.1
42
40
  Provides-Extra: audio
@@ -57,6 +55,7 @@ Requires-Dist: jax>=0.3.14; extra == "jax"
57
55
  Requires-Dist: jaxlib>=0.3.14; extra == "jax"
58
56
  Provides-Extra: streaming
59
57
  Provides-Extra: dev
58
+ Requires-Dist: biopython>=1.80; extra == "dev"
60
59
  Requires-Dist: numba>=0.56.4; python_version < "3.14" and extra == "dev"
61
60
  Requires-Dist: absl-py; extra == "dev"
62
61
  Requires-Dist: decorator; extra == "dev"
@@ -71,6 +70,7 @@ Requires-Dist: faiss-cpu>=1.8.0.post1; extra == "dev"
71
70
  Requires-Dist: h5py; extra == "dev"
72
71
  Requires-Dist: pylance; extra == "dev"
73
72
  Requires-Dist: pyiceberg[pyarrow,sql-sqlite]; extra == "dev"
73
+ Requires-Dist: vortex-data; (python_version >= "3.11" and sys_platform != "win32") and extra == "dev"
74
74
  Requires-Dist: jax>=0.3.14; sys_platform != "win32" and extra == "dev"
75
75
  Requires-Dist: jaxlib>=0.3.14; sys_platform != "win32" and extra == "dev"
76
76
  Requires-Dist: lz4; python_version < "3.14" and extra == "dev"
@@ -79,10 +79,11 @@ Requires-Dist: pyspark>=3.4; extra == "dev"
79
79
  Requires-Dist: py7zr; extra == "dev"
80
80
  Requires-Dist: rarfile>=4.0; extra == "dev"
81
81
  Requires-Dist: sqlalchemy; extra == "dev"
82
- Requires-Dist: protobuf<4.0.0; extra == "dev"
82
+ Requires-Dist: protobuf<4.0.0; python_version < "3.14" and extra == "dev"
83
83
  Requires-Dist: tensorflow>=2.6.0; (python_version < "3.10" and sys_platform != "win32") and extra == "dev"
84
84
  Requires-Dist: tensorflow>=2.16.0; (python_version >= "3.10" and sys_platform != "win32" and python_version < "3.14") and extra == "dev"
85
85
  Requires-Dist: tiktoken; extra == "dev"
86
+ Requires-Dist: tomli>=1.1.0; python_version < "3.11" and extra == "dev"
86
87
  Requires-Dist: torch>=2.8.0; extra == "dev"
87
88
  Requires-Dist: torchdata; extra == "dev"
88
89
  Requires-Dist: transformers>=4.42.0; extra == "dev"
@@ -98,6 +99,7 @@ Requires-Dist: transformers; extra == "dev"
98
99
  Requires-Dist: torch; extra == "dev"
99
100
  Requires-Dist: tensorflow>=2.6.0; extra == "dev"
100
101
  Provides-Extra: tests
102
+ Requires-Dist: biopython>=1.80; extra == "tests"
101
103
  Requires-Dist: numba>=0.56.4; python_version < "3.14" and extra == "tests"
102
104
  Requires-Dist: absl-py; extra == "tests"
103
105
  Requires-Dist: decorator; extra == "tests"
@@ -112,6 +114,7 @@ Requires-Dist: faiss-cpu>=1.8.0.post1; extra == "tests"
112
114
  Requires-Dist: h5py; extra == "tests"
113
115
  Requires-Dist: pylance; extra == "tests"
114
116
  Requires-Dist: pyiceberg[pyarrow,sql-sqlite]; extra == "tests"
117
+ Requires-Dist: vortex-data; (python_version >= "3.11" and sys_platform != "win32") and extra == "tests"
115
118
  Requires-Dist: jax>=0.3.14; sys_platform != "win32" and extra == "tests"
116
119
  Requires-Dist: jaxlib>=0.3.14; sys_platform != "win32" and extra == "tests"
117
120
  Requires-Dist: lz4; python_version < "3.14" and extra == "tests"
@@ -120,10 +123,11 @@ Requires-Dist: pyspark>=3.4; extra == "tests"
120
123
  Requires-Dist: py7zr; extra == "tests"
121
124
  Requires-Dist: rarfile>=4.0; extra == "tests"
122
125
  Requires-Dist: sqlalchemy; extra == "tests"
123
- Requires-Dist: protobuf<4.0.0; extra == "tests"
126
+ Requires-Dist: protobuf<4.0.0; python_version < "3.14" and extra == "tests"
124
127
  Requires-Dist: tensorflow>=2.6.0; (python_version < "3.10" and sys_platform != "win32") and extra == "tests"
125
128
  Requires-Dist: tensorflow>=2.16.0; (python_version >= "3.10" and sys_platform != "win32" and python_version < "3.14") and extra == "tests"
126
129
  Requires-Dist: tiktoken; extra == "tests"
130
+ Requires-Dist: tomli>=1.1.0; python_version < "3.11" and extra == "tests"
127
131
  Requires-Dist: torch>=2.8.0; extra == "tests"
128
132
  Requires-Dist: torchdata; extra == "tests"
129
133
  Requires-Dist: transformers>=4.42.0; extra == "tests"
@@ -135,6 +139,7 @@ Requires-Dist: nibabel>=5.3.1; extra == "tests"
135
139
  Requires-Dist: trimesh>=4.10.0; extra == "tests"
136
140
  Requires-Dist: teich==0.1.5; extra == "tests"
137
141
  Provides-Extra: tests-numpy2
142
+ Requires-Dist: biopython>=1.80; extra == "tests-numpy2"
138
143
  Requires-Dist: numba>=0.56.4; python_version < "3.14" and extra == "tests-numpy2"
139
144
  Requires-Dist: absl-py; extra == "tests-numpy2"
140
145
  Requires-Dist: decorator; extra == "tests-numpy2"
@@ -148,6 +153,7 @@ Requires-Dist: elasticsearch<8.0.0,>=7.17.12; extra == "tests-numpy2"
148
153
  Requires-Dist: h5py; extra == "tests-numpy2"
149
154
  Requires-Dist: pylance; extra == "tests-numpy2"
150
155
  Requires-Dist: pyiceberg[pyarrow,sql-sqlite]; extra == "tests-numpy2"
156
+ Requires-Dist: vortex-data; (python_version >= "3.11" and sys_platform != "win32") and extra == "tests-numpy2"
151
157
  Requires-Dist: jax>=0.3.14; sys_platform != "win32" and extra == "tests-numpy2"
152
158
  Requires-Dist: jaxlib>=0.3.14; sys_platform != "win32" and extra == "tests-numpy2"
153
159
  Requires-Dist: lz4; python_version < "3.14" and extra == "tests-numpy2"
@@ -156,8 +162,9 @@ Requires-Dist: pyspark>=3.4; extra == "tests-numpy2"
156
162
  Requires-Dist: py7zr; extra == "tests-numpy2"
157
163
  Requires-Dist: rarfile>=4.0; extra == "tests-numpy2"
158
164
  Requires-Dist: sqlalchemy; extra == "tests-numpy2"
159
- Requires-Dist: protobuf<4.0.0; extra == "tests-numpy2"
165
+ Requires-Dist: protobuf<4.0.0; python_version < "3.14" and extra == "tests-numpy2"
160
166
  Requires-Dist: tiktoken; extra == "tests-numpy2"
167
+ Requires-Dist: tomli>=1.1.0; python_version < "3.11" and extra == "tests-numpy2"
161
168
  Requires-Dist: torch>=2.8.0; extra == "tests-numpy2"
162
169
  Requires-Dist: torchdata; extra == "tests-numpy2"
163
170
  Requires-Dist: transformers>=4.42.0; extra == "tests-numpy2"
@@ -180,6 +187,8 @@ Requires-Dist: torch; extra == "docs"
180
187
  Requires-Dist: tensorflow>=2.6.0; extra == "docs"
181
188
  Provides-Extra: pdfs
182
189
  Requires-Dist: pdfplumber>=0.11.4; extra == "pdfs"
190
+ Provides-Extra: bio
191
+ Requires-Dist: biopython>=1.80; extra == "bio"
183
192
  Provides-Extra: nibabel
184
193
  Requires-Dist: nibabel>=5.3.2; extra == "nibabel"
185
194
  Requires-Dist: ipyniivue==2.4.2; extra == "nibabel"
@@ -407,7 +416,7 @@ For more details on using the library, check the [quick start guide](https://hug
407
416
  - [Processing audio data](https://huggingface.co/docs/datasets/audio_process)
408
417
  - [Processing image data](https://huggingface.co/docs/datasets/image_process)
409
418
  - [Processing text data](https://huggingface.co/docs/datasets/nlp_process)
410
- - [Processing PDF data](https://huggingface.co/docs/datasets/pdf_process)
419
+ - [Processing PDF data](https://huggingface.co/docs/datasets/document_dataset)
411
420
  - [Processing video data](https://huggingface.co/docs/datasets/video_process)
412
421
  - [Streaming a dataset](https://huggingface.co/docs/datasets/stream)
413
422
 
{datasets-5.0.1 → datasets-5.2.0}/README.md RENAMED
@@ -205,7 +205,7 @@ For more details on using the library, check the [quick start guide](https://hug
205
205
  - [Processing audio data](https://huggingface.co/docs/datasets/audio_process)
206
206
  - [Processing image data](https://huggingface.co/docs/datasets/image_process)
207
207
  - [Processing text data](https://huggingface.co/docs/datasets/nlp_process)
208
- - [Processing PDF data](https://huggingface.co/docs/datasets/pdf_process)
208
+ - [Processing PDF data](https://huggingface.co/docs/datasets/document_dataset)
209
209
  - [Processing video data](https://huggingface.co/docs/datasets/video_process)
210
210
  - [Streaming a dataset](https://huggingface.co/docs/datasets/stream)
211
211
 
{datasets-5.0.1 → datasets-5.2.0}/setup.py RENAMED
@@ -110,15 +110,12 @@ REQUIRED_PKGS = [
110
110
  # We use numpy>=1.17 to have np.random.Generator (Dataset shuffling)
111
111
  "numpy>=1.17",
112
112
  # Backend and serialization.
113
- # Minimum 21.0.0 to support `use_content_defined_chunking` in ParquetWriter
114
- "pyarrow>=21.0.0",
113
+ # Minimum 24.0.0 for view-type `nbytes` support used by Vortex
114
+ "pyarrow>=24.0.0",
115
115
  # For smart caching dataset processing
116
116
  "dill>=0.3.0,<0.4.2", # tmp pin until dill has official support for determinism see https://github.com/uqfoundation/dill/issues/19
117
117
  # For performance gains with apache arrow
118
118
  "pandas",
119
- # for downloading datasets over HTTPS
120
- "requests>=2.32.2",
121
- "httpx<1.0.0",
122
119
  # progress bars in downloads and data operations
123
120
  "tqdm>=4.66.3",
124
121
  # for fast hashing
@@ -127,9 +124,11 @@ REQUIRED_PKGS = [
127
124
  "multiprocess<0.70.20", # to align with dill<0.3.9 (see above)
128
125
  # to save datasets locally or on any filesystem
129
126
  # minimum 2023.1.0 to support protocol=kwargs in fsspec's `open`, `get_fs_token_paths`, etc.: see https://github.com/fsspec/filesystem_spec/pull/1143
130
- "fsspec[http]>=2023.1.0,<=2026.6.0",
127
+ "fsspec>=2023.1.0,<=2026.9.0",
131
128
  # To get datasets from the Datasets Hub on huggingface.co
132
- "huggingface-hub>=0.25.0,<2.0",
129
+ # (also provides the HTTP client: `from huggingface_hub.utils import httpx`)
130
+ # minimum 1.31.0 for the `huggingface_hub.utils.httpx` re-export
131
+ "huggingface-hub>=1.31.0,<3.0",
133
132
  # Utilities from PyPA to e.g., compare versions
134
133
  "packaging",
135
134
  # To parse YAML metadata from dataset cards
@@ -156,6 +155,8 @@ BENCHMARKS_REQUIRE = [
156
155
  ]
157
156
 
158
157
  TESTS_REQUIRE = [
158
+ # optional decoders exercised by the feature tests
159
+ "biopython>=1.80",
159
160
  # fix pip install issues for windows
160
161
  "numba>=0.56.4; python_version < '3.14'", # to get recent versions of llvmlite for windows ci, not available on 3.14
161
162
  # test dependencies
@@ -173,6 +174,7 @@ TESTS_REQUIRE = [
173
174
  "h5py",
174
175
  "pylance",
175
176
  "pyiceberg[sql-sqlite,pyarrow]",
177
+ "vortex-data; python_version >= '3.11' and sys_platform != 'win32'",
176
178
  "jax>=0.3.14; sys_platform != 'win32'",
177
179
  "jaxlib>=0.3.14; sys_platform != 'win32'",
178
180
  "lz4; python_version < '3.14'", # python 3.14 gives ImportError: cannot import name '_compression' from partially initialized module 'lz4.frame
@@ -181,10 +183,13 @@ TESTS_REQUIRE = [
181
183
  "py7zr",
182
184
  "rarfile>=4.0",
183
185
  "sqlalchemy",
184
- "protobuf<4.0.0", # 4.0.0 breaks compatibility with tensorflow<2.12
186
+ # Pinned for the tensorflow<2.12 builds; tensorflow has no python 3.14 wheels, where
187
+ # `substrait` (a vortex-data dependency) needs the protobuf 5 runtime.
188
+ "protobuf<4.0.0; python_version < '3.14'",
185
189
  "tensorflow>=2.6.0; python_version<'3.10' and sys_platform != 'win32'", # numpy-2 is not supported for Python < 3.10
186
190
  "tensorflow>=2.16.0; python_version>='3.10' and sys_platform != 'win32' and python_version < '3.14'", # Pins numpy < 2
187
191
  "tiktoken",
192
+ "tomli>=1.1.0; python_version < '3.11'", # tomllib is in the standard library from python 3.11
188
193
  "torch>=2.8.0",
189
194
  "torchdata",
190
195
  "transformers>=4.42.0", # Pins numpy < 2
@@ -216,6 +221,8 @@ DOCS_REQUIRE = [
216
221
 
217
222
  PDFS_REQUIRE = ["pdfplumber>=0.11.4"]
218
223
 
224
+ BIO_REQUIRE = ["biopython>=1.80"]
225
+
219
226
  NIBABEL_REQUIRE = ["nibabel>=5.3.2", "ipyniivue==2.4.2"]
220
227
 
221
228
  ICEBERG_REQUIRE = ["pyiceberg>=0.7.0"]
@@ -238,13 +245,14 @@ EXTRAS_REQUIRE = {
238
245
  "benchmarks": BENCHMARKS_REQUIRE,
239
246
  "docs": DOCS_REQUIRE,
240
247
  "pdfs": PDFS_REQUIRE,
248
+ "bio": BIO_REQUIRE,
241
249
  "nibabel": NIBABEL_REQUIRE,
242
250
  "iceberg": ICEBERG_REQUIRE,
243
251
  }
244
252
 
245
253
  setup(
246
254
  name="datasets",
247
- version="5.0.1", # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
255
+ version="5.2.0", # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
248
256
  description="HuggingFace community-driven open-source library of datasets",
249
257
  long_description=open("README.md", encoding="utf-8").read(),
250
258
  long_description_content_type="text/markdown",
{datasets-5.0.1 → datasets-5.2.0}/src/datasets/__init__.py RENAMED
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- __version__ = "5.0.1"
15
+ __version__ = "5.2.0"
16
16
 
17
17
  from .arrow_dataset import Column, Dataset
18
18
  from .arrow_reader import ReadInstruction
{datasets-5.0.1 → datasets-5.2.0}/src/datasets/arrow_dataset.py RENAMED
@@ -71,8 +71,9 @@ from huggingface_hub import (
71
71
  HfFileSystem,
72
72
  HfFileSystemResolvedPath,
73
73
  )
74
- from huggingface_hub.utils import EntryNotFoundError, HfHubHTTPError, RepositoryNotFoundError
75
- from packaging import version
74
+ from huggingface_hub.errors import BucketNotFoundError
75
+ from huggingface_hub.hf_file_system import HfFileSystemResolvedBucketPath, HfFileSystemResolvedRepositoryPath
76
+ from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError
76
77
  from tqdm.contrib.concurrent import thread_map
77
78
 
78
79
  from . import __version__, config
@@ -142,16 +143,6 @@ from .utils.tf_utils import dataset_to_tf, minimal_tf_collate_fn, multiprocess_d
142
143
  from .utils.typing import ListLike, PathLike
143
144
 
144
145
 
145
- if config.HF_HUB_VERSION >= version.parse("1.6.0"):
146
- from huggingface_hub.errors import BucketNotFoundError
147
- from huggingface_hub.hf_file_system import HfFileSystemResolvedBucketPath, HfFileSystemResolvedRepositoryPath
148
-
149
- else:
150
- BucketNotFoundError = None
151
- HfFileSystemResolvedBucketPath = None
152
- HfFileSystemResolvedRepositoryPath = HfFileSystemResolvedPath
153
-
154
-
155
146
  if TYPE_CHECKING:
156
147
  import sqlite3
157
148
 
@@ -864,7 +855,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
864
855
  table = InMemoryTable.from_buffer(buffer)
865
856
 
866
857
  if indices_buffer is not None:
867
- indices_table = InMemoryTable.from_buffer(buffer)
858
+ indices_table = InMemoryTable.from_buffer(indices_buffer)
868
859
  else:
869
860
  indices_table = None
870
861
 
@@ -1839,14 +1830,21 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
1839
1830
  if self.list_indexes():
1840
1831
  raise ValueError("please remove all the indexes using `dataset.drop_index` before saving a dataset")
1841
1832
 
1833
+ from .arrow_writer import get_writer_batch_size_from_data_size
1834
+
1835
+ dataset_nbytes = self._estimate_nbytes()
1842
1836
  if num_shards is None:
1843
- dataset_nbytes = self._estimate_nbytes()
1844
1837
  max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
1845
1838
  num_shards = int(dataset_nbytes / max_shard_size) + 1
1846
1839
  num_shards = max(num_shards, num_proc or 1)
1847
1840
  # if we have only a few large samples, we should only create as many shards as samples
1848
1841
  num_shards = min(len(self.data), num_shards)
1849
1842
 
1843
+ # use smaller batches for big samples (e.g. images) to stay below the 2GB limit of arrow binary arrays
1844
+ writer_batch_size = config.DEFAULT_MAX_BATCH_SIZE
1845
+ if dataset_nbytes > 0:
1846
+ writer_batch_size = min(writer_batch_size, get_writer_batch_size_from_data_size(len(self), dataset_nbytes))
1847
+
1850
1848
  fs: fsspec.AbstractFileSystem
1851
1849
  fs, _ = url_to_fs(dataset_path, **(storage_options or {}))
1852
1850
 
@@ -1899,6 +1897,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
1899
1897
  "shard": self.shard(num_shards=num_shards, index=shard_idx, contiguous=True),
1900
1898
  "fpath": posixpath.join(dataset_path, f"data-{shard_idx:05d}-of-{num_shards:05d}.arrow"),
1901
1899
  "storage_options": storage_options,
1900
+ "writer_batch_size": writer_batch_size,
1902
1901
  }
1903
1902
  for shard_idx in range(num_shards)
1904
1903
  )
@@ -1940,9 +1939,13 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
1940
1939
  json.dump(sorted_keys_dataset_info, dataset_info_file, indent=2)
1941
1940
 
1942
1941
  @staticmethod
1943
- def _save_to_disk_single(job_id: int, shard: "Dataset", fpath: str, storage_options: Optional[dict]):
1944
- batch_size = config.DEFAULT_MAX_BATCH_SIZE
1945
-
1942
+ def _save_to_disk_single(
1943
+ job_id: int,
1944
+ shard: "Dataset",
1945
+ fpath: str,
1946
+ storage_options: Optional[dict],
1947
+ writer_batch_size: int,
1948
+ ):
1946
1949
  num_examples_progress_update = 0
1947
1950
  writer = ArrowWriter(
1948
1951
  features=shard.features,
@@ -1952,7 +1955,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
1952
1955
  )
1953
1956
  try:
1954
1957
  _time = time.time()
1955
- for pa_table in shard.with_format("arrow").iter(batch_size):
1958
+ for pa_table in shard.with_format("arrow").iter(writer_batch_size):
1956
1959
  writer.write_table(pa_table)
1957
1960
  num_examples_progress_update += len(pa_table)
1958
1961
  if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
@@ -3853,7 +3856,14 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
3853
3856
  processed_inputs.pop(column)
3854
3857
  if check_same_num_examples:
3855
3858
  input_num_examples = len(pa_inputs)
3856
- processed_inputs_num_examples = len(processed_inputs[next(iter(processed_inputs.keys()))])
3859
+ if not isinstance(processed_inputs, Mapping):
3860
+ # pa.Table, pd.DataFrame or pl.DataFrame
3861
+ processed_inputs_num_examples = len(processed_inputs)
3862
+ elif processed_inputs:
3863
+ processed_inputs_num_examples = len(next(iter(processed_inputs.values())))
3864
+ else:
3865
+ # an empty dict doesn't update any column
3866
+ processed_inputs_num_examples = input_num_examples
3857
3867
  if input_num_examples != processed_inputs_num_examples:
3858
3868
  raise DatasetTransformationNotAllowedError(
3859
3869
  "Using `.map` in batched mode on a dataset with attached indexes is allowed only if it doesn't create or remove existing examples. You can first run `.drop_index() to remove your index and then re-add it."
@@ -4699,7 +4709,10 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
4699
4709
  'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}]
4700
4710
  ```
4701
4711
  """
4702
- return self.select(range(n))
4712
+ # Clamp `n` to the dataset length so taking more elements than available
4713
+ # returns the whole dataset instead of raising an IndexError, matching
4714
+ # the behavior of `IterableDataset.take`.
4715
+ return self.select(range(min(n, len(self))))
4703
4716
 
4704
4717
  @transmit_format
4705
4718
  @fingerprint_transform(inplace=False, ignore_kwargs=["load_from_cache_file", "indices_cache_file_name"])
@@ -4939,6 +4952,10 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
4939
4952
  seed = seed[pos] if pos < 624 else seed[0]
4940
4953
  _ = np.random.random() # do 1 step of rng
4941
4954
  generator = np.random.default_rng(seed)
4955
+ permutation = None
4956
+ else:
4957
+ # draw before the cache lookup so the caller's generator advances on cache hits too
4958
+ permutation = generator.permutation(len(self))
4942
4959
 
4943
4960
  # Check if we've already cached this computation (indexed by a hash)
4944
4961
  if self.cache_files:
@@ -4951,7 +4968,8 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
4951
4968
  fingerprint=new_fingerprint, indices_cache_file_name=indices_cache_file_name
4952
4969
  )
4953
4970
 
4954
- permutation = generator.permutation(len(self))
4971
+ if permutation is None:
4972
+ permutation = generator.permutation(len(self))
4955
4973
 
4956
4974
  return self.select(
4957
4975
  indices=permutation,
@@ -5726,7 +5744,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
5726
5744
  k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True)
5727
5745
  ]
5728
5746
 
5729
- if decodable_columns:
5747
+ if decodable_columns and len(self) > 0:
5730
5748
  # Approximate the space needed to store the bytes from the external files by analyzing the first 1000 examples
5731
5749
  extra_nbytes = 0
5732
5750
 
@@ -6014,12 +6032,14 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
6014
6032
  logger.warning(
6015
6033
  f"Setting num_proc from {num_jobs} back to 1 for the {split} split to disable multiprocessing as it only contains one shard."
6016
6034
  )
6035
+ num_proc = None
6017
6036
  num_jobs = 1
6018
6037
  elif num_shards < num_jobs:
6019
6038
  logger.warning(
6020
6039
  f"Setting num_proc from {num_jobs} to {num_shards} for the {split} split as it only contains {num_shards} shards."
6021
6040
  )
6022
6041
  num_proc = num_shards
6042
+ num_jobs = num_shards
6023
6043
  kwargs_iterable = [
6024
6044
  {
6025
6045
  "self": self.shard(num_shards=num_jobs, index=job_id, contiguous=True),
@@ -6125,7 +6145,7 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
6125
6145
  organization's default is private. This value is ignored if the repo already exists.
6126
6146
  token (`str`, *optional*):
6127
6147
  An optional authentication token for the Hugging Face Hub. If no token is passed, will default
6128
- to the token saved locally when logging in with `huggingface-cli login`. Will raise an error
6148
+ to the token saved locally when logging in with `hf auth login`. Will raise an error
6129
6149
  if no token is passed and the user is not logged-in.
6130
6150
  revision (`str`, *optional*):
6131
6151
  Branch to push the uploaded files to. Defaults to the `"main"` branch.
@@ -6207,8 +6227,6 @@ class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
6207
6227
 
6208
6228
  api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
6209
6229
  if repo_id.startswith("buckets/"):
6210
- if BucketNotFoundError is None:
6211
- raise ImportError("Pushing datasets to buckets requires huggingface_hub>=1.6.0")
6212
6230
  _, _namespace, _bucket_name, *_path_segments = repo_id.split("/")
6213
6231
  try:
6214
6232
  bucket_id = api.bucket_info(_namespace + "/" + _bucket_name).id
@@ -6754,10 +6772,7 @@ def _push_to_repo(
6754
6772
  dirfs = DirFileSystem(fs=hffs, path=hf_path)
6755
6773
 
6756
6774
  # Check the files to delete
6757
- try:
6758
- files_to_delete = dirfs.glob(f"{data_dir}/{split}-*", detail=True)
6759
- except EntryNotFoundError: # needed for huggingface_hub<=1.7.1
6760
- files_to_delete = {}
6775
+ files_to_delete = dirfs.glob(f"{data_dir}/{split}-*", detail=True)
6761
6776
 
6762
6777
  # Don't delete the new files
6763
6778
  deletions = [
@@ -6844,10 +6859,7 @@ def _push_to_bucket(
6844
6859
  dirfs = DirFileSystem(fs=hffs, path=hf_path)
6845
6860
 
6846
6861
  # Check the files to delete before uploading
6847
- try:
6848
- files_to_delete = dirfs.glob(f"{data_dir}/{split}-*", detail=True)
6849
- except EntryNotFoundError: # needed for huggingface_hub<=1.7.1
6850
- files_to_delete = {}
6862
+ files_to_delete = dirfs.glob(f"{data_dir}/{split}-*", detail=True)
6851
6863
 
6852
6864
  # Upload the Parquet files
6853
6865
  _, new_parquet_paths, features, split_info, uploaded_size = dset._push_parquet_shards_to_hub(
@@ -7127,7 +7139,9 @@ def _concatenate_map_style_datasets(
7127
7139
  if info is None:
7128
7140
  info = DatasetInfo.from_merge([dset.info for dset in dsets])
7129
7141
  fingerprint = update_fingerprint(
7130
- "".join(dset._fingerprint for dset in dsets), _concatenate_map_style_datasets, {"info": info, "split": split}
7142
+ "".join(dset._fingerprint for dset in dsets),
7143
+ _concatenate_map_style_datasets,
7144
+ {"info": info, "split": split, "axis": axis},
7131
7145
  )
7132
7146
 
7133
7147
  # Make final concatenated dataset
@@ -7236,16 +7250,98 @@ def _interleave_map_style_datasets(
7236
7250
  # We have to keep the indices to their respective dataset offsets and to flatten to effectively interleave the datasets
7237
7251
  indices = (indices + offsets).flatten().tolist()
7238
7252
 
7253
+ elif stopping_strategy in ("first_exhausted", "all_exhausted"):
7254
+ # Vectorized equivalent of the per-example Python loop below (kept for
7255
+ # `all_exhausted_without_replacement`). For `first_exhausted` /
7256
+ # `all_exhausted` the loop just appends, for each randomly drawn source,
7257
+ # that source's next index with wrap-around on exhaustion (rolling
7258
+ # window), and stops when the first (any) / every (all) source has been
7259
+ # drawn at least `length` times. That is fully vectorizable, giving a
7260
+ # large speedup for big outputs (e.g. ~90 min -> ~5 s for a ~93M-row
7261
+ # interleave) while producing bit-identical output for a fixed `seed`,
7262
+ # since we consume the RNG in the exact same 1000-sized `choice` blocks.
7263
+ lengths_arr = np.asarray(lengths, dtype=np.int64)
7264
+ n_datasets = len(lengths)
7265
+
7266
+ # A source with probability 0 is never drawn, so it can neither be
7267
+ # exhausted nor contribute rows. Exclude those from the stopping
7268
+ # condition rather than treating them as ordinary sources.
7269
+ drawable = np.asarray(probabilities, dtype=np.float64) > 0
7270
+
7271
+ # Empty sources: a length-0 dataset that can actually be drawn can never
7272
+ # be sampled to its length, so the stopping condition is ill-defined. The
7273
+ # previous implementation crashed here with a cryptic `IndexError: Index N
7274
+ # out of range` (it sampled the empty source and indexed into it). Raise a
7275
+ # clear error instead -- for both strategies, since such a source is
7276
+ # degenerate either way and silently dropping it would change results.
7277
+ # An empty source with probability 0 is simply never drawn, which worked
7278
+ # before, so it stays allowed.
7279
+ empty = [i for i in range(n_datasets) if lengths[i] == 0 and drawable[i]]
7280
+ if empty:
7281
+ raise ValueError(
7282
+ "interleave_datasets with probabilities requires every dataset "
7283
+ "that can be sampled (probability > 0) to be non-empty; "
7284
+ f"datasets at indices {empty} are empty."
7285
+ )
7286
+
7287
+ # Under `all_exhausted` every source must be exhausted, but a source with
7288
+ # probability 0 never will be -- the original loop spun forever in this
7289
+ # case. Fail with a clear error instead of looping unboundedly.
7290
+ if oversampling and not drawable.all():
7291
+ unreachable = [i for i in range(n_datasets) if not drawable[i]]
7292
+ raise ValueError(
7293
+ 'interleave_datasets with stopping_strategy="all_exhausted" cannot '
7294
+ "exhaust a dataset that is never sampled; datasets at indices "
7295
+ f"{unreachable} have probability 0."
7296
+ )
7297
+
7298
+ rng = np.random.default_rng(seed)
7299
+
7300
+ # Draw source indices in 1000-sized blocks (matching the original
7301
+ # iter_random_indices) until the stopping condition can be evaluated,
7302
+ # i.e. until enough sources have reached their length.
7303
+ blocks = []
7304
+ counts = np.zeros(n_datasets, dtype=np.int64)
7305
+ reached = np.zeros(n_datasets, dtype=bool)
7306
+ while not (reached[drawable].any() if not oversampling else reached[drawable].all()):
7307
+ block = rng.choice(n_datasets, size=1000, p=probabilities)
7308
+ blocks.append(block)
7309
+ counts += np.bincount(block, minlength=n_datasets)
7310
+ reached = counts >= lengths_arr
7311
+ draws = np.concatenate(blocks)
7312
+
7313
+ # A source becomes exhausted right AFTER its `length`-th draw, and
7314
+ # the original loop checks the stop condition BEFORE appending. So
7315
+ # the last draw we keep (inclusive) is at that length-th occurrence:
7316
+ # - first_exhausted (any): the earliest such position over sources
7317
+ # - all_exhausted (all): the latest such position over sources
7318
+ exhaust_pos = np.full(n_datasets, -1, dtype=np.int64)
7319
+ for s in range(n_datasets):
7320
+ if not drawable[s]:
7321
+ continue
7322
+ occ = np.flatnonzero(draws == s)
7323
+ if len(occ) >= lengths[s]:
7324
+ exhaust_pos[s] = occ[lengths[s] - 1]
7325
+ valid_pos = exhaust_pos[exhaust_pos >= 0]
7326
+ stop_at = valid_pos.min() if not oversampling else valid_pos.max()
7327
+ used = draws[: stop_at + 1]
7328
+
7329
+ # Map each source's k-th appearance to its k-th index with
7330
+ # wrap-around: concatenated row = (k % length) + offset.
7331
+ indices_arr = np.empty(len(used), dtype=np.int64)
7332
+ for s in range(n_datasets):
7333
+ if not drawable[s]:
7334
+ continue
7335
+ pos = np.flatnonzero(used == s)
7336
+ indices_arr[pos] = (np.arange(len(pos)) % lengths[s]) + offsets[s]
7337
+ indices = indices_arr.tolist()
7338
+
7239
7339
  else:
7240
- # boolean array indicating if at index i if the dataset_i has been fully exhausted
7340
+ # all_exhausted_without_replacement: each sample appears exactly once, so
7341
+ # an exhausted source is skipped rather than wrapped -- the output length
7342
+ # is not simply the draw count, so keep the explicit per-example loop.
7241
7343
  is_exhausted = np.full(len(lengths), False)
7242
7344
 
7243
- # if undersampling ("first_exhausted"), we stop as soon as one dataset is exhausted
7244
- # if oversampling ("all_exhausted"), we stop as soons as every dataset is exhausted, i.e as soon as every samples of every dataset has been visited at least once
7245
- bool_strategy_func = (
7246
- np.all if (oversampling or stopping_strategy == "all_exhausted_without_replacement") else np.any
7247
- )
7248
-
7249
7345
  def iter_random_indices():
7250
7346
  """Get an infinite iterator that randomly samples the index of the source to pick examples from."""
7251
7347
  rng = np.random.default_rng(seed)
@@ -7255,24 +7351,19 @@ def _interleave_map_style_datasets(
7255
7351
  current_index = [0] * len(datasets)
7256
7352
  indices = []
7257
7353
  for source_idx in iter_random_indices():
7258
- # If no oversampling, we stop as soon as a dataset has ran out of examples (np.any)
7259
- # Otherwise, we stop as soon as every dataset has ran out of examples (np.all)
7260
- if bool_strategy_func(is_exhausted):
7261
- # the stopping condition was reached, let's stop
7354
+ # Stop as soon as every dataset has run out of examples (np.all).
7355
+ if np.all(is_exhausted):
7262
7356
  break
7263
7357
 
7264
- # let's add the example at the current index of the `source_idx`-th dataset
7265
- # For without replacement sampling we additionally need to make sure the current source is not exhausted to not oversample.
7266
- if stopping_strategy != "all_exhausted_without_replacement" or not is_exhausted[source_idx]:
7358
+ # Add the example at the current index of the source_idx-th dataset,
7359
+ # unless that source is already exhausted (no oversampling).
7360
+ if not is_exhausted[source_idx]:
7267
7361
  indices.append(current_index[source_idx] + offsets[source_idx])
7268
7362
  current_index[source_idx] += 1
7269
7363
 
7270
- # we've ran out of examples for the current dataset, let's update our boolean array and bring the current_index back to 0
7364
+ # Mark exhaustion; do not reset the index (no replacement).
7271
7365
  if current_index[source_idx] >= lengths[source_idx]:
7272
7366
  is_exhausted[source_idx] = True
7273
- # We don't want to reset the iterator when stopping strategy is without replacement.
7274
- if stopping_strategy != "all_exhausted_without_replacement":
7275
- current_index[source_idx] = 0
7276
7367
 
7277
7368
  return concatenated_datasets.select(indices, **kwargs)
7278
7369