justdata 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. justdata/__init__.py +3 -0
  2. justdata/acoustic/__init__.py +381 -0
  3. justdata/acoustic/adapters.py +329 -0
  4. justdata/acoustic/augment/__init__.py +119 -0
  5. justdata/acoustic/augment/batch.py +813 -0
  6. justdata/acoustic/augment/configs.py +204 -0
  7. justdata/acoustic/augment/patchout.py +331 -0
  8. justdata/acoustic/augment/spectrogram.py +806 -0
  9. justdata/acoustic/augment/waveform.py +879 -0
  10. justdata/acoustic/batching.py +27 -0
  11. justdata/acoustic/channel.py +53 -0
  12. justdata/acoustic/compat/__init__.py +95 -0
  13. justdata/acoustic/compat/ast.py +626 -0
  14. justdata/acoustic/compat/ced.py +152 -0
  15. justdata/acoustic/compat/efficientat.py +243 -0
  16. justdata/acoustic/compat/panns.py +115 -0
  17. justdata/acoustic/compat/passt.py +191 -0
  18. justdata/acoustic/configs.py +519 -0
  19. justdata/acoustic/corruptions/__init__.py +33 -0
  20. justdata/acoustic/corruptions/codec.py +92 -0
  21. justdata/acoustic/corruptions/datasets.py +155 -0
  22. justdata/acoustic/corruptions/device.py +39 -0
  23. justdata/acoustic/corruptions/dynamics.py +55 -0
  24. justdata/acoustic/corruptions/filters.py +105 -0
  25. justdata/acoustic/corruptions/noise.py +173 -0
  26. justdata/acoustic/corruptions/registry.py +230 -0
  27. justdata/acoustic/corruptions/reverb.py +49 -0
  28. justdata/acoustic/corruptions/timefreq.py +127 -0
  29. justdata/acoustic/datasets.py +52 -0
  30. justdata/acoustic/dcase2025.py +814 -0
  31. justdata/acoustic/decoding.py +73 -0
  32. justdata/acoustic/eval_views.py +54 -0
  33. justdata/acoustic/evals.py +71 -0
  34. justdata/acoustic/filters.py +18 -0
  35. justdata/acoustic/frontends/__init__.py +37 -0
  36. justdata/acoustic/frontends/ast.py +116 -0
  37. justdata/acoustic/frontends/kaldi.py +34 -0
  38. justdata/acoustic/frontends/log.py +94 -0
  39. justdata/acoustic/frontends/mel.py +173 -0
  40. justdata/acoustic/frontends/mfcc.py +26 -0
  41. justdata/acoustic/frontends/pcen.py +26 -0
  42. justdata/acoustic/frontends/stft.py +174 -0
  43. justdata/acoustic/jax.py +23 -0
  44. justdata/acoustic/labels.py +169 -0
  45. justdata/acoustic/layouts.py +86 -0
  46. justdata/acoustic/metadata.py +152 -0
  47. justdata/acoustic/normalization.py +196 -0
  48. justdata/acoustic/pipelines.py +112 -0
  49. justdata/acoustic/postprocessing.py +328 -0
  50. justdata/acoustic/preprocessing.py +175 -0
  51. justdata/acoustic/presets.py +257 -0
  52. justdata/acoustic/registry.py +381 -0
  53. justdata/acoustic/resampling.py +218 -0
  54. justdata/acoustic/schema.py +51 -0
  55. justdata/acoustic/segment.py +369 -0
  56. justdata/acoustic/sources.py +503 -0
  57. justdata/acoustic/stages.py +59 -0
  58. justdata/acoustic/stats.py +50 -0
  59. justdata/acoustic/tasks.py +375 -0
  60. justdata/audio/__init__.py +20 -0
  61. justdata/core/__init__.py +74 -0
  62. justdata/core/adapters.py +46 -0
  63. justdata/core/augmentations.py +34 -0
  64. justdata/core/filters.py +94 -0
  65. justdata/core/finalization.py +173 -0
  66. justdata/core/label_mixing.py +129 -0
  67. justdata/core/loader.py +384 -0
  68. justdata/core/metadata.py +250 -0
  69. justdata/core/presets.py +155 -0
  70. justdata/core/registry.py +229 -0
  71. justdata/core/sources.py +87 -0
  72. justdata/core/stats.py +227 -0
  73. justdata/corruptions.py +52 -0
  74. justdata/py.typed +0 -0
  75. justdata/vision/__init__.py +17 -0
  76. justdata/vision/augmentations/__init__.py +31 -0
  77. justdata/vision/augmentations/auto.py +480 -0
  78. justdata/vision/augmentations/color.py +163 -0
  79. justdata/vision/augmentations/composed.py +156 -0
  80. justdata/vision/augmentations/geometric.py +302 -0
  81. justdata/vision/augmentations/mixing.py +232 -0
  82. justdata/vision/augmentations/registry.py +142 -0
  83. justdata/vision/corruptions/__init__.py +16 -0
  84. justdata/vision/corruptions/blur.py +65 -0
  85. justdata/vision/corruptions/digital.py +26 -0
  86. justdata/vision/corruptions/noise.py +20 -0
  87. justdata/vision/corruptions/registry.py +113 -0
  88. justdata/vision/corruptions/weather.py +100 -0
  89. justdata/vision/datasets.py +34 -0
  90. justdata/vision/encodings/__init__.py +1 -0
  91. justdata/vision/encodings/heatmaps.py +196 -0
  92. justdata/vision/minic.py +69 -0
  93. justdata/vision/pipelines.py +62 -0
  94. justdata/vision/presets.py +509 -0
  95. justdata/vision/sources.py +1080 -0
  96. justdata/vision/stages.py +271 -0
  97. justdata/vision/tasks/__init__.py +1 -0
  98. justdata/vision/tasks/classification.py +321 -0
  99. justdata/vision/tasks/depth.py +25 -0
  100. justdata/vision/tasks/segmentation.py +194 -0
  101. justdata/vision/transforms.py +113 -0
  102. justdata/vision/utils.py +741 -0
  103. justdata-1.0.0.dist-info/METADATA +860 -0
  104. justdata-1.0.0.dist-info/RECORD +106 -0
  105. justdata-1.0.0.dist-info/WHEEL +4 -0
  106. justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
@@ -0,0 +1,1080 @@
1
+ import hashlib
2
+ import json
3
+ import os
4
+ import re
5
+ import shutil
6
+ import stat
7
+ import tarfile
8
+ import tempfile
9
+ import time
10
+ import urllib.request
11
+ import zipfile
12
+ from collections.abc import Iterator
13
+ from contextlib import contextmanager
14
+ from dataclasses import dataclass
15
+ from pathlib import Path, PurePosixPath, PureWindowsPath
16
+ from typing import Any, Union
17
+ from urllib.parse import parse_qsl, quote, urlparse
18
+
19
+ import numpy as np
20
+ import tensorflow as tf
21
+ from loguru import logger
22
+
23
+ from justdata.core.sources import register_source_loader, source_cache_dir
24
+
25
+ _ZENODO_PREFIX = "zenodo:"
26
+ _ZENODO_QUERY_PARAMS = {"file"}
27
+ _ZENODO_ORIGIN = "zenodo.org"
28
+ _ZENODO_NETWORK_TIMEOUT_SECONDS = 60
29
+ _ZENODO_DOWNLOAD_TIMEOUT_SECONDS = 60 * 60
30
+ _ZENODO_DOWNLOAD_CHUNK_BYTES = 1024 * 1024
31
+ _ZENODO_MAX_ARCHIVE_BYTES = 20 * 1024**3
32
+ _ZENODO_MAX_EXTRACTED_BYTES = 100 * 1024**3
33
+ _ZENODO_MAX_ARCHIVE_MEMBERS = 100_000
34
+ _ZENODO_MAX_IMAGE_FILE_BYTES = 256 * 1024**2
35
+ _ZENODO_MAX_IMAGE_DIMENSION = 32_768
36
+ _ZENODO_MAX_IMAGE_PIXELS = 64_000_000
37
+ _IMAGE_EXTENSIONS = {
38
+ ".bmp",
39
+ ".gif",
40
+ ".jpeg",
41
+ ".jpg",
42
+ ".png",
43
+ ".webp",
44
+ }
45
+ _WILDS_PREFIX = "wilds:"
46
+ _SUPPORTED_WILDS_IMAGE_CLASSIFICATION = {
47
+ "camelyon17",
48
+ "fmow",
49
+ "iwildcam",
50
+ "rxrx1",
51
+ }
52
+ _UNSUPPORTED_WILDS_DATASETS = {
53
+ "amazon": "text",
54
+ "civilcomments": "text",
55
+ "globalwheat": "object detection",
56
+ "ogb-molpcba": "graph",
57
+ "poverty": "regression",
58
+ "py150": "code/text",
59
+ }
60
+ _WILDS_QUERY_PARAMS = {"download", "split_scheme", "unlabeled", "version"}
61
+
62
+
63
+ @dataclass(frozen=True)
64
+ class _ZenodoImageFolderSpec:
65
+ record_id: str
66
+ filename: str
67
+
68
+
69
+ @dataclass(frozen=True)
70
+ class _WILDSDatasetSpec:
71
+ name: str
72
+ split_scheme: str
73
+ version: str | None
74
+ download: bool
75
+ unlabeled: bool
76
+
77
+
78
+ def _strip_prefix(dataset_name: str, prefix: str) -> str:
79
+ if not dataset_name.startswith(prefix):
80
+ raise ValueError(
81
+ f"Expected dataset name to start with {prefix!r}; got {dataset_name!r}."
82
+ )
83
+ return dataset_name[len(prefix) :]
84
+
85
+
86
+ def _import_datasets():
87
+ try:
88
+ import datasets
89
+ except ImportError as e:
90
+ raise ImportError(
91
+ "The 'datasets' package is required for hf: vision source loading. "
92
+ "Install justdata with the vision extra."
93
+ ) from e
94
+ return datasets
95
+
96
+
97
+ def _parse_zenodo_spec(dataset_name: str) -> _ZenodoImageFolderSpec:
98
+ raw = _strip_prefix(dataset_name, _ZENODO_PREFIX)
99
+ record_id, sep, query = raw.partition("?")
100
+ if not re.fullmatch(r"[1-9][0-9]*", record_id):
101
+ raise ValueError("Zenodo record id must be a positive integer.")
102
+
103
+ values: dict[str, str] = {}
104
+ for key, value in parse_qsl(query if sep else "", keep_blank_values=True):
105
+ if key not in _ZENODO_QUERY_PARAMS:
106
+ allowed = ", ".join(sorted(_ZENODO_QUERY_PARAMS))
107
+ raise ValueError(
108
+ f"Unsupported Zenodo option {key!r}. Supported options: {allowed}."
109
+ )
110
+ if key in values:
111
+ raise ValueError(f"Duplicate Zenodo option {key!r}.")
112
+ values[key] = value
113
+
114
+ filename = values.get("file", "")
115
+ if not filename:
116
+ raise ValueError("Zenodo source requires ?file=<archive-name>.")
117
+ _validate_zenodo_filename(filename)
118
+
119
+ return _ZenodoImageFolderSpec(record_id=record_id, filename=filename)
120
+
121
+
122
+ def _validate_zenodo_filename(filename: str) -> None:
123
+ posix_path = PurePosixPath(filename)
124
+ windows_path = PureWindowsPath(filename)
125
+ if (
126
+ filename in {".", ".."}
127
+ or "\x00" in filename
128
+ or posix_path.is_absolute()
129
+ or windows_path.is_absolute()
130
+ or windows_path.drive
131
+ or posix_path.name != filename
132
+ or windows_path.name != filename
133
+ or "/" in filename
134
+ or "\\" in filename
135
+ ):
136
+ raise ValueError(
137
+ f"Zenodo archive filename must be a basename without path segments: "
138
+ f"{filename!r}."
139
+ )
140
+
141
+
142
+ def _zenodo_record_dir(
143
+ spec: _ZenodoImageFolderSpec,
144
+ data_dir: Union[None, str, os.PathLike],
145
+ ) -> Path:
146
+ return source_cache_dir(data_dir, "zenodo", spec.record_id)
147
+
148
+
149
+ def _fetch_zenodo_record(record_id: str) -> dict[str, Any]:
150
+ url = f"https://zenodo.org/api/records/{record_id}"
151
+ with urllib.request.urlopen(url, timeout=60) as response:
152
+ return json.load(response)
153
+
154
+
155
+ def _select_zenodo_file(record: dict[str, Any], filename: str) -> dict[str, Any]:
156
+ if not isinstance(record, dict):
157
+ raise ValueError("Zenodo record metadata must be an object.")
158
+ files = record.get("files")
159
+ if not isinstance(files, list):
160
+ raise ValueError("Zenodo record metadata does not contain a files list.")
161
+
162
+ for file_info in files:
163
+ if not isinstance(file_info, dict):
164
+ continue
165
+ if file_info.get("key") == filename or file_info.get("filename") == filename:
166
+ return file_info
167
+
168
+ available = ", ".join(
169
+ str(file_info.get("key") or file_info.get("filename")) for file_info in files
170
+ )
171
+ raise ValueError(
172
+ f"Zenodo record does not contain file {filename!r}. "
173
+ f"Available files: {available}."
174
+ )
175
+
176
+
177
+ def _zenodo_download_url(
178
+ spec: _ZenodoImageFolderSpec,
179
+ file_info: dict[str, Any],
180
+ ) -> str:
181
+ links = file_info.get("links")
182
+ if links is None:
183
+ links = {}
184
+ if not isinstance(links, dict):
185
+ raise ValueError("Zenodo file links metadata must be an object.")
186
+ for key in ("content", "download"):
187
+ value = links.get(key)
188
+ if value:
189
+ return _validate_zenodo_download_url(str(value))
190
+
191
+ value = links.get("self")
192
+ if value and str(value).rstrip("/").endswith("/content"):
193
+ return _validate_zenodo_download_url(str(value))
194
+
195
+ quoted = quote(spec.filename)
196
+ return _validate_zenodo_download_url(
197
+ f"https://zenodo.org/records/{spec.record_id}/files/{quoted}?download=1"
198
+ )
199
+
200
+
201
+ def _validate_zenodo_download_url(url: str) -> str:
202
+ parsed = urlparse(url)
203
+ try:
204
+ port = parsed.port
205
+ except ValueError as e:
206
+ raise ValueError(
207
+ "Zenodo download link must use the https://zenodo.org origin."
208
+ ) from e
209
+ if (
210
+ parsed.scheme != "https"
211
+ or parsed.hostname != _ZENODO_ORIGIN
212
+ or port not in (None, 443)
213
+ or parsed.username is not None
214
+ or parsed.password is not None
215
+ ):
216
+ raise ValueError("Zenodo download link must use the https://zenodo.org origin.")
217
+ return url
218
+
219
+
220
+ def _zenodo_declared_size(file_info: dict[str, Any]) -> int | None:
221
+ size = file_info.get("size")
222
+ if size is None:
223
+ return None
224
+ if isinstance(size, bool) or not isinstance(size, int) or size < 0:
225
+ raise ValueError("Zenodo file size must be a non-negative integer.")
226
+ if size > _ZENODO_MAX_ARCHIVE_BYTES:
227
+ raise ValueError(
228
+ f"Zenodo file size {size} exceeds the archive byte limit "
229
+ f"({_ZENODO_MAX_ARCHIVE_BYTES})."
230
+ )
231
+ return size
232
+
233
+
234
+ def _checksum_parts(checksum: str | None) -> tuple[str, str] | None:
235
+ if not checksum:
236
+ return None
237
+ if not isinstance(checksum, str):
238
+ raise ValueError("Zenodo checksum must be a string.")
239
+
240
+ algorithm, sep, expected = checksum.partition(":")
241
+ if not sep:
242
+ algorithm, expected = "md5", algorithm
243
+ algorithm = algorithm.lower()
244
+ if algorithm not in hashlib.algorithms_available:
245
+ raise ValueError(f"Unsupported Zenodo checksum algorithm {algorithm!r}.")
246
+ return algorithm, expected.lower()
247
+
248
+
249
+ def _file_matches_checksum(path: Path, checksum: str | None) -> bool:
250
+ if not path.exists():
251
+ return False
252
+
253
+ parts = _checksum_parts(checksum)
254
+ if parts is None:
255
+ return True
256
+
257
+ algorithm, expected = parts
258
+ digest = hashlib.new(algorithm)
259
+ with path.open("rb") as handle:
260
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
261
+ digest.update(chunk)
262
+ return digest.hexdigest().lower() == expected
263
+
264
+
265
+ def _download_file(
266
+ url: str,
267
+ target: Path,
268
+ *,
269
+ expected_size: int | None,
270
+ checksum: str | None,
271
+ ) -> None:
272
+ target.parent.mkdir(parents=True, exist_ok=True)
273
+ parts = _checksum_parts(checksum)
274
+ digest = hashlib.new(parts[0]) if parts is not None else None
275
+ started_at = time.monotonic()
276
+ bytes_downloaded = 0
277
+ tmp_path: Path | None = None
278
+ try:
279
+ with tempfile.NamedTemporaryFile(
280
+ mode="wb",
281
+ prefix=f".{target.name}.",
282
+ suffix=".tmp",
283
+ dir=target.parent,
284
+ delete=False,
285
+ ) as tmp:
286
+ tmp_path = Path(tmp.name)
287
+ with urllib.request.urlopen(
288
+ url, timeout=_ZENODO_NETWORK_TIMEOUT_SECONDS
289
+ ) as response:
290
+ while True:
291
+ if time.monotonic() - started_at > _ZENODO_DOWNLOAD_TIMEOUT_SECONDS:
292
+ raise TimeoutError(
293
+ "Zenodo download exceeded the transfer time limit."
294
+ )
295
+ chunk = response.read(_ZENODO_DOWNLOAD_CHUNK_BYTES)
296
+ if not chunk:
297
+ break
298
+ bytes_downloaded += len(chunk)
299
+ byte_limit = (
300
+ expected_size
301
+ if expected_size is not None
302
+ else _ZENODO_MAX_ARCHIVE_BYTES
303
+ )
304
+ if bytes_downloaded > byte_limit:
305
+ raise ValueError(
306
+ "Zenodo download exceeded its declared or configured "
307
+ "archive byte limit."
308
+ )
309
+ tmp.write(chunk)
310
+ if digest is not None:
311
+ digest.update(chunk)
312
+
313
+ if expected_size is not None and bytes_downloaded != expected_size:
314
+ raise ValueError(
315
+ f"Zenodo download size mismatch: expected {expected_size} "
316
+ f"bytes, received {bytes_downloaded}."
317
+ )
318
+ if parts is not None and digest.hexdigest().lower() != parts[1]:
319
+ raise ValueError(
320
+ f"Downloaded Zenodo file {target.name!r} failed checksum "
321
+ "validation."
322
+ )
323
+ tmp.flush()
324
+ os.fsync(tmp.fileno())
325
+ os.replace(tmp_path, target)
326
+ tmp_path = None
327
+ finally:
328
+ if tmp_path is not None:
329
+ tmp_path.unlink(missing_ok=True)
330
+
331
+
332
+ def _archive_stem(filename: str) -> str:
333
+ path = Path(filename)
334
+ suffixes = path.suffixes
335
+ if len(suffixes) >= 2 and suffixes[-2:] in (
336
+ [".tar", ".gz"],
337
+ [".tar", ".bz2"],
338
+ [".tar", ".xz"],
339
+ ):
340
+ return path.name[: -len("".join(suffixes[-2:]))]
341
+ return path.stem
342
+
343
+
344
+ def _assert_safe_extract_path(destination: Path, member_name: str) -> None:
345
+ path = PurePosixPath(member_name)
346
+ windows_path = PureWindowsPath(member_name)
347
+ if (
348
+ not member_name
349
+ or "\\" in member_name
350
+ or path.is_absolute()
351
+ or windows_path.is_absolute()
352
+ or windows_path.drive
353
+ or any(part in {"", ".", ".."} for part in path.parts)
354
+ ):
355
+ raise ValueError(f"Unsafe archive member path: {member_name!r}.")
356
+ target = (destination / member_name).resolve()
357
+ root = destination.resolve()
358
+ if target != root and root not in target.parents:
359
+ raise ValueError(f"Archive member escapes extraction directory: {member_name}")
360
+
361
+
362
+ def _check_archive_budget(
363
+ destination: Path,
364
+ members: list[tuple[str, int]],
365
+ ) -> None:
366
+ if len(members) > _ZENODO_MAX_ARCHIVE_MEMBERS:
367
+ raise ValueError(
368
+ f"Archive member count {len(members)} exceeds the limit "
369
+ f"({_ZENODO_MAX_ARCHIVE_MEMBERS})."
370
+ )
371
+ expanded_bytes = sum(size for _, size in members)
372
+ if expanded_bytes > _ZENODO_MAX_EXTRACTED_BYTES:
373
+ raise ValueError(
374
+ f"Archive expanded size {expanded_bytes} exceeds the byte limit "
375
+ f"({_ZENODO_MAX_EXTRACTED_BYTES})."
376
+ )
377
+ free_bytes = shutil.disk_usage(destination.parent).free
378
+ if expanded_bytes > free_bytes:
379
+ raise ValueError(
380
+ f"Archive requires {expanded_bytes} extracted bytes but the cache has "
381
+ f"only {free_bytes} bytes free."
382
+ )
383
+
384
+
385
+ def _normalized_archive_member(destination: Path, member_name: str) -> str:
386
+ _assert_safe_extract_path(destination, member_name)
387
+ return PurePosixPath(member_name).as_posix().rstrip("/")
388
+
389
+
390
+ def _preflight_zip(archive: zipfile.ZipFile, destination: Path) -> None:
391
+ members: list[tuple[str, int]] = []
392
+ destinations: set[str] = set()
393
+ for member in archive.infolist():
394
+ normalized = _normalized_archive_member(destination, member.filename)
395
+ if normalized in destinations:
396
+ raise ValueError(
397
+ f"Archive contains duplicate member path: {member.filename!r}."
398
+ )
399
+ destinations.add(normalized)
400
+
401
+ mode = member.external_attr >> 16
402
+ file_type = stat.S_IFMT(mode)
403
+ is_directory = member.is_dir()
404
+ if file_type == stat.S_IFLNK:
405
+ raise ValueError(f"Archive member uses a link: {member.filename!r}.")
406
+ if file_type not in (0, stat.S_IFREG, stat.S_IFDIR) or (
407
+ file_type == stat.S_IFDIR and not is_directory
408
+ ):
409
+ raise ValueError(
410
+ f"Archive member is not a regular file or directory: "
411
+ f"{member.filename!r}."
412
+ )
413
+ members.append((member.filename, 0 if is_directory else member.file_size))
414
+ _check_archive_budget(destination, members)
415
+
416
+
417
+ def _preflight_tar(archive: tarfile.TarFile, destination: Path) -> None:
418
+ members: list[tuple[str, int]] = []
419
+ destinations: set[str] = set()
420
+ for member in archive.getmembers():
421
+ normalized = _normalized_archive_member(destination, member.name)
422
+ if normalized in destinations:
423
+ raise ValueError(
424
+ f"Archive contains duplicate member path: {member.name!r}."
425
+ )
426
+ destinations.add(normalized)
427
+ if not (member.isfile() or member.isdir()):
428
+ category = "link" if member.issym() or member.islnk() else "special file"
429
+ raise ValueError(f"Archive member uses a {category}: {member.name!r}.")
430
+ members.append((member.name, member.size if member.isfile() else 0))
431
+ _check_archive_budget(destination, members)
432
+
433
+
434
+ def _extract_archive(archive_path: Path, extract_dir: Path) -> None:
435
+ extract_dir.parent.mkdir(parents=True, exist_ok=True)
436
+ tmp_dir = Path(
437
+ tempfile.mkdtemp(prefix=f".{extract_dir.name}.", dir=extract_dir.parent)
438
+ )
439
+
440
+ try:
441
+ if zipfile.is_zipfile(archive_path):
442
+ with zipfile.ZipFile(archive_path) as archive:
443
+ _preflight_zip(archive, tmp_dir)
444
+ archive.extractall(tmp_dir)
445
+ elif tarfile.is_tarfile(archive_path):
446
+ with tarfile.open(archive_path) as archive:
447
+ _preflight_tar(archive, tmp_dir)
448
+ if hasattr(tarfile, "data_filter"):
449
+ archive.extractall(tmp_dir, filter="data")
450
+ else:
451
+ archive.extractall(tmp_dir)
452
+ else:
453
+ raise ValueError(
454
+ f"Zenodo file {archive_path.name!r} is not a supported ZIP/TAR archive."
455
+ )
456
+
457
+ if extract_dir.exists():
458
+ shutil.rmtree(extract_dir)
459
+ os.replace(tmp_dir, extract_dir)
460
+ except Exception:
461
+ if tmp_dir.exists():
462
+ shutil.rmtree(tmp_dir)
463
+ raise
464
+
465
+
466
+ def _prepare_zenodo_archive(
467
+ spec: _ZenodoImageFolderSpec,
468
+ data_dir: Union[None, str, os.PathLike],
469
+ ) -> Path:
470
+ record = _fetch_zenodo_record(spec.record_id)
471
+ file_info = _select_zenodo_file(record, spec.filename)
472
+ checksum = file_info.get("checksum")
473
+ expected_size = _zenodo_declared_size(file_info)
474
+ download_url = _zenodo_download_url(spec, file_info)
475
+
476
+ record_dir = _zenodo_record_dir(spec, data_dir)
477
+ archive_path = record_dir / "files" / spec.filename
478
+ extract_dir = record_dir / "extracted" / _archive_stem(spec.filename)
479
+ lock_path = record_dir / "locks" / f"{spec.filename}.lock"
480
+ lock_path.parent.mkdir(parents=True, exist_ok=True)
481
+
482
+ try:
483
+ from filelock import FileLock
484
+ except ImportError as e:
485
+ raise ImportError(
486
+ "Zenodo source loading requires the 'filelock' package. Install "
487
+ "justdata with the vision extra."
488
+ ) from e
489
+
490
+ with FileLock(lock_path):
491
+ if not _file_matches_checksum(archive_path, checksum):
492
+ archive_path.unlink(missing_ok=True)
493
+ logger.info(
494
+ f"Downloading Zenodo record {spec.record_id} file {spec.filename}."
495
+ )
496
+ _download_file(
497
+ download_url,
498
+ archive_path,
499
+ expected_size=expected_size,
500
+ checksum=checksum,
501
+ )
502
+ if extract_dir.exists():
503
+ shutil.rmtree(extract_dir)
504
+
505
+ if not extract_dir.exists():
506
+ logger.info(f"Extracting Zenodo archive {archive_path}.")
507
+ _extract_archive(archive_path, extract_dir)
508
+
509
+ return extract_dir
510
+
511
+
512
+ def _resolve_imagefolder_root(extract_dir: Path, splits: list[str]) -> Path:
513
+ if all((extract_dir / split).is_dir() for split in splits):
514
+ return extract_dir
515
+
516
+ candidates = [
517
+ child
518
+ for child in extract_dir.iterdir()
519
+ if child.is_dir() and all((child / split).is_dir() for split in splits)
520
+ ]
521
+ if len(candidates) == 1:
522
+ return candidates[0]
523
+ if len(candidates) > 1:
524
+ names = ", ".join(sorted(candidate.name for candidate in candidates))
525
+ raise ValueError(
526
+ f"Multiple ImageFolder roots contain requested splits: {names}."
527
+ )
528
+
529
+ requested = ", ".join(splits)
530
+ raise ValueError(
531
+ f"Zenodo archive does not contain requested ImageFolder splits: {requested}."
532
+ )
533
+
534
+
535
+ def _is_image_file(path: Path) -> bool:
536
+ return path.is_file() and path.suffix.lower() in _IMAGE_EXTENSIONS
537
+
538
+
539
+ def _validate_image_file(path: Path) -> None:
540
+ size = path.stat().st_size
541
+ if size > _ZENODO_MAX_IMAGE_FILE_BYTES:
542
+ raise ValueError(
543
+ f"Zenodo image {path} exceeds the encoded-file limit of "
544
+ f"{_ZENODO_MAX_IMAGE_FILE_BYTES} bytes."
545
+ )
546
+
547
+ try:
548
+ from PIL import Image, UnidentifiedImageError
549
+ except ImportError as e:
550
+ raise ImportError(
551
+ "Pillow is required for bounded Zenodo image validation. "
552
+ "Install justdata with the vision extra."
553
+ ) from e
554
+
555
+ try:
556
+ with Image.open(path) as image:
557
+ width, height = image.size
558
+ except (Image.DecompressionBombError, UnidentifiedImageError, OSError) as e:
559
+ raise ValueError(f"Zenodo image {path} has an invalid image header.") from e
560
+
561
+ if width <= 0 or height <= 0:
562
+ raise ValueError(
563
+ f"Zenodo image {path} has invalid dimensions {width}x{height}."
564
+ )
565
+ if width > _ZENODO_MAX_IMAGE_DIMENSION or height > _ZENODO_MAX_IMAGE_DIMENSION:
566
+ raise ValueError(
567
+ f"Zenodo image {path} dimensions {width}x{height} exceed the "
568
+ f"per-dimension limit of {_ZENODO_MAX_IMAGE_DIMENSION}."
569
+ )
570
+ pixels = width * height
571
+ if pixels > _ZENODO_MAX_IMAGE_PIXELS:
572
+ raise ValueError(
573
+ f"Zenodo image {path} has {pixels} decoded pixels, exceeding the "
574
+ f"limit of {_ZENODO_MAX_IMAGE_PIXELS}."
575
+ )
576
+
577
+
578
+ def _split_class_dirs(split_dir: Path) -> list[Path]:
579
+ return sorted(
580
+ child
581
+ for child in split_dir.iterdir()
582
+ if child.is_dir() and not child.name.startswith(".")
583
+ )
584
+
585
+
586
+ def _imagefolder_split_dirs(root: Path) -> list[Path]:
587
+ return sorted(
588
+ child
589
+ for child in root.iterdir()
590
+ if child.is_dir()
591
+ and not child.name.startswith(".")
592
+ and _split_class_dirs(child)
593
+ )
594
+
595
+
596
+ def _imagefolder_class_to_label(root: Path) -> dict[str, np.int64]:
597
+ class_names = sorted(
598
+ {
599
+ class_dir.name
600
+ for split_dir in _imagefolder_split_dirs(root)
601
+ for class_dir in _split_class_dirs(split_dir)
602
+ }
603
+ )
604
+ if not class_names:
605
+ raise ValueError("Zenodo ImageFolder archive does not contain class folders.")
606
+ return {name: np.int64(index) for index, name in enumerate(class_names)}
607
+
608
+
609
+ def _imagefolder_records(
610
+ root: Path,
611
+ splits: list[str],
612
+ *,
613
+ dataset_name: str,
614
+ spec: _ZenodoImageFolderSpec,
615
+ class_to_label: dict[str, np.int64],
616
+ ) -> list[dict[str, Any]]:
617
+ split_dirs = {split: root / split for split in splits}
618
+ records: list[dict[str, Any]] = []
619
+ for split in splits:
620
+ split_records = []
621
+ split_dir = split_dirs[split]
622
+ class_dirs = _split_class_dirs(split_dir)
623
+ unknown_classes = sorted(
624
+ class_dir.name
625
+ for class_dir in class_dirs
626
+ if class_dir.name not in class_to_label
627
+ )
628
+ if unknown_classes:
629
+ names = ", ".join(unknown_classes)
630
+ raise ValueError(
631
+ f"Zenodo ImageFolder split {split!r} contains classes absent from "
632
+ f"the archive-wide vocabulary: {names}."
633
+ )
634
+ for class_dir in class_dirs:
635
+ label = class_to_label[class_dir.name]
636
+ for path in sorted(class_dir.rglob("*")):
637
+ if not _is_image_file(path):
638
+ continue
639
+ _validate_image_file(path)
640
+ relative_path = path.relative_to(root).as_posix()
641
+ split_records.append(
642
+ {
643
+ "_path": os.fspath(path),
644
+ "label": label,
645
+ "metadata": {
646
+ "dataset": dataset_name,
647
+ "record_id": spec.record_id,
648
+ "archive": spec.filename,
649
+ "split": split,
650
+ "class_name": class_dir.name,
651
+ "class_index": label,
652
+ "filename": path.name,
653
+ "path": os.fspath(path),
654
+ "example_id": relative_path,
655
+ },
656
+ }
657
+ )
658
+ if not split_records:
659
+ raise ValueError(f"Zenodo ImageFolder split {split!r} has no images.")
660
+ records.extend(split_records)
661
+
662
+ return records
663
+
664
+
665
+ def _records_to_vision_dataset(records: list[dict[str, Any]]) -> tf.data.Dataset:
666
+ def gen():
667
+ yield from records
668
+
669
+ ds = tf.data.Dataset.from_generator(
670
+ gen,
671
+ output_signature={
672
+ "_path": tf.TensorSpec(shape=(), dtype=tf.string),
673
+ "label": tf.TensorSpec(shape=(), dtype=tf.int64),
674
+ "metadata": {
675
+ "dataset": tf.TensorSpec(shape=(), dtype=tf.string),
676
+ "record_id": tf.TensorSpec(shape=(), dtype=tf.string),
677
+ "archive": tf.TensorSpec(shape=(), dtype=tf.string),
678
+ "split": tf.TensorSpec(shape=(), dtype=tf.string),
679
+ "class_name": tf.TensorSpec(shape=(), dtype=tf.string),
680
+ "class_index": tf.TensorSpec(shape=(), dtype=tf.int64),
681
+ "filename": tf.TensorSpec(shape=(), dtype=tf.string),
682
+ "path": tf.TensorSpec(shape=(), dtype=tf.string),
683
+ "example_id": tf.TensorSpec(shape=(), dtype=tf.string),
684
+ },
685
+ },
686
+ )
687
+
688
+ def decode_image(sample):
689
+ image = tf.io.decode_image(
690
+ tf.io.read_file(sample["_path"]),
691
+ channels=3,
692
+ expand_animations=False,
693
+ )
694
+ image.set_shape([None, None, 3])
695
+ return {
696
+ "image": image,
697
+ "label": sample["label"],
698
+ "metadata": sample["metadata"],
699
+ }
700
+
701
+ ds = ds.map(decode_image, num_parallel_calls=tf.data.AUTOTUNE)
702
+ try:
703
+ ds = ds.apply(tf.data.experimental.assert_cardinality(len(records)))
704
+ except Exception as e:
705
+ logger.warning(f"Failed to assert cardinality for Zenodo ImageFolder: {e}")
706
+ return ds
707
+
708
+
709
+ @register_source_loader(_ZENODO_PREFIX)
710
+ def load_zenodo_imagefolder_splits(
711
+ dataset_name: str,
712
+ splits: list[str],
713
+ data_dir: Union[None, str, os.PathLike] = None,
714
+ ) -> list[tf.data.Dataset]:
715
+ spec = _parse_zenodo_spec(dataset_name)
716
+ extract_dir = _prepare_zenodo_archive(spec, data_dir)
717
+ root = _resolve_imagefolder_root(extract_dir, splits)
718
+ class_to_label = _imagefolder_class_to_label(root)
719
+ return [
720
+ _records_to_vision_dataset(
721
+ _imagefolder_records(
722
+ root,
723
+ [split],
724
+ dataset_name=dataset_name,
725
+ spec=spec,
726
+ class_to_label=class_to_label,
727
+ )
728
+ )
729
+ for split in splits
730
+ ]
731
+
732
+
733
+ @register_source_loader("hf:")
734
+ def load_huggingface_vision_splits(
735
+ dataset_name: str,
736
+ splits: list[str],
737
+ data_dir: Union[None, str, os.PathLike] = None,
738
+ *,
739
+ include_metadata: bool = False,
740
+ ) -> list[tf.data.Dataset]:
741
+ datasets = _import_datasets()
742
+ hf_name = dataset_name[3:]
743
+ loaded_splits = []
744
+
745
+ for split in splits:
746
+ ds_hf = datasets.load_dataset(
747
+ hf_name,
748
+ split=split,
749
+ cache_dir=os.fspath(source_cache_dir(data_dir, "hf", "vision")),
750
+ )
751
+
752
+ if "image" not in ds_hf.column_names or "label" not in ds_hf.column_names:
753
+ raise ValueError(
754
+ f"Hugging Face vision dataset '{hf_name}' must contain 'image' and "
755
+ f"'label' columns. Found columns: {ds_hf.column_names}."
756
+ )
757
+
758
+ def gen(_ds=ds_hf, _split=split):
759
+ example_index = 0
760
+ for batch in _ds.iter(batch_size=1024):
761
+ for img, lbl in zip(batch["image"], batch["label"]):
762
+ item = {
763
+ "image": np.array(img),
764
+ "label": lbl,
765
+ }
766
+ if include_metadata:
767
+ filename = getattr(img, "filename", "") or ""
768
+ item["metadata"] = {
769
+ "dataset": hf_name,
770
+ "split": _split,
771
+ "example_id": str(example_index),
772
+ "filename": os.path.basename(filename) if filename else "",
773
+ }
774
+ example_index += 1
775
+ yield item
776
+
777
+ output_signature = {
778
+ "image": tf.TensorSpec(shape=None, dtype=tf.uint8),
779
+ "label": tf.TensorSpec(shape=(), dtype=tf.int64),
780
+ }
781
+ if include_metadata:
782
+ output_signature["metadata"] = {
783
+ "dataset": tf.TensorSpec(shape=(), dtype=tf.string),
784
+ "split": tf.TensorSpec(shape=(), dtype=tf.string),
785
+ "example_id": tf.TensorSpec(shape=(), dtype=tf.string),
786
+ "filename": tf.TensorSpec(shape=(), dtype=tf.string),
787
+ }
788
+
789
+ ds_tf = tf.data.Dataset.from_generator(
790
+ gen,
791
+ output_signature=output_signature,
792
+ )
793
+
794
+ try:
795
+ ds_tf = ds_tf.apply(tf.data.experimental.assert_cardinality(len(ds_hf)))
796
+ except Exception as e:
797
+ logger.warning(
798
+ f"Failed to assert cardinality for HF dataset {hf_name}: {e}"
799
+ )
800
+
801
+ loaded_splits.append(ds_tf)
802
+
803
+ return loaded_splits
804
+
805
+
806
+ def _parse_bool(value: str, *, key: str) -> bool:
807
+ normalized = value.strip().lower()
808
+ if normalized in {"1", "true", "yes", "y"}:
809
+ return True
810
+ if normalized in {"0", "false", "no", "n"}:
811
+ return False
812
+ raise ValueError(f"{key} must be a boolean value; got {value!r}.")
813
+
814
+
815
+ def _parse_wilds_spec(dataset_name: str) -> _WILDSDatasetSpec:
816
+ if not dataset_name.startswith(_WILDS_PREFIX):
817
+ raise ValueError(
818
+ f"Expected WILDS dataset name to start with {_WILDS_PREFIX!r}; "
819
+ f"got {dataset_name!r}."
820
+ )
821
+
822
+ raw = dataset_name[len(_WILDS_PREFIX) :]
823
+ name, sep, query = raw.partition("?")
824
+ if not name:
825
+ raise ValueError("WILDS dataset name cannot be empty.")
826
+
827
+ values: dict[str, str] = {}
828
+ for key, value in parse_qsl(query if sep else "", keep_blank_values=True):
829
+ if key not in _WILDS_QUERY_PARAMS:
830
+ allowed = ", ".join(sorted(_WILDS_QUERY_PARAMS))
831
+ raise ValueError(
832
+ f"Unsupported WILDS option {key!r}. Supported options: {allowed}."
833
+ )
834
+ if key in values:
835
+ raise ValueError(f"Duplicate WILDS option {key!r}.")
836
+ values[key] = value
837
+
838
+ if name not in _SUPPORTED_WILDS_IMAGE_CLASSIFICATION:
839
+ reason = _UNSUPPORTED_WILDS_DATASETS.get(name)
840
+ if reason is not None:
841
+ raise ValueError(
842
+ f"WILDS dataset {name!r} is a {reason} dataset and is not "
843
+ "supported by justdata's vision classification loader."
844
+ )
845
+ supported = ", ".join(sorted(_SUPPORTED_WILDS_IMAGE_CLASSIFICATION))
846
+ raise ValueError(
847
+ f"Unsupported WILDS dataset {name!r}. Supported image classification "
848
+ f"datasets: {supported}."
849
+ )
850
+
851
+ return _WILDSDatasetSpec(
852
+ name=name,
853
+ split_scheme=values.get("split_scheme", "official"),
854
+ version=values.get("version") or None,
855
+ download=_parse_bool(values.get("download", "false"), key="download"),
856
+ unlabeled=_parse_bool(values.get("unlabeled", "false"), key="unlabeled"),
857
+ )
858
+
859
+
860
+ def _is_unlabeled_wilds_split(split: str) -> bool:
861
+ return split.endswith("_unlabeled") or split == "extra_unlabeled"
862
+
863
+
864
+ def _validate_wilds_splits(splits: list[str], spec: _WILDSDatasetSpec) -> bool:
865
+ has_unlabeled = any(_is_unlabeled_wilds_split(split) for split in splits)
866
+ has_labeled = any(not _is_unlabeled_wilds_split(split) for split in splits)
867
+ if has_labeled and has_unlabeled:
868
+ raise ValueError(
869
+ "WILDS labeled and unlabeled splits must be loaded in separate "
870
+ "load_ds calls."
871
+ )
872
+ if spec.unlabeled and has_labeled:
873
+ raise ValueError(
874
+ "?unlabeled=true can only be used with WILDS unlabeled splits."
875
+ )
876
+ return has_unlabeled
877
+
878
+
879
+ def _import_wilds():
880
+ try:
881
+ import wilds
882
+ except ImportError as e:
883
+ raise ImportError(
884
+ "The 'wilds' package is required for wilds: source loading. "
885
+ "Install justdata with the wilds extra."
886
+ ) from e
887
+ return wilds
888
+
889
+
890
+ def _as_numpy(value: Any) -> np.ndarray:
891
+ if hasattr(value, "detach"):
892
+ value = value.detach().cpu()
893
+ if hasattr(value, "numpy"):
894
+ value = value.numpy()
895
+ return np.asarray(value)
896
+
897
+
898
+ def _image_to_uint8_array(image: Any) -> np.ndarray:
899
+ if hasattr(image, "convert"):
900
+ image = image.convert("RGB")
901
+ array = _as_numpy(image)
902
+ if array.ndim == 3 and array.shape[0] in (1, 3) and array.shape[-1] > 4:
903
+ array = np.transpose(array, (1, 2, 0))
904
+ if array.dtype == np.uint8:
905
+ return array
906
+ if np.issubdtype(array.dtype, np.floating) and array.size:
907
+ max_value = float(np.nanmax(array))
908
+ if max_value <= 1.0:
909
+ array = array * 255.0
910
+ return np.clip(array, 0, 255).astype(np.uint8)
911
+
912
+
913
+ def _label_to_int64(label: Any) -> np.int64:
914
+ label_array = _as_numpy(label)
915
+ if label_array.size != 1:
916
+ raise ValueError("WILDS vision classification labels must be scalar values.")
917
+ return np.int64(label_array.reshape(()))
918
+
919
+
920
+ def _metadata_to_int64_dict(
921
+ metadata: Any,
922
+ metadata_fields: tuple[str, ...] | list[str],
923
+ ) -> dict[str, np.int64]:
924
+ if isinstance(metadata, dict):
925
+ return {str(key): _label_to_int64(value) for key, value in metadata.items()}
926
+
927
+ metadata_array = _as_numpy(metadata).reshape(-1)
928
+ result = {}
929
+ for i, field in enumerate(metadata_fields):
930
+ if i >= metadata_array.shape[0]:
931
+ break
932
+ result[str(field)] = np.int64(metadata_array[i])
933
+ return result
934
+
935
+
936
+ def _is_fmow_timestamp_arg(value: Any) -> bool:
937
+ return getattr(value, "name", None) == "timestamp"
938
+
939
+
940
+ @contextmanager
941
+ def _wilds_fmow_datetime_compat(enabled: bool) -> Iterator[None]:
942
+ if not enabled:
943
+ yield
944
+ return
945
+
946
+ try:
947
+ import pandas as pd
948
+ except ImportError:
949
+ yield
950
+ return
951
+
952
+ original_to_datetime = pd.to_datetime
953
+
954
+ def to_datetime_compat(arg, *args, **kwargs):
955
+ if kwargs.get("format") is not None or not _is_fmow_timestamp_arg(arg):
956
+ return original_to_datetime(arg, *args, **kwargs)
957
+
958
+ try:
959
+ return original_to_datetime(arg, *args, **kwargs)
960
+ except ValueError as e:
961
+ if "match format" not in str(e):
962
+ raise
963
+ compat_kwargs = dict(kwargs)
964
+ compat_kwargs["format"] = "ISO8601"
965
+ return original_to_datetime(arg, *args, **compat_kwargs)
966
+
967
+ pd.to_datetime = to_datetime_compat
968
+ try:
969
+ yield
970
+ finally:
971
+ pd.to_datetime = original_to_datetime
972
+
973
+
974
+ def _split_to_dataset(
975
+ dataset,
976
+ split: str,
977
+ *,
978
+ spec: _WILDSDatasetSpec,
979
+ include_label: bool,
980
+ ) -> tf.data.Dataset:
981
+ subset = dataset.get_subset(split, transform=None)
982
+ metadata_fields = tuple(getattr(subset, "metadata_fields", ()))
983
+ version = str(getattr(dataset, "version", spec.version or ""))
984
+ split_scheme = str(getattr(dataset, "split_scheme", spec.split_scheme))
985
+ indices = getattr(subset, "indices", None)
986
+
987
+ def gen():
988
+ for subset_index, item in enumerate(subset):
989
+ if include_label:
990
+ image, label, metadata = item
991
+ else:
992
+ if len(item) == 2:
993
+ image, metadata = item
994
+ elif len(item) == 3:
995
+ image, _label, metadata = item
996
+ else:
997
+ raise ValueError("WILDS unlabeled samples must be 2- or 3-tuples.")
998
+
999
+ if indices is None:
1000
+ wilds_index = subset_index
1001
+ else:
1002
+ wilds_index = int(_as_numpy(indices[subset_index]).reshape(()))
1003
+
1004
+ sample = {
1005
+ "image": _image_to_uint8_array(image),
1006
+ "metadata": {
1007
+ "dataset": spec.name,
1008
+ "split": split,
1009
+ "example_id": str(wilds_index),
1010
+ "wilds_index": np.int64(wilds_index),
1011
+ "split_scheme": split_scheme,
1012
+ "version": version,
1013
+ "wilds": _metadata_to_int64_dict(metadata, metadata_fields),
1014
+ },
1015
+ }
1016
+ if include_label:
1017
+ sample["label"] = _label_to_int64(label)
1018
+ yield sample
1019
+
1020
+ output_signature = {
1021
+ "image": tf.TensorSpec(shape=None, dtype=tf.uint8),
1022
+ "metadata": {
1023
+ "dataset": tf.TensorSpec(shape=(), dtype=tf.string),
1024
+ "split": tf.TensorSpec(shape=(), dtype=tf.string),
1025
+ "example_id": tf.TensorSpec(shape=(), dtype=tf.string),
1026
+ "wilds_index": tf.TensorSpec(shape=(), dtype=tf.int64),
1027
+ "split_scheme": tf.TensorSpec(shape=(), dtype=tf.string),
1028
+ "version": tf.TensorSpec(shape=(), dtype=tf.string),
1029
+ "wilds": {
1030
+ field: tf.TensorSpec(shape=(), dtype=tf.int64)
1031
+ for field in metadata_fields
1032
+ },
1033
+ },
1034
+ }
1035
+ if include_label:
1036
+ output_signature["label"] = tf.TensorSpec(shape=(), dtype=tf.int64)
1037
+
1038
+ ds_tf = tf.data.Dataset.from_generator(gen, output_signature=output_signature)
1039
+ try:
1040
+ ds_tf = ds_tf.apply(tf.data.experimental.assert_cardinality(len(subset)))
1041
+ except Exception as e:
1042
+ logger.warning(
1043
+ f"Failed to assert cardinality for WILDS dataset {spec.name}: {e}"
1044
+ )
1045
+ return ds_tf
1046
+
1047
+
1048
+ @register_source_loader(_WILDS_PREFIX)
1049
+ def load_wilds_vision_splits(
1050
+ dataset_name: str,
1051
+ splits: list[str],
1052
+ data_dir: Union[None, str, os.PathLike] = None,
1053
+ ) -> list[tf.data.Dataset]:
1054
+ spec = _parse_wilds_spec(dataset_name)
1055
+ uses_unlabeled = _validate_wilds_splits(splits, spec)
1056
+ wilds = _import_wilds()
1057
+
1058
+ kwargs = {
1059
+ "dataset": spec.name,
1060
+ "download": spec.download,
1061
+ "split_scheme": spec.split_scheme,
1062
+ }
1063
+ kwargs["root_dir"] = os.fspath(source_cache_dir(data_dir, "wilds"))
1064
+ if spec.version is not None:
1065
+ kwargs["version"] = spec.version
1066
+ if spec.unlabeled or uses_unlabeled:
1067
+ kwargs["unlabeled"] = True
1068
+
1069
+ with _wilds_fmow_datetime_compat(spec.name == "fmow"):
1070
+ dataset = wilds.get_dataset(**kwargs)
1071
+ include_label = not (spec.unlabeled or uses_unlabeled)
1072
+ return [
1073
+ _split_to_dataset(
1074
+ dataset,
1075
+ split,
1076
+ spec=spec,
1077
+ include_label=include_label,
1078
+ )
1079
+ for split in splits
1080
+ ]