justdata 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- justdata/__init__.py +3 -0
- justdata/acoustic/__init__.py +381 -0
- justdata/acoustic/adapters.py +329 -0
- justdata/acoustic/augment/__init__.py +119 -0
- justdata/acoustic/augment/batch.py +813 -0
- justdata/acoustic/augment/configs.py +204 -0
- justdata/acoustic/augment/patchout.py +331 -0
- justdata/acoustic/augment/spectrogram.py +806 -0
- justdata/acoustic/augment/waveform.py +879 -0
- justdata/acoustic/batching.py +27 -0
- justdata/acoustic/channel.py +53 -0
- justdata/acoustic/compat/__init__.py +95 -0
- justdata/acoustic/compat/ast.py +626 -0
- justdata/acoustic/compat/ced.py +152 -0
- justdata/acoustic/compat/efficientat.py +243 -0
- justdata/acoustic/compat/panns.py +115 -0
- justdata/acoustic/compat/passt.py +191 -0
- justdata/acoustic/configs.py +519 -0
- justdata/acoustic/corruptions/__init__.py +33 -0
- justdata/acoustic/corruptions/codec.py +92 -0
- justdata/acoustic/corruptions/datasets.py +155 -0
- justdata/acoustic/corruptions/device.py +39 -0
- justdata/acoustic/corruptions/dynamics.py +55 -0
- justdata/acoustic/corruptions/filters.py +105 -0
- justdata/acoustic/corruptions/noise.py +173 -0
- justdata/acoustic/corruptions/registry.py +230 -0
- justdata/acoustic/corruptions/reverb.py +49 -0
- justdata/acoustic/corruptions/timefreq.py +127 -0
- justdata/acoustic/datasets.py +52 -0
- justdata/acoustic/dcase2025.py +814 -0
- justdata/acoustic/decoding.py +73 -0
- justdata/acoustic/eval_views.py +54 -0
- justdata/acoustic/evals.py +71 -0
- justdata/acoustic/filters.py +18 -0
- justdata/acoustic/frontends/__init__.py +37 -0
- justdata/acoustic/frontends/ast.py +116 -0
- justdata/acoustic/frontends/kaldi.py +34 -0
- justdata/acoustic/frontends/log.py +94 -0
- justdata/acoustic/frontends/mel.py +173 -0
- justdata/acoustic/frontends/mfcc.py +26 -0
- justdata/acoustic/frontends/pcen.py +26 -0
- justdata/acoustic/frontends/stft.py +174 -0
- justdata/acoustic/jax.py +23 -0
- justdata/acoustic/labels.py +169 -0
- justdata/acoustic/layouts.py +86 -0
- justdata/acoustic/metadata.py +152 -0
- justdata/acoustic/normalization.py +196 -0
- justdata/acoustic/pipelines.py +112 -0
- justdata/acoustic/postprocessing.py +328 -0
- justdata/acoustic/preprocessing.py +175 -0
- justdata/acoustic/presets.py +257 -0
- justdata/acoustic/registry.py +381 -0
- justdata/acoustic/resampling.py +218 -0
- justdata/acoustic/schema.py +51 -0
- justdata/acoustic/segment.py +369 -0
- justdata/acoustic/sources.py +503 -0
- justdata/acoustic/stages.py +59 -0
- justdata/acoustic/stats.py +50 -0
- justdata/acoustic/tasks.py +375 -0
- justdata/audio/__init__.py +20 -0
- justdata/core/__init__.py +74 -0
- justdata/core/adapters.py +46 -0
- justdata/core/augmentations.py +34 -0
- justdata/core/filters.py +94 -0
- justdata/core/finalization.py +173 -0
- justdata/core/label_mixing.py +129 -0
- justdata/core/loader.py +384 -0
- justdata/core/metadata.py +250 -0
- justdata/core/presets.py +155 -0
- justdata/core/registry.py +229 -0
- justdata/core/sources.py +87 -0
- justdata/core/stats.py +227 -0
- justdata/corruptions.py +52 -0
- justdata/py.typed +0 -0
- justdata/vision/__init__.py +17 -0
- justdata/vision/augmentations/__init__.py +31 -0
- justdata/vision/augmentations/auto.py +480 -0
- justdata/vision/augmentations/color.py +163 -0
- justdata/vision/augmentations/composed.py +156 -0
- justdata/vision/augmentations/geometric.py +302 -0
- justdata/vision/augmentations/mixing.py +232 -0
- justdata/vision/augmentations/registry.py +142 -0
- justdata/vision/corruptions/__init__.py +16 -0
- justdata/vision/corruptions/blur.py +65 -0
- justdata/vision/corruptions/digital.py +26 -0
- justdata/vision/corruptions/noise.py +20 -0
- justdata/vision/corruptions/registry.py +113 -0
- justdata/vision/corruptions/weather.py +100 -0
- justdata/vision/datasets.py +34 -0
- justdata/vision/encodings/__init__.py +1 -0
- justdata/vision/encodings/heatmaps.py +196 -0
- justdata/vision/minic.py +69 -0
- justdata/vision/pipelines.py +62 -0
- justdata/vision/presets.py +509 -0
- justdata/vision/sources.py +1080 -0
- justdata/vision/stages.py +271 -0
- justdata/vision/tasks/__init__.py +1 -0
- justdata/vision/tasks/classification.py +321 -0
- justdata/vision/tasks/depth.py +25 -0
- justdata/vision/tasks/segmentation.py +194 -0
- justdata/vision/transforms.py +113 -0
- justdata/vision/utils.py +741 -0
- justdata-1.0.0.dist-info/METADATA +860 -0
- justdata-1.0.0.dist-info/RECORD +106 -0
- justdata-1.0.0.dist-info/WHEEL +4 -0
- justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
|
@@ -0,0 +1,1080 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import shutil
|
|
6
|
+
import stat
|
|
7
|
+
import tarfile
|
|
8
|
+
import tempfile
|
|
9
|
+
import time
|
|
10
|
+
import urllib.request
|
|
11
|
+
import zipfile
|
|
12
|
+
from collections.abc import Iterator
|
|
13
|
+
from contextlib import contextmanager
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path, PurePosixPath, PureWindowsPath
|
|
16
|
+
from typing import Any, Union
|
|
17
|
+
from urllib.parse import parse_qsl, quote, urlparse
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
import tensorflow as tf
|
|
21
|
+
from loguru import logger
|
|
22
|
+
|
|
23
|
+
from justdata.core.sources import register_source_loader, source_cache_dir
|
|
24
|
+
|
|
25
|
+
_ZENODO_PREFIX = "zenodo:"
|
|
26
|
+
_ZENODO_QUERY_PARAMS = {"file"}
|
|
27
|
+
_ZENODO_ORIGIN = "zenodo.org"
|
|
28
|
+
_ZENODO_NETWORK_TIMEOUT_SECONDS = 60
|
|
29
|
+
_ZENODO_DOWNLOAD_TIMEOUT_SECONDS = 60 * 60
|
|
30
|
+
_ZENODO_DOWNLOAD_CHUNK_BYTES = 1024 * 1024
|
|
31
|
+
_ZENODO_MAX_ARCHIVE_BYTES = 20 * 1024**3
|
|
32
|
+
_ZENODO_MAX_EXTRACTED_BYTES = 100 * 1024**3
|
|
33
|
+
_ZENODO_MAX_ARCHIVE_MEMBERS = 100_000
|
|
34
|
+
_ZENODO_MAX_IMAGE_FILE_BYTES = 256 * 1024**2
|
|
35
|
+
_ZENODO_MAX_IMAGE_DIMENSION = 32_768
|
|
36
|
+
_ZENODO_MAX_IMAGE_PIXELS = 64_000_000
|
|
37
|
+
_IMAGE_EXTENSIONS = {
|
|
38
|
+
".bmp",
|
|
39
|
+
".gif",
|
|
40
|
+
".jpeg",
|
|
41
|
+
".jpg",
|
|
42
|
+
".png",
|
|
43
|
+
".webp",
|
|
44
|
+
}
|
|
45
|
+
_WILDS_PREFIX = "wilds:"
|
|
46
|
+
_SUPPORTED_WILDS_IMAGE_CLASSIFICATION = {
|
|
47
|
+
"camelyon17",
|
|
48
|
+
"fmow",
|
|
49
|
+
"iwildcam",
|
|
50
|
+
"rxrx1",
|
|
51
|
+
}
|
|
52
|
+
_UNSUPPORTED_WILDS_DATASETS = {
|
|
53
|
+
"amazon": "text",
|
|
54
|
+
"civilcomments": "text",
|
|
55
|
+
"globalwheat": "object detection",
|
|
56
|
+
"ogb-molpcba": "graph",
|
|
57
|
+
"poverty": "regression",
|
|
58
|
+
"py150": "code/text",
|
|
59
|
+
}
|
|
60
|
+
_WILDS_QUERY_PARAMS = {"download", "split_scheme", "unlabeled", "version"}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True)
|
|
64
|
+
class _ZenodoImageFolderSpec:
|
|
65
|
+
record_id: str
|
|
66
|
+
filename: str
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True)
|
|
70
|
+
class _WILDSDatasetSpec:
|
|
71
|
+
name: str
|
|
72
|
+
split_scheme: str
|
|
73
|
+
version: str | None
|
|
74
|
+
download: bool
|
|
75
|
+
unlabeled: bool
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _strip_prefix(dataset_name: str, prefix: str) -> str:
|
|
79
|
+
if not dataset_name.startswith(prefix):
|
|
80
|
+
raise ValueError(
|
|
81
|
+
f"Expected dataset name to start with {prefix!r}; got {dataset_name!r}."
|
|
82
|
+
)
|
|
83
|
+
return dataset_name[len(prefix) :]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _import_datasets():
|
|
87
|
+
try:
|
|
88
|
+
import datasets
|
|
89
|
+
except ImportError as e:
|
|
90
|
+
raise ImportError(
|
|
91
|
+
"The 'datasets' package is required for hf: vision source loading. "
|
|
92
|
+
"Install justdata with the vision extra."
|
|
93
|
+
) from e
|
|
94
|
+
return datasets
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _parse_zenodo_spec(dataset_name: str) -> _ZenodoImageFolderSpec:
|
|
98
|
+
raw = _strip_prefix(dataset_name, _ZENODO_PREFIX)
|
|
99
|
+
record_id, sep, query = raw.partition("?")
|
|
100
|
+
if not re.fullmatch(r"[1-9][0-9]*", record_id):
|
|
101
|
+
raise ValueError("Zenodo record id must be a positive integer.")
|
|
102
|
+
|
|
103
|
+
values: dict[str, str] = {}
|
|
104
|
+
for key, value in parse_qsl(query if sep else "", keep_blank_values=True):
|
|
105
|
+
if key not in _ZENODO_QUERY_PARAMS:
|
|
106
|
+
allowed = ", ".join(sorted(_ZENODO_QUERY_PARAMS))
|
|
107
|
+
raise ValueError(
|
|
108
|
+
f"Unsupported Zenodo option {key!r}. Supported options: {allowed}."
|
|
109
|
+
)
|
|
110
|
+
if key in values:
|
|
111
|
+
raise ValueError(f"Duplicate Zenodo option {key!r}.")
|
|
112
|
+
values[key] = value
|
|
113
|
+
|
|
114
|
+
filename = values.get("file", "")
|
|
115
|
+
if not filename:
|
|
116
|
+
raise ValueError("Zenodo source requires ?file=<archive-name>.")
|
|
117
|
+
_validate_zenodo_filename(filename)
|
|
118
|
+
|
|
119
|
+
return _ZenodoImageFolderSpec(record_id=record_id, filename=filename)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _validate_zenodo_filename(filename: str) -> None:
|
|
123
|
+
posix_path = PurePosixPath(filename)
|
|
124
|
+
windows_path = PureWindowsPath(filename)
|
|
125
|
+
if (
|
|
126
|
+
filename in {".", ".."}
|
|
127
|
+
or "\x00" in filename
|
|
128
|
+
or posix_path.is_absolute()
|
|
129
|
+
or windows_path.is_absolute()
|
|
130
|
+
or windows_path.drive
|
|
131
|
+
or posix_path.name != filename
|
|
132
|
+
or windows_path.name != filename
|
|
133
|
+
or "/" in filename
|
|
134
|
+
or "\\" in filename
|
|
135
|
+
):
|
|
136
|
+
raise ValueError(
|
|
137
|
+
f"Zenodo archive filename must be a basename without path segments: "
|
|
138
|
+
f"{filename!r}."
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _zenodo_record_dir(
|
|
143
|
+
spec: _ZenodoImageFolderSpec,
|
|
144
|
+
data_dir: Union[None, str, os.PathLike],
|
|
145
|
+
) -> Path:
|
|
146
|
+
return source_cache_dir(data_dir, "zenodo", spec.record_id)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _fetch_zenodo_record(record_id: str) -> dict[str, Any]:
|
|
150
|
+
url = f"https://zenodo.org/api/records/{record_id}"
|
|
151
|
+
with urllib.request.urlopen(url, timeout=60) as response:
|
|
152
|
+
return json.load(response)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _select_zenodo_file(record: dict[str, Any], filename: str) -> dict[str, Any]:
|
|
156
|
+
if not isinstance(record, dict):
|
|
157
|
+
raise ValueError("Zenodo record metadata must be an object.")
|
|
158
|
+
files = record.get("files")
|
|
159
|
+
if not isinstance(files, list):
|
|
160
|
+
raise ValueError("Zenodo record metadata does not contain a files list.")
|
|
161
|
+
|
|
162
|
+
for file_info in files:
|
|
163
|
+
if not isinstance(file_info, dict):
|
|
164
|
+
continue
|
|
165
|
+
if file_info.get("key") == filename or file_info.get("filename") == filename:
|
|
166
|
+
return file_info
|
|
167
|
+
|
|
168
|
+
available = ", ".join(
|
|
169
|
+
str(file_info.get("key") or file_info.get("filename")) for file_info in files
|
|
170
|
+
)
|
|
171
|
+
raise ValueError(
|
|
172
|
+
f"Zenodo record does not contain file {filename!r}. "
|
|
173
|
+
f"Available files: {available}."
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _zenodo_download_url(
|
|
178
|
+
spec: _ZenodoImageFolderSpec,
|
|
179
|
+
file_info: dict[str, Any],
|
|
180
|
+
) -> str:
|
|
181
|
+
links = file_info.get("links")
|
|
182
|
+
if links is None:
|
|
183
|
+
links = {}
|
|
184
|
+
if not isinstance(links, dict):
|
|
185
|
+
raise ValueError("Zenodo file links metadata must be an object.")
|
|
186
|
+
for key in ("content", "download"):
|
|
187
|
+
value = links.get(key)
|
|
188
|
+
if value:
|
|
189
|
+
return _validate_zenodo_download_url(str(value))
|
|
190
|
+
|
|
191
|
+
value = links.get("self")
|
|
192
|
+
if value and str(value).rstrip("/").endswith("/content"):
|
|
193
|
+
return _validate_zenodo_download_url(str(value))
|
|
194
|
+
|
|
195
|
+
quoted = quote(spec.filename)
|
|
196
|
+
return _validate_zenodo_download_url(
|
|
197
|
+
f"https://zenodo.org/records/{spec.record_id}/files/{quoted}?download=1"
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _validate_zenodo_download_url(url: str) -> str:
|
|
202
|
+
parsed = urlparse(url)
|
|
203
|
+
try:
|
|
204
|
+
port = parsed.port
|
|
205
|
+
except ValueError as e:
|
|
206
|
+
raise ValueError(
|
|
207
|
+
"Zenodo download link must use the https://zenodo.org origin."
|
|
208
|
+
) from e
|
|
209
|
+
if (
|
|
210
|
+
parsed.scheme != "https"
|
|
211
|
+
or parsed.hostname != _ZENODO_ORIGIN
|
|
212
|
+
or port not in (None, 443)
|
|
213
|
+
or parsed.username is not None
|
|
214
|
+
or parsed.password is not None
|
|
215
|
+
):
|
|
216
|
+
raise ValueError("Zenodo download link must use the https://zenodo.org origin.")
|
|
217
|
+
return url
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _zenodo_declared_size(file_info: dict[str, Any]) -> int | None:
|
|
221
|
+
size = file_info.get("size")
|
|
222
|
+
if size is None:
|
|
223
|
+
return None
|
|
224
|
+
if isinstance(size, bool) or not isinstance(size, int) or size < 0:
|
|
225
|
+
raise ValueError("Zenodo file size must be a non-negative integer.")
|
|
226
|
+
if size > _ZENODO_MAX_ARCHIVE_BYTES:
|
|
227
|
+
raise ValueError(
|
|
228
|
+
f"Zenodo file size {size} exceeds the archive byte limit "
|
|
229
|
+
f"({_ZENODO_MAX_ARCHIVE_BYTES})."
|
|
230
|
+
)
|
|
231
|
+
return size
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _checksum_parts(checksum: str | None) -> tuple[str, str] | None:
|
|
235
|
+
if not checksum:
|
|
236
|
+
return None
|
|
237
|
+
if not isinstance(checksum, str):
|
|
238
|
+
raise ValueError("Zenodo checksum must be a string.")
|
|
239
|
+
|
|
240
|
+
algorithm, sep, expected = checksum.partition(":")
|
|
241
|
+
if not sep:
|
|
242
|
+
algorithm, expected = "md5", algorithm
|
|
243
|
+
algorithm = algorithm.lower()
|
|
244
|
+
if algorithm not in hashlib.algorithms_available:
|
|
245
|
+
raise ValueError(f"Unsupported Zenodo checksum algorithm {algorithm!r}.")
|
|
246
|
+
return algorithm, expected.lower()
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _file_matches_checksum(path: Path, checksum: str | None) -> bool:
|
|
250
|
+
if not path.exists():
|
|
251
|
+
return False
|
|
252
|
+
|
|
253
|
+
parts = _checksum_parts(checksum)
|
|
254
|
+
if parts is None:
|
|
255
|
+
return True
|
|
256
|
+
|
|
257
|
+
algorithm, expected = parts
|
|
258
|
+
digest = hashlib.new(algorithm)
|
|
259
|
+
with path.open("rb") as handle:
|
|
260
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
261
|
+
digest.update(chunk)
|
|
262
|
+
return digest.hexdigest().lower() == expected
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _download_file(
|
|
266
|
+
url: str,
|
|
267
|
+
target: Path,
|
|
268
|
+
*,
|
|
269
|
+
expected_size: int | None,
|
|
270
|
+
checksum: str | None,
|
|
271
|
+
) -> None:
|
|
272
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
273
|
+
parts = _checksum_parts(checksum)
|
|
274
|
+
digest = hashlib.new(parts[0]) if parts is not None else None
|
|
275
|
+
started_at = time.monotonic()
|
|
276
|
+
bytes_downloaded = 0
|
|
277
|
+
tmp_path: Path | None = None
|
|
278
|
+
try:
|
|
279
|
+
with tempfile.NamedTemporaryFile(
|
|
280
|
+
mode="wb",
|
|
281
|
+
prefix=f".{target.name}.",
|
|
282
|
+
suffix=".tmp",
|
|
283
|
+
dir=target.parent,
|
|
284
|
+
delete=False,
|
|
285
|
+
) as tmp:
|
|
286
|
+
tmp_path = Path(tmp.name)
|
|
287
|
+
with urllib.request.urlopen(
|
|
288
|
+
url, timeout=_ZENODO_NETWORK_TIMEOUT_SECONDS
|
|
289
|
+
) as response:
|
|
290
|
+
while True:
|
|
291
|
+
if time.monotonic() - started_at > _ZENODO_DOWNLOAD_TIMEOUT_SECONDS:
|
|
292
|
+
raise TimeoutError(
|
|
293
|
+
"Zenodo download exceeded the transfer time limit."
|
|
294
|
+
)
|
|
295
|
+
chunk = response.read(_ZENODO_DOWNLOAD_CHUNK_BYTES)
|
|
296
|
+
if not chunk:
|
|
297
|
+
break
|
|
298
|
+
bytes_downloaded += len(chunk)
|
|
299
|
+
byte_limit = (
|
|
300
|
+
expected_size
|
|
301
|
+
if expected_size is not None
|
|
302
|
+
else _ZENODO_MAX_ARCHIVE_BYTES
|
|
303
|
+
)
|
|
304
|
+
if bytes_downloaded > byte_limit:
|
|
305
|
+
raise ValueError(
|
|
306
|
+
"Zenodo download exceeded its declared or configured "
|
|
307
|
+
"archive byte limit."
|
|
308
|
+
)
|
|
309
|
+
tmp.write(chunk)
|
|
310
|
+
if digest is not None:
|
|
311
|
+
digest.update(chunk)
|
|
312
|
+
|
|
313
|
+
if expected_size is not None and bytes_downloaded != expected_size:
|
|
314
|
+
raise ValueError(
|
|
315
|
+
f"Zenodo download size mismatch: expected {expected_size} "
|
|
316
|
+
f"bytes, received {bytes_downloaded}."
|
|
317
|
+
)
|
|
318
|
+
if parts is not None and digest.hexdigest().lower() != parts[1]:
|
|
319
|
+
raise ValueError(
|
|
320
|
+
f"Downloaded Zenodo file {target.name!r} failed checksum "
|
|
321
|
+
"validation."
|
|
322
|
+
)
|
|
323
|
+
tmp.flush()
|
|
324
|
+
os.fsync(tmp.fileno())
|
|
325
|
+
os.replace(tmp_path, target)
|
|
326
|
+
tmp_path = None
|
|
327
|
+
finally:
|
|
328
|
+
if tmp_path is not None:
|
|
329
|
+
tmp_path.unlink(missing_ok=True)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _archive_stem(filename: str) -> str:
|
|
333
|
+
path = Path(filename)
|
|
334
|
+
suffixes = path.suffixes
|
|
335
|
+
if len(suffixes) >= 2 and suffixes[-2:] in (
|
|
336
|
+
[".tar", ".gz"],
|
|
337
|
+
[".tar", ".bz2"],
|
|
338
|
+
[".tar", ".xz"],
|
|
339
|
+
):
|
|
340
|
+
return path.name[: -len("".join(suffixes[-2:]))]
|
|
341
|
+
return path.stem
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _assert_safe_extract_path(destination: Path, member_name: str) -> None:
|
|
345
|
+
path = PurePosixPath(member_name)
|
|
346
|
+
windows_path = PureWindowsPath(member_name)
|
|
347
|
+
if (
|
|
348
|
+
not member_name
|
|
349
|
+
or "\\" in member_name
|
|
350
|
+
or path.is_absolute()
|
|
351
|
+
or windows_path.is_absolute()
|
|
352
|
+
or windows_path.drive
|
|
353
|
+
or any(part in {"", ".", ".."} for part in path.parts)
|
|
354
|
+
):
|
|
355
|
+
raise ValueError(f"Unsafe archive member path: {member_name!r}.")
|
|
356
|
+
target = (destination / member_name).resolve()
|
|
357
|
+
root = destination.resolve()
|
|
358
|
+
if target != root and root not in target.parents:
|
|
359
|
+
raise ValueError(f"Archive member escapes extraction directory: {member_name}")
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _check_archive_budget(
|
|
363
|
+
destination: Path,
|
|
364
|
+
members: list[tuple[str, int]],
|
|
365
|
+
) -> None:
|
|
366
|
+
if len(members) > _ZENODO_MAX_ARCHIVE_MEMBERS:
|
|
367
|
+
raise ValueError(
|
|
368
|
+
f"Archive member count {len(members)} exceeds the limit "
|
|
369
|
+
f"({_ZENODO_MAX_ARCHIVE_MEMBERS})."
|
|
370
|
+
)
|
|
371
|
+
expanded_bytes = sum(size for _, size in members)
|
|
372
|
+
if expanded_bytes > _ZENODO_MAX_EXTRACTED_BYTES:
|
|
373
|
+
raise ValueError(
|
|
374
|
+
f"Archive expanded size {expanded_bytes} exceeds the byte limit "
|
|
375
|
+
f"({_ZENODO_MAX_EXTRACTED_BYTES})."
|
|
376
|
+
)
|
|
377
|
+
free_bytes = shutil.disk_usage(destination.parent).free
|
|
378
|
+
if expanded_bytes > free_bytes:
|
|
379
|
+
raise ValueError(
|
|
380
|
+
f"Archive requires {expanded_bytes} extracted bytes but the cache has "
|
|
381
|
+
f"only {free_bytes} bytes free."
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _normalized_archive_member(destination: Path, member_name: str) -> str:
|
|
386
|
+
_assert_safe_extract_path(destination, member_name)
|
|
387
|
+
return PurePosixPath(member_name).as_posix().rstrip("/")
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _preflight_zip(archive: zipfile.ZipFile, destination: Path) -> None:
|
|
391
|
+
members: list[tuple[str, int]] = []
|
|
392
|
+
destinations: set[str] = set()
|
|
393
|
+
for member in archive.infolist():
|
|
394
|
+
normalized = _normalized_archive_member(destination, member.filename)
|
|
395
|
+
if normalized in destinations:
|
|
396
|
+
raise ValueError(
|
|
397
|
+
f"Archive contains duplicate member path: {member.filename!r}."
|
|
398
|
+
)
|
|
399
|
+
destinations.add(normalized)
|
|
400
|
+
|
|
401
|
+
mode = member.external_attr >> 16
|
|
402
|
+
file_type = stat.S_IFMT(mode)
|
|
403
|
+
is_directory = member.is_dir()
|
|
404
|
+
if file_type == stat.S_IFLNK:
|
|
405
|
+
raise ValueError(f"Archive member uses a link: {member.filename!r}.")
|
|
406
|
+
if file_type not in (0, stat.S_IFREG, stat.S_IFDIR) or (
|
|
407
|
+
file_type == stat.S_IFDIR and not is_directory
|
|
408
|
+
):
|
|
409
|
+
raise ValueError(
|
|
410
|
+
f"Archive member is not a regular file or directory: "
|
|
411
|
+
f"{member.filename!r}."
|
|
412
|
+
)
|
|
413
|
+
members.append((member.filename, 0 if is_directory else member.file_size))
|
|
414
|
+
_check_archive_budget(destination, members)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _preflight_tar(archive: tarfile.TarFile, destination: Path) -> None:
|
|
418
|
+
members: list[tuple[str, int]] = []
|
|
419
|
+
destinations: set[str] = set()
|
|
420
|
+
for member in archive.getmembers():
|
|
421
|
+
normalized = _normalized_archive_member(destination, member.name)
|
|
422
|
+
if normalized in destinations:
|
|
423
|
+
raise ValueError(
|
|
424
|
+
f"Archive contains duplicate member path: {member.name!r}."
|
|
425
|
+
)
|
|
426
|
+
destinations.add(normalized)
|
|
427
|
+
if not (member.isfile() or member.isdir()):
|
|
428
|
+
category = "link" if member.issym() or member.islnk() else "special file"
|
|
429
|
+
raise ValueError(f"Archive member uses a {category}: {member.name!r}.")
|
|
430
|
+
members.append((member.name, member.size if member.isfile() else 0))
|
|
431
|
+
_check_archive_budget(destination, members)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _extract_archive(archive_path: Path, extract_dir: Path) -> None:
|
|
435
|
+
extract_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
436
|
+
tmp_dir = Path(
|
|
437
|
+
tempfile.mkdtemp(prefix=f".{extract_dir.name}.", dir=extract_dir.parent)
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
try:
|
|
441
|
+
if zipfile.is_zipfile(archive_path):
|
|
442
|
+
with zipfile.ZipFile(archive_path) as archive:
|
|
443
|
+
_preflight_zip(archive, tmp_dir)
|
|
444
|
+
archive.extractall(tmp_dir)
|
|
445
|
+
elif tarfile.is_tarfile(archive_path):
|
|
446
|
+
with tarfile.open(archive_path) as archive:
|
|
447
|
+
_preflight_tar(archive, tmp_dir)
|
|
448
|
+
if hasattr(tarfile, "data_filter"):
|
|
449
|
+
archive.extractall(tmp_dir, filter="data")
|
|
450
|
+
else:
|
|
451
|
+
archive.extractall(tmp_dir)
|
|
452
|
+
else:
|
|
453
|
+
raise ValueError(
|
|
454
|
+
f"Zenodo file {archive_path.name!r} is not a supported ZIP/TAR archive."
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
if extract_dir.exists():
|
|
458
|
+
shutil.rmtree(extract_dir)
|
|
459
|
+
os.replace(tmp_dir, extract_dir)
|
|
460
|
+
except Exception:
|
|
461
|
+
if tmp_dir.exists():
|
|
462
|
+
shutil.rmtree(tmp_dir)
|
|
463
|
+
raise
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def _prepare_zenodo_archive(
|
|
467
|
+
spec: _ZenodoImageFolderSpec,
|
|
468
|
+
data_dir: Union[None, str, os.PathLike],
|
|
469
|
+
) -> Path:
|
|
470
|
+
record = _fetch_zenodo_record(spec.record_id)
|
|
471
|
+
file_info = _select_zenodo_file(record, spec.filename)
|
|
472
|
+
checksum = file_info.get("checksum")
|
|
473
|
+
expected_size = _zenodo_declared_size(file_info)
|
|
474
|
+
download_url = _zenodo_download_url(spec, file_info)
|
|
475
|
+
|
|
476
|
+
record_dir = _zenodo_record_dir(spec, data_dir)
|
|
477
|
+
archive_path = record_dir / "files" / spec.filename
|
|
478
|
+
extract_dir = record_dir / "extracted" / _archive_stem(spec.filename)
|
|
479
|
+
lock_path = record_dir / "locks" / f"{spec.filename}.lock"
|
|
480
|
+
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
481
|
+
|
|
482
|
+
try:
|
|
483
|
+
from filelock import FileLock
|
|
484
|
+
except ImportError as e:
|
|
485
|
+
raise ImportError(
|
|
486
|
+
"Zenodo source loading requires the 'filelock' package. Install "
|
|
487
|
+
"justdata with the vision extra."
|
|
488
|
+
) from e
|
|
489
|
+
|
|
490
|
+
with FileLock(lock_path):
|
|
491
|
+
if not _file_matches_checksum(archive_path, checksum):
|
|
492
|
+
archive_path.unlink(missing_ok=True)
|
|
493
|
+
logger.info(
|
|
494
|
+
f"Downloading Zenodo record {spec.record_id} file {spec.filename}."
|
|
495
|
+
)
|
|
496
|
+
_download_file(
|
|
497
|
+
download_url,
|
|
498
|
+
archive_path,
|
|
499
|
+
expected_size=expected_size,
|
|
500
|
+
checksum=checksum,
|
|
501
|
+
)
|
|
502
|
+
if extract_dir.exists():
|
|
503
|
+
shutil.rmtree(extract_dir)
|
|
504
|
+
|
|
505
|
+
if not extract_dir.exists():
|
|
506
|
+
logger.info(f"Extracting Zenodo archive {archive_path}.")
|
|
507
|
+
_extract_archive(archive_path, extract_dir)
|
|
508
|
+
|
|
509
|
+
return extract_dir
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _resolve_imagefolder_root(extract_dir: Path, splits: list[str]) -> Path:
|
|
513
|
+
if all((extract_dir / split).is_dir() for split in splits):
|
|
514
|
+
return extract_dir
|
|
515
|
+
|
|
516
|
+
candidates = [
|
|
517
|
+
child
|
|
518
|
+
for child in extract_dir.iterdir()
|
|
519
|
+
if child.is_dir() and all((child / split).is_dir() for split in splits)
|
|
520
|
+
]
|
|
521
|
+
if len(candidates) == 1:
|
|
522
|
+
return candidates[0]
|
|
523
|
+
if len(candidates) > 1:
|
|
524
|
+
names = ", ".join(sorted(candidate.name for candidate in candidates))
|
|
525
|
+
raise ValueError(
|
|
526
|
+
f"Multiple ImageFolder roots contain requested splits: {names}."
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
requested = ", ".join(splits)
|
|
530
|
+
raise ValueError(
|
|
531
|
+
f"Zenodo archive does not contain requested ImageFolder splits: {requested}."
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _is_image_file(path: Path) -> bool:
|
|
536
|
+
return path.is_file() and path.suffix.lower() in _IMAGE_EXTENSIONS
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _validate_image_file(path: Path) -> None:
|
|
540
|
+
size = path.stat().st_size
|
|
541
|
+
if size > _ZENODO_MAX_IMAGE_FILE_BYTES:
|
|
542
|
+
raise ValueError(
|
|
543
|
+
f"Zenodo image {path} exceeds the encoded-file limit of "
|
|
544
|
+
f"{_ZENODO_MAX_IMAGE_FILE_BYTES} bytes."
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
try:
|
|
548
|
+
from PIL import Image, UnidentifiedImageError
|
|
549
|
+
except ImportError as e:
|
|
550
|
+
raise ImportError(
|
|
551
|
+
"Pillow is required for bounded Zenodo image validation. "
|
|
552
|
+
"Install justdata with the vision extra."
|
|
553
|
+
) from e
|
|
554
|
+
|
|
555
|
+
try:
|
|
556
|
+
with Image.open(path) as image:
|
|
557
|
+
width, height = image.size
|
|
558
|
+
except (Image.DecompressionBombError, UnidentifiedImageError, OSError) as e:
|
|
559
|
+
raise ValueError(f"Zenodo image {path} has an invalid image header.") from e
|
|
560
|
+
|
|
561
|
+
if width <= 0 or height <= 0:
|
|
562
|
+
raise ValueError(
|
|
563
|
+
f"Zenodo image {path} has invalid dimensions {width}x{height}."
|
|
564
|
+
)
|
|
565
|
+
if width > _ZENODO_MAX_IMAGE_DIMENSION or height > _ZENODO_MAX_IMAGE_DIMENSION:
|
|
566
|
+
raise ValueError(
|
|
567
|
+
f"Zenodo image {path} dimensions {width}x{height} exceed the "
|
|
568
|
+
f"per-dimension limit of {_ZENODO_MAX_IMAGE_DIMENSION}."
|
|
569
|
+
)
|
|
570
|
+
pixels = width * height
|
|
571
|
+
if pixels > _ZENODO_MAX_IMAGE_PIXELS:
|
|
572
|
+
raise ValueError(
|
|
573
|
+
f"Zenodo image {path} has {pixels} decoded pixels, exceeding the "
|
|
574
|
+
f"limit of {_ZENODO_MAX_IMAGE_PIXELS}."
|
|
575
|
+
)
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _split_class_dirs(split_dir: Path) -> list[Path]:
|
|
579
|
+
return sorted(
|
|
580
|
+
child
|
|
581
|
+
for child in split_dir.iterdir()
|
|
582
|
+
if child.is_dir() and not child.name.startswith(".")
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def _imagefolder_split_dirs(root: Path) -> list[Path]:
|
|
587
|
+
return sorted(
|
|
588
|
+
child
|
|
589
|
+
for child in root.iterdir()
|
|
590
|
+
if child.is_dir()
|
|
591
|
+
and not child.name.startswith(".")
|
|
592
|
+
and _split_class_dirs(child)
|
|
593
|
+
)
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _imagefolder_class_to_label(root: Path) -> dict[str, np.int64]:
|
|
597
|
+
class_names = sorted(
|
|
598
|
+
{
|
|
599
|
+
class_dir.name
|
|
600
|
+
for split_dir in _imagefolder_split_dirs(root)
|
|
601
|
+
for class_dir in _split_class_dirs(split_dir)
|
|
602
|
+
}
|
|
603
|
+
)
|
|
604
|
+
if not class_names:
|
|
605
|
+
raise ValueError("Zenodo ImageFolder archive does not contain class folders.")
|
|
606
|
+
return {name: np.int64(index) for index, name in enumerate(class_names)}
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def _imagefolder_records(
|
|
610
|
+
root: Path,
|
|
611
|
+
splits: list[str],
|
|
612
|
+
*,
|
|
613
|
+
dataset_name: str,
|
|
614
|
+
spec: _ZenodoImageFolderSpec,
|
|
615
|
+
class_to_label: dict[str, np.int64],
|
|
616
|
+
) -> list[dict[str, Any]]:
|
|
617
|
+
split_dirs = {split: root / split for split in splits}
|
|
618
|
+
records: list[dict[str, Any]] = []
|
|
619
|
+
for split in splits:
|
|
620
|
+
split_records = []
|
|
621
|
+
split_dir = split_dirs[split]
|
|
622
|
+
class_dirs = _split_class_dirs(split_dir)
|
|
623
|
+
unknown_classes = sorted(
|
|
624
|
+
class_dir.name
|
|
625
|
+
for class_dir in class_dirs
|
|
626
|
+
if class_dir.name not in class_to_label
|
|
627
|
+
)
|
|
628
|
+
if unknown_classes:
|
|
629
|
+
names = ", ".join(unknown_classes)
|
|
630
|
+
raise ValueError(
|
|
631
|
+
f"Zenodo ImageFolder split {split!r} contains classes absent from "
|
|
632
|
+
f"the archive-wide vocabulary: {names}."
|
|
633
|
+
)
|
|
634
|
+
for class_dir in class_dirs:
|
|
635
|
+
label = class_to_label[class_dir.name]
|
|
636
|
+
for path in sorted(class_dir.rglob("*")):
|
|
637
|
+
if not _is_image_file(path):
|
|
638
|
+
continue
|
|
639
|
+
_validate_image_file(path)
|
|
640
|
+
relative_path = path.relative_to(root).as_posix()
|
|
641
|
+
split_records.append(
|
|
642
|
+
{
|
|
643
|
+
"_path": os.fspath(path),
|
|
644
|
+
"label": label,
|
|
645
|
+
"metadata": {
|
|
646
|
+
"dataset": dataset_name,
|
|
647
|
+
"record_id": spec.record_id,
|
|
648
|
+
"archive": spec.filename,
|
|
649
|
+
"split": split,
|
|
650
|
+
"class_name": class_dir.name,
|
|
651
|
+
"class_index": label,
|
|
652
|
+
"filename": path.name,
|
|
653
|
+
"path": os.fspath(path),
|
|
654
|
+
"example_id": relative_path,
|
|
655
|
+
},
|
|
656
|
+
}
|
|
657
|
+
)
|
|
658
|
+
if not split_records:
|
|
659
|
+
raise ValueError(f"Zenodo ImageFolder split {split!r} has no images.")
|
|
660
|
+
records.extend(split_records)
|
|
661
|
+
|
|
662
|
+
return records
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def _records_to_vision_dataset(records: list[dict[str, Any]]) -> tf.data.Dataset:
|
|
666
|
+
def gen():
|
|
667
|
+
yield from records
|
|
668
|
+
|
|
669
|
+
ds = tf.data.Dataset.from_generator(
|
|
670
|
+
gen,
|
|
671
|
+
output_signature={
|
|
672
|
+
"_path": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
673
|
+
"label": tf.TensorSpec(shape=(), dtype=tf.int64),
|
|
674
|
+
"metadata": {
|
|
675
|
+
"dataset": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
676
|
+
"record_id": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
677
|
+
"archive": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
678
|
+
"split": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
679
|
+
"class_name": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
680
|
+
"class_index": tf.TensorSpec(shape=(), dtype=tf.int64),
|
|
681
|
+
"filename": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
682
|
+
"path": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
683
|
+
"example_id": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
684
|
+
},
|
|
685
|
+
},
|
|
686
|
+
)
|
|
687
|
+
|
|
688
|
+
def decode_image(sample):
|
|
689
|
+
image = tf.io.decode_image(
|
|
690
|
+
tf.io.read_file(sample["_path"]),
|
|
691
|
+
channels=3,
|
|
692
|
+
expand_animations=False,
|
|
693
|
+
)
|
|
694
|
+
image.set_shape([None, None, 3])
|
|
695
|
+
return {
|
|
696
|
+
"image": image,
|
|
697
|
+
"label": sample["label"],
|
|
698
|
+
"metadata": sample["metadata"],
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
ds = ds.map(decode_image, num_parallel_calls=tf.data.AUTOTUNE)
|
|
702
|
+
try:
|
|
703
|
+
ds = ds.apply(tf.data.experimental.assert_cardinality(len(records)))
|
|
704
|
+
except Exception as e:
|
|
705
|
+
logger.warning(f"Failed to assert cardinality for Zenodo ImageFolder: {e}")
|
|
706
|
+
return ds
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
@register_source_loader(_ZENODO_PREFIX)
|
|
710
|
+
def load_zenodo_imagefolder_splits(
|
|
711
|
+
dataset_name: str,
|
|
712
|
+
splits: list[str],
|
|
713
|
+
data_dir: Union[None, str, os.PathLike] = None,
|
|
714
|
+
) -> list[tf.data.Dataset]:
|
|
715
|
+
spec = _parse_zenodo_spec(dataset_name)
|
|
716
|
+
extract_dir = _prepare_zenodo_archive(spec, data_dir)
|
|
717
|
+
root = _resolve_imagefolder_root(extract_dir, splits)
|
|
718
|
+
class_to_label = _imagefolder_class_to_label(root)
|
|
719
|
+
return [
|
|
720
|
+
_records_to_vision_dataset(
|
|
721
|
+
_imagefolder_records(
|
|
722
|
+
root,
|
|
723
|
+
[split],
|
|
724
|
+
dataset_name=dataset_name,
|
|
725
|
+
spec=spec,
|
|
726
|
+
class_to_label=class_to_label,
|
|
727
|
+
)
|
|
728
|
+
)
|
|
729
|
+
for split in splits
|
|
730
|
+
]
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
@register_source_loader("hf:")
|
|
734
|
+
def load_huggingface_vision_splits(
|
|
735
|
+
dataset_name: str,
|
|
736
|
+
splits: list[str],
|
|
737
|
+
data_dir: Union[None, str, os.PathLike] = None,
|
|
738
|
+
*,
|
|
739
|
+
include_metadata: bool = False,
|
|
740
|
+
) -> list[tf.data.Dataset]:
|
|
741
|
+
datasets = _import_datasets()
|
|
742
|
+
hf_name = dataset_name[3:]
|
|
743
|
+
loaded_splits = []
|
|
744
|
+
|
|
745
|
+
for split in splits:
|
|
746
|
+
ds_hf = datasets.load_dataset(
|
|
747
|
+
hf_name,
|
|
748
|
+
split=split,
|
|
749
|
+
cache_dir=os.fspath(source_cache_dir(data_dir, "hf", "vision")),
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
if "image" not in ds_hf.column_names or "label" not in ds_hf.column_names:
|
|
753
|
+
raise ValueError(
|
|
754
|
+
f"Hugging Face vision dataset '{hf_name}' must contain 'image' and "
|
|
755
|
+
f"'label' columns. Found columns: {ds_hf.column_names}."
|
|
756
|
+
)
|
|
757
|
+
|
|
758
|
+
def gen(_ds=ds_hf, _split=split):
|
|
759
|
+
example_index = 0
|
|
760
|
+
for batch in _ds.iter(batch_size=1024):
|
|
761
|
+
for img, lbl in zip(batch["image"], batch["label"]):
|
|
762
|
+
item = {
|
|
763
|
+
"image": np.array(img),
|
|
764
|
+
"label": lbl,
|
|
765
|
+
}
|
|
766
|
+
if include_metadata:
|
|
767
|
+
filename = getattr(img, "filename", "") or ""
|
|
768
|
+
item["metadata"] = {
|
|
769
|
+
"dataset": hf_name,
|
|
770
|
+
"split": _split,
|
|
771
|
+
"example_id": str(example_index),
|
|
772
|
+
"filename": os.path.basename(filename) if filename else "",
|
|
773
|
+
}
|
|
774
|
+
example_index += 1
|
|
775
|
+
yield item
|
|
776
|
+
|
|
777
|
+
output_signature = {
|
|
778
|
+
"image": tf.TensorSpec(shape=None, dtype=tf.uint8),
|
|
779
|
+
"label": tf.TensorSpec(shape=(), dtype=tf.int64),
|
|
780
|
+
}
|
|
781
|
+
if include_metadata:
|
|
782
|
+
output_signature["metadata"] = {
|
|
783
|
+
"dataset": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
784
|
+
"split": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
785
|
+
"example_id": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
786
|
+
"filename": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
ds_tf = tf.data.Dataset.from_generator(
|
|
790
|
+
gen,
|
|
791
|
+
output_signature=output_signature,
|
|
792
|
+
)
|
|
793
|
+
|
|
794
|
+
try:
|
|
795
|
+
ds_tf = ds_tf.apply(tf.data.experimental.assert_cardinality(len(ds_hf)))
|
|
796
|
+
except Exception as e:
|
|
797
|
+
logger.warning(
|
|
798
|
+
f"Failed to assert cardinality for HF dataset {hf_name}: {e}"
|
|
799
|
+
)
|
|
800
|
+
|
|
801
|
+
loaded_splits.append(ds_tf)
|
|
802
|
+
|
|
803
|
+
return loaded_splits
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def _parse_bool(value: str, *, key: str) -> bool:
|
|
807
|
+
normalized = value.strip().lower()
|
|
808
|
+
if normalized in {"1", "true", "yes", "y"}:
|
|
809
|
+
return True
|
|
810
|
+
if normalized in {"0", "false", "no", "n"}:
|
|
811
|
+
return False
|
|
812
|
+
raise ValueError(f"{key} must be a boolean value; got {value!r}.")
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _parse_wilds_spec(dataset_name: str) -> _WILDSDatasetSpec:
|
|
816
|
+
if not dataset_name.startswith(_WILDS_PREFIX):
|
|
817
|
+
raise ValueError(
|
|
818
|
+
f"Expected WILDS dataset name to start with {_WILDS_PREFIX!r}; "
|
|
819
|
+
f"got {dataset_name!r}."
|
|
820
|
+
)
|
|
821
|
+
|
|
822
|
+
raw = dataset_name[len(_WILDS_PREFIX) :]
|
|
823
|
+
name, sep, query = raw.partition("?")
|
|
824
|
+
if not name:
|
|
825
|
+
raise ValueError("WILDS dataset name cannot be empty.")
|
|
826
|
+
|
|
827
|
+
values: dict[str, str] = {}
|
|
828
|
+
for key, value in parse_qsl(query if sep else "", keep_blank_values=True):
|
|
829
|
+
if key not in _WILDS_QUERY_PARAMS:
|
|
830
|
+
allowed = ", ".join(sorted(_WILDS_QUERY_PARAMS))
|
|
831
|
+
raise ValueError(
|
|
832
|
+
f"Unsupported WILDS option {key!r}. Supported options: {allowed}."
|
|
833
|
+
)
|
|
834
|
+
if key in values:
|
|
835
|
+
raise ValueError(f"Duplicate WILDS option {key!r}.")
|
|
836
|
+
values[key] = value
|
|
837
|
+
|
|
838
|
+
if name not in _SUPPORTED_WILDS_IMAGE_CLASSIFICATION:
|
|
839
|
+
reason = _UNSUPPORTED_WILDS_DATASETS.get(name)
|
|
840
|
+
if reason is not None:
|
|
841
|
+
raise ValueError(
|
|
842
|
+
f"WILDS dataset {name!r} is a {reason} dataset and is not "
|
|
843
|
+
"supported by justdata's vision classification loader."
|
|
844
|
+
)
|
|
845
|
+
supported = ", ".join(sorted(_SUPPORTED_WILDS_IMAGE_CLASSIFICATION))
|
|
846
|
+
raise ValueError(
|
|
847
|
+
f"Unsupported WILDS dataset {name!r}. Supported image classification "
|
|
848
|
+
f"datasets: {supported}."
|
|
849
|
+
)
|
|
850
|
+
|
|
851
|
+
return _WILDSDatasetSpec(
|
|
852
|
+
name=name,
|
|
853
|
+
split_scheme=values.get("split_scheme", "official"),
|
|
854
|
+
version=values.get("version") or None,
|
|
855
|
+
download=_parse_bool(values.get("download", "false"), key="download"),
|
|
856
|
+
unlabeled=_parse_bool(values.get("unlabeled", "false"), key="unlabeled"),
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def _is_unlabeled_wilds_split(split: str) -> bool:
|
|
861
|
+
return split.endswith("_unlabeled") or split == "extra_unlabeled"
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
def _validate_wilds_splits(splits: list[str], spec: _WILDSDatasetSpec) -> bool:
|
|
865
|
+
has_unlabeled = any(_is_unlabeled_wilds_split(split) for split in splits)
|
|
866
|
+
has_labeled = any(not _is_unlabeled_wilds_split(split) for split in splits)
|
|
867
|
+
if has_labeled and has_unlabeled:
|
|
868
|
+
raise ValueError(
|
|
869
|
+
"WILDS labeled and unlabeled splits must be loaded in separate "
|
|
870
|
+
"load_ds calls."
|
|
871
|
+
)
|
|
872
|
+
if spec.unlabeled and has_labeled:
|
|
873
|
+
raise ValueError(
|
|
874
|
+
"?unlabeled=true can only be used with WILDS unlabeled splits."
|
|
875
|
+
)
|
|
876
|
+
return has_unlabeled
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
def _import_wilds():
|
|
880
|
+
try:
|
|
881
|
+
import wilds
|
|
882
|
+
except ImportError as e:
|
|
883
|
+
raise ImportError(
|
|
884
|
+
"The 'wilds' package is required for wilds: source loading. "
|
|
885
|
+
"Install justdata with the wilds extra."
|
|
886
|
+
) from e
|
|
887
|
+
return wilds
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
def _as_numpy(value: Any) -> np.ndarray:
|
|
891
|
+
if hasattr(value, "detach"):
|
|
892
|
+
value = value.detach().cpu()
|
|
893
|
+
if hasattr(value, "numpy"):
|
|
894
|
+
value = value.numpy()
|
|
895
|
+
return np.asarray(value)
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
def _image_to_uint8_array(image: Any) -> np.ndarray:
|
|
899
|
+
if hasattr(image, "convert"):
|
|
900
|
+
image = image.convert("RGB")
|
|
901
|
+
array = _as_numpy(image)
|
|
902
|
+
if array.ndim == 3 and array.shape[0] in (1, 3) and array.shape[-1] > 4:
|
|
903
|
+
array = np.transpose(array, (1, 2, 0))
|
|
904
|
+
if array.dtype == np.uint8:
|
|
905
|
+
return array
|
|
906
|
+
if np.issubdtype(array.dtype, np.floating) and array.size:
|
|
907
|
+
max_value = float(np.nanmax(array))
|
|
908
|
+
if max_value <= 1.0:
|
|
909
|
+
array = array * 255.0
|
|
910
|
+
return np.clip(array, 0, 255).astype(np.uint8)
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def _label_to_int64(label: Any) -> np.int64:
|
|
914
|
+
label_array = _as_numpy(label)
|
|
915
|
+
if label_array.size != 1:
|
|
916
|
+
raise ValueError("WILDS vision classification labels must be scalar values.")
|
|
917
|
+
return np.int64(label_array.reshape(()))
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
def _metadata_to_int64_dict(
|
|
921
|
+
metadata: Any,
|
|
922
|
+
metadata_fields: tuple[str, ...] | list[str],
|
|
923
|
+
) -> dict[str, np.int64]:
|
|
924
|
+
if isinstance(metadata, dict):
|
|
925
|
+
return {str(key): _label_to_int64(value) for key, value in metadata.items()}
|
|
926
|
+
|
|
927
|
+
metadata_array = _as_numpy(metadata).reshape(-1)
|
|
928
|
+
result = {}
|
|
929
|
+
for i, field in enumerate(metadata_fields):
|
|
930
|
+
if i >= metadata_array.shape[0]:
|
|
931
|
+
break
|
|
932
|
+
result[str(field)] = np.int64(metadata_array[i])
|
|
933
|
+
return result
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def _is_fmow_timestamp_arg(value: Any) -> bool:
|
|
937
|
+
return getattr(value, "name", None) == "timestamp"
|
|
938
|
+
|
|
939
|
+
|
|
940
|
+
@contextmanager
|
|
941
|
+
def _wilds_fmow_datetime_compat(enabled: bool) -> Iterator[None]:
|
|
942
|
+
if not enabled:
|
|
943
|
+
yield
|
|
944
|
+
return
|
|
945
|
+
|
|
946
|
+
try:
|
|
947
|
+
import pandas as pd
|
|
948
|
+
except ImportError:
|
|
949
|
+
yield
|
|
950
|
+
return
|
|
951
|
+
|
|
952
|
+
original_to_datetime = pd.to_datetime
|
|
953
|
+
|
|
954
|
+
def to_datetime_compat(arg, *args, **kwargs):
|
|
955
|
+
if kwargs.get("format") is not None or not _is_fmow_timestamp_arg(arg):
|
|
956
|
+
return original_to_datetime(arg, *args, **kwargs)
|
|
957
|
+
|
|
958
|
+
try:
|
|
959
|
+
return original_to_datetime(arg, *args, **kwargs)
|
|
960
|
+
except ValueError as e:
|
|
961
|
+
if "match format" not in str(e):
|
|
962
|
+
raise
|
|
963
|
+
compat_kwargs = dict(kwargs)
|
|
964
|
+
compat_kwargs["format"] = "ISO8601"
|
|
965
|
+
return original_to_datetime(arg, *args, **compat_kwargs)
|
|
966
|
+
|
|
967
|
+
pd.to_datetime = to_datetime_compat
|
|
968
|
+
try:
|
|
969
|
+
yield
|
|
970
|
+
finally:
|
|
971
|
+
pd.to_datetime = original_to_datetime
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
def _split_to_dataset(
|
|
975
|
+
dataset,
|
|
976
|
+
split: str,
|
|
977
|
+
*,
|
|
978
|
+
spec: _WILDSDatasetSpec,
|
|
979
|
+
include_label: bool,
|
|
980
|
+
) -> tf.data.Dataset:
|
|
981
|
+
subset = dataset.get_subset(split, transform=None)
|
|
982
|
+
metadata_fields = tuple(getattr(subset, "metadata_fields", ()))
|
|
983
|
+
version = str(getattr(dataset, "version", spec.version or ""))
|
|
984
|
+
split_scheme = str(getattr(dataset, "split_scheme", spec.split_scheme))
|
|
985
|
+
indices = getattr(subset, "indices", None)
|
|
986
|
+
|
|
987
|
+
def gen():
|
|
988
|
+
for subset_index, item in enumerate(subset):
|
|
989
|
+
if include_label:
|
|
990
|
+
image, label, metadata = item
|
|
991
|
+
else:
|
|
992
|
+
if len(item) == 2:
|
|
993
|
+
image, metadata = item
|
|
994
|
+
elif len(item) == 3:
|
|
995
|
+
image, _label, metadata = item
|
|
996
|
+
else:
|
|
997
|
+
raise ValueError("WILDS unlabeled samples must be 2- or 3-tuples.")
|
|
998
|
+
|
|
999
|
+
if indices is None:
|
|
1000
|
+
wilds_index = subset_index
|
|
1001
|
+
else:
|
|
1002
|
+
wilds_index = int(_as_numpy(indices[subset_index]).reshape(()))
|
|
1003
|
+
|
|
1004
|
+
sample = {
|
|
1005
|
+
"image": _image_to_uint8_array(image),
|
|
1006
|
+
"metadata": {
|
|
1007
|
+
"dataset": spec.name,
|
|
1008
|
+
"split": split,
|
|
1009
|
+
"example_id": str(wilds_index),
|
|
1010
|
+
"wilds_index": np.int64(wilds_index),
|
|
1011
|
+
"split_scheme": split_scheme,
|
|
1012
|
+
"version": version,
|
|
1013
|
+
"wilds": _metadata_to_int64_dict(metadata, metadata_fields),
|
|
1014
|
+
},
|
|
1015
|
+
}
|
|
1016
|
+
if include_label:
|
|
1017
|
+
sample["label"] = _label_to_int64(label)
|
|
1018
|
+
yield sample
|
|
1019
|
+
|
|
1020
|
+
output_signature = {
|
|
1021
|
+
"image": tf.TensorSpec(shape=None, dtype=tf.uint8),
|
|
1022
|
+
"metadata": {
|
|
1023
|
+
"dataset": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
1024
|
+
"split": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
1025
|
+
"example_id": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
1026
|
+
"wilds_index": tf.TensorSpec(shape=(), dtype=tf.int64),
|
|
1027
|
+
"split_scheme": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
1028
|
+
"version": tf.TensorSpec(shape=(), dtype=tf.string),
|
|
1029
|
+
"wilds": {
|
|
1030
|
+
field: tf.TensorSpec(shape=(), dtype=tf.int64)
|
|
1031
|
+
for field in metadata_fields
|
|
1032
|
+
},
|
|
1033
|
+
},
|
|
1034
|
+
}
|
|
1035
|
+
if include_label:
|
|
1036
|
+
output_signature["label"] = tf.TensorSpec(shape=(), dtype=tf.int64)
|
|
1037
|
+
|
|
1038
|
+
ds_tf = tf.data.Dataset.from_generator(gen, output_signature=output_signature)
|
|
1039
|
+
try:
|
|
1040
|
+
ds_tf = ds_tf.apply(tf.data.experimental.assert_cardinality(len(subset)))
|
|
1041
|
+
except Exception as e:
|
|
1042
|
+
logger.warning(
|
|
1043
|
+
f"Failed to assert cardinality for WILDS dataset {spec.name}: {e}"
|
|
1044
|
+
)
|
|
1045
|
+
return ds_tf
|
|
1046
|
+
|
|
1047
|
+
|
|
1048
|
+
@register_source_loader(_WILDS_PREFIX)
|
|
1049
|
+
def load_wilds_vision_splits(
|
|
1050
|
+
dataset_name: str,
|
|
1051
|
+
splits: list[str],
|
|
1052
|
+
data_dir: Union[None, str, os.PathLike] = None,
|
|
1053
|
+
) -> list[tf.data.Dataset]:
|
|
1054
|
+
spec = _parse_wilds_spec(dataset_name)
|
|
1055
|
+
uses_unlabeled = _validate_wilds_splits(splits, spec)
|
|
1056
|
+
wilds = _import_wilds()
|
|
1057
|
+
|
|
1058
|
+
kwargs = {
|
|
1059
|
+
"dataset": spec.name,
|
|
1060
|
+
"download": spec.download,
|
|
1061
|
+
"split_scheme": spec.split_scheme,
|
|
1062
|
+
}
|
|
1063
|
+
kwargs["root_dir"] = os.fspath(source_cache_dir(data_dir, "wilds"))
|
|
1064
|
+
if spec.version is not None:
|
|
1065
|
+
kwargs["version"] = spec.version
|
|
1066
|
+
if spec.unlabeled or uses_unlabeled:
|
|
1067
|
+
kwargs["unlabeled"] = True
|
|
1068
|
+
|
|
1069
|
+
with _wilds_fmow_datetime_compat(spec.name == "fmow"):
|
|
1070
|
+
dataset = wilds.get_dataset(**kwargs)
|
|
1071
|
+
include_label = not (spec.unlabeled or uses_unlabeled)
|
|
1072
|
+
return [
|
|
1073
|
+
_split_to_dataset(
|
|
1074
|
+
dataset,
|
|
1075
|
+
split,
|
|
1076
|
+
spec=spec,
|
|
1077
|
+
include_label=include_label,
|
|
1078
|
+
)
|
|
1079
|
+
for split in splits
|
|
1080
|
+
]
|