maite-datasets 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- maite_datasets/__init__.py +1 -0
- maite_datasets/_base.py +254 -0
- maite_datasets/_fileio.py +174 -0
- maite_datasets/_mixin/__init__.py +0 -0
- maite_datasets/_mixin/_numpy.py +28 -0
- maite_datasets/_mixin/_torch.py +28 -0
- maite_datasets/_protocols.py +224 -0
- maite_datasets/_types.py +54 -0
- maite_datasets/image_classification/__init__.py +11 -0
- maite_datasets/image_classification/_cifar10.py +233 -0
- maite_datasets/image_classification/_mnist.py +215 -0
- maite_datasets/image_classification/_ships.py +150 -0
- maite_datasets/object_detection/__init__.py +20 -0
- maite_datasets/object_detection/_antiuav.py +200 -0
- maite_datasets/object_detection/_milco.py +207 -0
- maite_datasets/object_detection/_seadrone.py +551 -0
- maite_datasets/object_detection/_voc.py +510 -0
- maite_datasets/object_detection/_voc_torch.py +65 -0
- maite_datasets/py.typed +0 -0
- maite_datasets-0.0.1.dist-info/METADATA +91 -0
- maite_datasets-0.0.1.dist-info/RECORD +23 -0
- maite_datasets-0.0.1.dist-info/WHEEL +4 -0
- maite_datasets-0.0.1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,510 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
__all__ = []
|
4
|
+
|
5
|
+
import os
|
6
|
+
import shutil
|
7
|
+
from pathlib import Path
|
8
|
+
from typing import Any, Literal, Sequence, TypeVar
|
9
|
+
|
10
|
+
import numpy as np
|
11
|
+
from defusedxml.ElementTree import parse
|
12
|
+
from numpy.typing import NDArray
|
13
|
+
|
14
|
+
from maite_datasets._base import (
|
15
|
+
BaseDataset,
|
16
|
+
BaseODDataset,
|
17
|
+
DataLocation,
|
18
|
+
_ensure_exists,
|
19
|
+
_TArray,
|
20
|
+
_TTarget,
|
21
|
+
)
|
22
|
+
from maite_datasets._mixin._numpy import BaseDatasetNumpyMixin
|
23
|
+
from maite_datasets._protocols import Transform
|
24
|
+
from maite_datasets._types import ObjectDetectionTarget
|
25
|
+
|
26
|
+
VOCClassStringMap = Literal[
|
27
|
+
"aeroplane",
|
28
|
+
"bicycle",
|
29
|
+
"bird",
|
30
|
+
"boat",
|
31
|
+
"bottle",
|
32
|
+
"bus",
|
33
|
+
"car",
|
34
|
+
"cat",
|
35
|
+
"chair",
|
36
|
+
"cow",
|
37
|
+
"diningtable",
|
38
|
+
"dog",
|
39
|
+
"horse",
|
40
|
+
"motorbike",
|
41
|
+
"person",
|
42
|
+
"pottedplant",
|
43
|
+
"sheep",
|
44
|
+
"sofa",
|
45
|
+
"train",
|
46
|
+
"tvmonitor",
|
47
|
+
]
|
48
|
+
TVOCClassMap = TypeVar(
|
49
|
+
"TVOCClassMap", VOCClassStringMap, int, list[VOCClassStringMap], list[int]
|
50
|
+
)
|
51
|
+
|
52
|
+
|
53
|
+
class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
|
54
|
+
_resources = [
|
55
|
+
DataLocation(
|
56
|
+
url="https://data.brainchip.com/dataset-mirror/voc/VOCtrainval_11-May-2012.tar",
|
57
|
+
filename="VOCtrainval_11-May-2012.tar",
|
58
|
+
md5=False,
|
59
|
+
checksum="e14f763270cf193d0b5f74b169f44157a4b0c6efa708f4dd0ff78ee691763bcb",
|
60
|
+
),
|
61
|
+
DataLocation(
|
62
|
+
url="http://host.robots.ox.ac.uk/pascal/VOC/voc2011/VOCtrainval_25-May-2011.tar",
|
63
|
+
filename="VOCtrainval_25-May-2011.tar",
|
64
|
+
md5=False,
|
65
|
+
checksum="0a7f5f5d154f7290ec65ec3f78b72ef72c6d93ff6d79acd40dc222a9ee5248ba",
|
66
|
+
),
|
67
|
+
DataLocation(
|
68
|
+
url="http://host.robots.ox.ac.uk/pascal/VOC/voc2010/VOCtrainval_03-May-2010.tar",
|
69
|
+
filename="VOCtrainval_03-May-2010.tar",
|
70
|
+
md5=False,
|
71
|
+
checksum="1af4189cbe44323ab212bff7afbc7d0f55a267cc191eb3aac911037887e5c7d4",
|
72
|
+
),
|
73
|
+
DataLocation(
|
74
|
+
url="http://host.robots.ox.ac.uk/pascal/VOC/voc2009/VOCtrainval_11-May-2009.tar",
|
75
|
+
filename="VOCtrainval_11-May-2009.tar",
|
76
|
+
md5=False,
|
77
|
+
checksum="11cbe1741fb5bdadbbca3c08e9ec62cd95c14884845527d50847bc2cf57e7fd6",
|
78
|
+
),
|
79
|
+
DataLocation(
|
80
|
+
url="http://host.robots.ox.ac.uk/pascal/VOC/voc2008/VOCtrainval_14-Jul-2008.tar",
|
81
|
+
filename="VOCtrainval_14-Jul-2008.tar",
|
82
|
+
md5=False,
|
83
|
+
checksum="7f0ca53c1b5a838fbe946965fc106c6e86832183240af5c88e3f6c306318d42e",
|
84
|
+
),
|
85
|
+
DataLocation(
|
86
|
+
url="https://data.brainchip.com/dataset-mirror/voc/VOCtrainval_06-Nov-2007.tar",
|
87
|
+
filename="VOCtrainval_06-Nov-2007.tar",
|
88
|
+
md5=False,
|
89
|
+
checksum="7d8cd951101b0957ddfd7a530bdc8a94f06121cfc1e511bb5937e973020c7508",
|
90
|
+
),
|
91
|
+
DataLocation(
|
92
|
+
url="https://data.brainchip.com/dataset-mirror/voc/VOC2012test.tar",
|
93
|
+
filename="VOC2012test.tar",
|
94
|
+
md5=False,
|
95
|
+
checksum="f08582b1935816c5eab3bbb1eb6d06201a789eaa173cdf1cf400c26f0cac2fb3",
|
96
|
+
),
|
97
|
+
DataLocation(
|
98
|
+
url="https://data.brainchip.com/dataset-mirror/voc/VOCtest_06-Nov-2007.tar",
|
99
|
+
filename="VOCtest_06-Nov-2007.tar",
|
100
|
+
md5=False,
|
101
|
+
checksum="6836888e2e01dca84577a849d339fa4f73e1e4f135d312430c4856b5609b4892",
|
102
|
+
),
|
103
|
+
]
|
104
|
+
_base2007: tuple[int, int] = (5, 7)
|
105
|
+
_base2012: tuple[int, int] = (0, 6)
|
106
|
+
|
107
|
+
index2label: dict[int, str] = {
|
108
|
+
0: "aeroplane",
|
109
|
+
1: "bicycle",
|
110
|
+
2: "bird",
|
111
|
+
3: "boat",
|
112
|
+
4: "bottle",
|
113
|
+
5: "bus",
|
114
|
+
6: "car",
|
115
|
+
7: "cat",
|
116
|
+
8: "chair",
|
117
|
+
9: "cow",
|
118
|
+
10: "diningtable",
|
119
|
+
11: "dog",
|
120
|
+
12: "horse",
|
121
|
+
13: "motorbike",
|
122
|
+
14: "person",
|
123
|
+
15: "pottedplant",
|
124
|
+
16: "sheep",
|
125
|
+
17: "sofa",
|
126
|
+
18: "train",
|
127
|
+
19: "tvmonitor",
|
128
|
+
}
|
129
|
+
|
130
|
+
def __init__(
|
131
|
+
self,
|
132
|
+
root: str | Path,
|
133
|
+
image_set: Literal["train", "val", "test", "base"] = "train",
|
134
|
+
year: Literal["2007", "2008", "2009", "2010", "2011", "2012"] = "2012",
|
135
|
+
transforms: Transform[_TArray] | Sequence[Transform[_TArray]] | None = None,
|
136
|
+
download: bool = False,
|
137
|
+
verbose: bool = False,
|
138
|
+
) -> None:
|
139
|
+
self.year = year
|
140
|
+
self._resource_index = self._get_year_image_set_index(year, image_set)
|
141
|
+
super().__init__(
|
142
|
+
root,
|
143
|
+
image_set,
|
144
|
+
transforms,
|
145
|
+
download,
|
146
|
+
verbose,
|
147
|
+
)
|
148
|
+
|
149
|
+
def _get_dataset_dir(self) -> Path:
|
150
|
+
"""Overrides the base function to determine correct dataset directory for VOC class"""
|
151
|
+
return self._find_main_VOC_dir(self._root)
|
152
|
+
|
153
|
+
def _find_main_VOC_dir(self, base: Path) -> Path:
|
154
|
+
"""
|
155
|
+
Determine the correct dataset directory for VOC detection and segmentation classes.
|
156
|
+
Handles various directory structure possibilities and validates existence.
|
157
|
+
"""
|
158
|
+
|
159
|
+
# VOCdataset directory possibilities
|
160
|
+
dataset_dir = base if base.stem.lower() == "vocdataset" else base / "vocdataset"
|
161
|
+
|
162
|
+
# Define possible directory structures based on patterns
|
163
|
+
# 1. Root is already the specific VOC year directory
|
164
|
+
# 2. Root is the VOCdevkit directory
|
165
|
+
# 3. Standard structure
|
166
|
+
# 4. Special case for year 2011
|
167
|
+
# 5. Within VOCdataset directory
|
168
|
+
# 6. Special case for year 2011 within VOCdataset
|
169
|
+
possible_paths = [
|
170
|
+
base if base.stem == f"VOC{self.year}" else None,
|
171
|
+
base / f"VOC{self.year}" if base.stem == "VOCdevkit" else None,
|
172
|
+
base / "VOCdevkit" / f"VOC{self.year}",
|
173
|
+
base / "TrainVal" / "VOCdevkit" / f"VOC{self.year}"
|
174
|
+
if self.year == "2011"
|
175
|
+
else None,
|
176
|
+
dataset_dir / "VOCdevkit" / f"VOC{self.year}",
|
177
|
+
dataset_dir / "TrainVal" / "VOCdevkit" / f"VOC{self.year}"
|
178
|
+
if self.year == "2011"
|
179
|
+
else None,
|
180
|
+
]
|
181
|
+
|
182
|
+
# Filter out None values and check each path
|
183
|
+
for path in filter(None, possible_paths):
|
184
|
+
if path.exists():
|
185
|
+
return path
|
186
|
+
|
187
|
+
# If no existing path is found, create and return the dataset directory
|
188
|
+
if not dataset_dir.exists():
|
189
|
+
dataset_dir.mkdir(parents=True, exist_ok=True)
|
190
|
+
|
191
|
+
return dataset_dir
|
192
|
+
|
193
|
+
def _get_year_image_set_index(self, year: str, image_set: str) -> int:
|
194
|
+
"""Function to ensure that the correct resource file is accessed"""
|
195
|
+
if year == "2007" and image_set == "test":
|
196
|
+
return -1
|
197
|
+
if year == "2012" and image_set == "test":
|
198
|
+
return -2
|
199
|
+
if year != "2007" and image_set == "test":
|
200
|
+
raise ValueError(
|
201
|
+
f"The only test sets available are for the years 2007 and 2012, not {year}. "
|
202
|
+
"Either select the year 2007 or 2012, or use a different image_set."
|
203
|
+
)
|
204
|
+
return 2012 - int(year)
|
205
|
+
|
206
|
+
def _update_path(self) -> None:
|
207
|
+
"""Update the path to the new folder structure"""
|
208
|
+
if self.year == "2011" and self.path.stem.lower() == "vocdataset":
|
209
|
+
self.path: Path = self.path / "TrainVal" / "VOCdevkit" / f"VOC{self.year}"
|
210
|
+
elif self.path.stem.lower() == "vocdataset":
|
211
|
+
self.path: Path = self.path / "VOCdevkit" / f"VOC{self.year}"
|
212
|
+
|
213
|
+
def _load_data_exception(self) -> tuple[list[str], list[str], dict[str, Any]]:
|
214
|
+
"""Adjust how the directory is created for the 2007 and 2012 test set"""
|
215
|
+
filepaths: list[str] = []
|
216
|
+
targets: list[str] = []
|
217
|
+
datum_metadata: dict[str, list[Any]] = {}
|
218
|
+
tmp_path: Path = self._root / "tmp_directory_for_download"
|
219
|
+
tmp_path.mkdir(exist_ok=True)
|
220
|
+
resource_idx = self._base2007 if self.year == "2007" else self._base2012
|
221
|
+
|
222
|
+
# Determine if text files exist
|
223
|
+
train_file = self.path / "ImageSets" / "Main" / "trainval.txt"
|
224
|
+
test_file = self.path / "ImageSets" / "Main" / "test.txt"
|
225
|
+
train_exists = train_file.exists()
|
226
|
+
test_exists = test_file.exists()
|
227
|
+
|
228
|
+
if self.image_set == "base":
|
229
|
+
if not train_exists and not test_exists:
|
230
|
+
_ensure_exists(
|
231
|
+
*self._resources[resource_idx[0]],
|
232
|
+
self.path,
|
233
|
+
self._root,
|
234
|
+
self._download,
|
235
|
+
self._verbose,
|
236
|
+
)
|
237
|
+
self._update_path()
|
238
|
+
_ensure_exists(
|
239
|
+
*self._resources[resource_idx[1]],
|
240
|
+
tmp_path,
|
241
|
+
self._root,
|
242
|
+
self._download,
|
243
|
+
self._verbose,
|
244
|
+
)
|
245
|
+
self._merge_voc_directories(tmp_path)
|
246
|
+
|
247
|
+
elif train_exists and not test_exists:
|
248
|
+
_ensure_exists(
|
249
|
+
*self._resources[resource_idx[1]],
|
250
|
+
tmp_path,
|
251
|
+
self._root,
|
252
|
+
self._download,
|
253
|
+
self._verbose,
|
254
|
+
)
|
255
|
+
self._merge_voc_directories(tmp_path)
|
256
|
+
|
257
|
+
elif not train_exists and test_exists:
|
258
|
+
_ensure_exists(
|
259
|
+
*self._resources[resource_idx[0]],
|
260
|
+
tmp_path,
|
261
|
+
self._root,
|
262
|
+
self._download,
|
263
|
+
self._verbose,
|
264
|
+
)
|
265
|
+
self._merge_voc_directories(tmp_path)
|
266
|
+
|
267
|
+
# Code to determine what is needed in each category
|
268
|
+
metadata_list: list[dict[str, Any]] = []
|
269
|
+
|
270
|
+
for img_set in ["test", "base"]:
|
271
|
+
self.image_set = img_set
|
272
|
+
resource_filepaths, resource_targets, resource_metadata = (
|
273
|
+
self._load_data_inner()
|
274
|
+
)
|
275
|
+
filepaths.extend(resource_filepaths)
|
276
|
+
targets.extend(resource_targets)
|
277
|
+
metadata_list.append(resource_metadata)
|
278
|
+
|
279
|
+
# Combine metadata from all resources
|
280
|
+
for data_dict in metadata_list:
|
281
|
+
for key, val in data_dict.items():
|
282
|
+
str_key = str(key) # Ensure key is string
|
283
|
+
if str_key not in datum_metadata:
|
284
|
+
datum_metadata[str_key] = []
|
285
|
+
datum_metadata[str_key].extend(val)
|
286
|
+
|
287
|
+
else:
|
288
|
+
self._resource = self._resources[resource_idx[1]]
|
289
|
+
|
290
|
+
if train_exists and not test_exists:
|
291
|
+
_ensure_exists(
|
292
|
+
*self._resource, tmp_path, self._root, self._download, self._verbose
|
293
|
+
)
|
294
|
+
self._merge_voc_directories(tmp_path)
|
295
|
+
|
296
|
+
resource_filepaths, resource_targets, resource_metadata = (
|
297
|
+
self._load_try_and_update()
|
298
|
+
)
|
299
|
+
filepaths.extend(resource_filepaths)
|
300
|
+
targets.extend(resource_targets)
|
301
|
+
datum_metadata.update(resource_metadata)
|
302
|
+
|
303
|
+
return filepaths, targets, datum_metadata
|
304
|
+
|
305
|
+
def _merge_voc_directories(self, source_dir: Path) -> None:
|
306
|
+
"""Merge two VOC directories, handling file conflicts intelligently."""
|
307
|
+
base: Path = self._find_main_VOC_dir(source_dir)
|
308
|
+
# Create all subdirectories in target if they don't exist
|
309
|
+
for dirpath, dirnames, filenames in os.walk(base):
|
310
|
+
# Convert to Path objects
|
311
|
+
source_path = Path(dirpath)
|
312
|
+
|
313
|
+
# Get the relative path from source_dir
|
314
|
+
rel_path = source_path.relative_to(base)
|
315
|
+
|
316
|
+
# Create the corresponding target path
|
317
|
+
target_path = self.path / rel_path
|
318
|
+
target_path.mkdir(parents=True, exist_ok=True)
|
319
|
+
|
320
|
+
# Copy all files
|
321
|
+
for filename in filenames:
|
322
|
+
source_file = source_path / filename
|
323
|
+
target_file = target_path / filename
|
324
|
+
|
325
|
+
# File doesn't exist in target, just move it
|
326
|
+
if not target_file.exists():
|
327
|
+
shutil.move(source_file, target_file)
|
328
|
+
else:
|
329
|
+
# File exists in both assume they're identical and skip
|
330
|
+
pass
|
331
|
+
|
332
|
+
shutil.rmtree(source_dir)
|
333
|
+
|
334
|
+
def _load_try_and_update(self) -> tuple[list[str], list[str], dict[str, Any]]:
|
335
|
+
"""Test if data needs to be downloaded and update path if it does"""
|
336
|
+
if self._verbose:
|
337
|
+
print(f"Determining if {self._resource.filename} needs to be downloaded.")
|
338
|
+
|
339
|
+
try:
|
340
|
+
result = self._load_data_inner()
|
341
|
+
if self._verbose:
|
342
|
+
print("No download needed, loaded data successfully.")
|
343
|
+
except FileNotFoundError:
|
344
|
+
_ensure_exists(
|
345
|
+
*self._resource, self.path, self._root, self._download, self._verbose
|
346
|
+
)
|
347
|
+
self._update_path()
|
348
|
+
result = self._load_data_inner()
|
349
|
+
return result
|
350
|
+
|
351
|
+
def _load_data(self) -> tuple[list[str], list[str], dict[str, Any]]:
|
352
|
+
"""
|
353
|
+
Function to determine if data can be accessed or if it needs to be downloaded and/or extracted.
|
354
|
+
"""
|
355
|
+
# Exception - test sets
|
356
|
+
year_set_bool = (self.image_set == "test" or self.image_set == "base") and (
|
357
|
+
self.year == "2012" or self.year == "2007"
|
358
|
+
)
|
359
|
+
if year_set_bool:
|
360
|
+
return self._load_data_exception()
|
361
|
+
|
362
|
+
return self._load_try_and_update()
|
363
|
+
|
364
|
+
def _get_image_sets(self) -> dict[str, list[str]]:
|
365
|
+
"""Function to create the list of images in each image set"""
|
366
|
+
image_folder = self.path / "JPEGImages"
|
367
|
+
image_set_list = (
|
368
|
+
["train", "val", "trainval"] if self.image_set != "test" else ["test"]
|
369
|
+
)
|
370
|
+
image_sets = {}
|
371
|
+
for image_set in image_set_list:
|
372
|
+
text_file = self.path / "ImageSets" / "Main" / (image_set + ".txt")
|
373
|
+
selected_images: list[str] = []
|
374
|
+
with open(text_file) as f:
|
375
|
+
for line in f.readlines():
|
376
|
+
out = line.strip()
|
377
|
+
selected_images.append(str(image_folder / (out + ".jpg")))
|
378
|
+
|
379
|
+
name = "base" if image_set == "trainval" else image_set
|
380
|
+
image_sets[name] = selected_images
|
381
|
+
return image_sets
|
382
|
+
|
383
|
+
def _load_data_inner(self) -> tuple[list[str], list[str], dict[str, Any]]:
|
384
|
+
"""Function to load in the file paths for the data, annotations and segmentation masks"""
|
385
|
+
file_meta = {"year": [], "image_id": [], "mask_path": []}
|
386
|
+
ann_folder = self.path / "Annotations"
|
387
|
+
seg_folder = self.path / "SegmentationClass"
|
388
|
+
|
389
|
+
# Load in the image sets
|
390
|
+
image_sets = self._get_image_sets()
|
391
|
+
|
392
|
+
# Get the data, annotations and metadata
|
393
|
+
annotations = []
|
394
|
+
data = image_sets[self.image_set]
|
395
|
+
for entry in data:
|
396
|
+
file_name = Path(entry).name
|
397
|
+
file_stem = Path(entry).stem
|
398
|
+
if self.year != "2007":
|
399
|
+
# Remove file extension and split by "_"
|
400
|
+
parts = file_stem.split("_")
|
401
|
+
file_meta["year"].append(parts[0])
|
402
|
+
file_meta["image_id"].append(parts[1])
|
403
|
+
else:
|
404
|
+
file_meta["year"].append(self.year)
|
405
|
+
file_meta["image_id"].append(file_stem)
|
406
|
+
file_meta["mask_path"].append(str(seg_folder / file_name))
|
407
|
+
annotations.append(str(ann_folder / file_stem) + ".xml")
|
408
|
+
|
409
|
+
return data, annotations, file_meta
|
410
|
+
|
411
|
+
def _read_annotations(
|
412
|
+
self, annotation: str
|
413
|
+
) -> tuple[list[list[float]], list[int], dict[str, Any]]:
|
414
|
+
boxes: list[list[float]] = []
|
415
|
+
label_str = []
|
416
|
+
if not Path(annotation).exists():
|
417
|
+
return boxes, label_str, {}
|
418
|
+
root = parse(annotation).getroot()
|
419
|
+
if root is None:
|
420
|
+
raise ValueError(f"Unable to parse {annotation}")
|
421
|
+
additional_meta: dict[str, Any] = {
|
422
|
+
"folder": root.findtext("folder", default=""),
|
423
|
+
"filename": root.findtext("filename", default=""),
|
424
|
+
"database": root.findtext("source/database", default=""),
|
425
|
+
"annotation_source": root.findtext("source/annotation", default=""),
|
426
|
+
"image_source": root.findtext("source/image", default=""),
|
427
|
+
"image_width": int(root.findtext("size/width", default="-1")),
|
428
|
+
"image_height": int(root.findtext("size/height", default="-1")),
|
429
|
+
"image_depth": int(root.findtext("size/depth", default="-1")),
|
430
|
+
"segmented": int(root.findtext("segmented", default="-1")),
|
431
|
+
"pose": [],
|
432
|
+
"truncated": [],
|
433
|
+
"difficult": [],
|
434
|
+
}
|
435
|
+
for obj in root.findall("object"):
|
436
|
+
label_str.append(obj.findtext("name", default=""))
|
437
|
+
additional_meta["pose"].append(obj.findtext("pose", default=""))
|
438
|
+
additional_meta["truncated"].append(
|
439
|
+
int(obj.findtext("truncated", default="-1"))
|
440
|
+
)
|
441
|
+
additional_meta["difficult"].append(
|
442
|
+
int(obj.findtext("difficult", default="-1"))
|
443
|
+
)
|
444
|
+
boxes.append(
|
445
|
+
[
|
446
|
+
float(obj.findtext("bndbox/xmin", default="0")),
|
447
|
+
float(obj.findtext("bndbox/ymin", default="0")),
|
448
|
+
float(obj.findtext("bndbox/xmax", default="0")),
|
449
|
+
float(obj.findtext("bndbox/ymax", default="0")),
|
450
|
+
]
|
451
|
+
)
|
452
|
+
labels = [self.label2index[label] for label in label_str]
|
453
|
+
return boxes, labels, additional_meta
|
454
|
+
|
455
|
+
|
456
|
+
class VOCDetection(
|
457
|
+
BaseVOCDataset[
|
458
|
+
NDArray[np.number[Any]], ObjectDetectionTarget[NDArray[np.number[Any]]]
|
459
|
+
],
|
460
|
+
BaseODDataset[NDArray[np.number[Any]], list[str], str],
|
461
|
+
BaseDatasetNumpyMixin,
|
462
|
+
):
|
463
|
+
"""
|
464
|
+
`Pascal VOC <http://host.robots.ox.ac.uk/pascal/VOC/>`_ Detection Dataset.
|
465
|
+
|
466
|
+
Parameters
|
467
|
+
----------
|
468
|
+
root : str or pathlib.Path
|
469
|
+
Because of the structure of the PASCAL VOC datasets, the root needs to be one of 4 folders.
|
470
|
+
1) Directory containing the year of the **already downloaded** dataset (i.e. .../VOCdevkit/VOC2012 <-)
|
471
|
+
2) Directory to the VOCdevkit folder of the **already downloaded** dataset (i.e. .../VOCdevkit <- /VOC2012)
|
472
|
+
3) Directory to the folder one level up from the VOCdevkit folder,
|
473
|
+
data **may** or **may not** be already downloaded (i.e. ... <- /VOCdevkit/VOC2012)
|
474
|
+
4) Directory to where you would like the dataset to be downloaded
|
475
|
+
image_set : "train", "val", "test", or "base", default "train"
|
476
|
+
If "test", then dataset year must be "2007" or "2012". Note that the 2012 test set does not contain annotations.
|
477
|
+
If "base", then the combined dataset of "train" and "val" is returned.
|
478
|
+
year : "2007", "2008", "2009", "2010", "2011" or "2012", default "2012"
|
479
|
+
The dataset year.
|
480
|
+
transforms : Transform, Sequence[Transform] or None, default None
|
481
|
+
Transform(s) to apply to the data.
|
482
|
+
download : bool, default False
|
483
|
+
If True, downloads the dataset from the internet and puts it in root directory.
|
484
|
+
Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
|
485
|
+
verbose : bool, default False
|
486
|
+
If True, outputs print statements.
|
487
|
+
|
488
|
+
Attributes
|
489
|
+
----------
|
490
|
+
path : pathlib.Path
|
491
|
+
Location of the folder containing the data.
|
492
|
+
year : "2007", "2008", "2009", "2010", "2011" or "2012"
|
493
|
+
The selected dataset year.
|
494
|
+
image_set : "train", "val", "test" or "base"
|
495
|
+
The selected image set from the dataset.
|
496
|
+
index2label : dict[int, str]
|
497
|
+
Dictionary which translates from class integers to the associated class strings.
|
498
|
+
label2index : dict[str, int]
|
499
|
+
Dictionary which translates from class strings to the associated class integers.
|
500
|
+
metadata : DatasetMetadata
|
501
|
+
Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
|
502
|
+
transforms : Sequence[Transform]
|
503
|
+
The transforms to be applied to the data.
|
504
|
+
size : int
|
505
|
+
The size of the dataset.
|
506
|
+
|
507
|
+
Note
|
508
|
+
----
|
509
|
+
Data License: `Flickr Terms of Use <http://www.flickr.com/terms.gne?legacy=1>`_
|
510
|
+
"""
|
@@ -0,0 +1,65 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
__all__ = []
|
4
|
+
|
5
|
+
from torch import Tensor
|
6
|
+
|
7
|
+
from maite_datasets._base import BaseODDataset
|
8
|
+
from maite_datasets._types import ObjectDetectionTarget
|
9
|
+
from maite_datasets._mixin._torch import BaseDatasetTorchMixin
|
10
|
+
from maite_datasets.object_detection._voc import BaseVOCDataset
|
11
|
+
|
12
|
+
|
13
|
+
class VOCDetectionTorch(
|
14
|
+
BaseVOCDataset[Tensor, ObjectDetectionTarget[Tensor]],
|
15
|
+
BaseODDataset[Tensor, list[str], str],
|
16
|
+
BaseDatasetTorchMixin,
|
17
|
+
):
|
18
|
+
"""
|
19
|
+
`Pascal VOC <http://host.robots.ox.ac.uk/pascal/VOC/>`_ Detection Dataset as PyTorch tensors.
|
20
|
+
|
21
|
+
Parameters
|
22
|
+
----------
|
23
|
+
root : str or pathlib.Path
|
24
|
+
Because of the structure of the PASCAL VOC datasets, the root needs to be one of 4 folders.
|
25
|
+
1) Directory containing the year of the **already downloaded** dataset (i.e. .../VOCdevkit/VOC2012 <-)
|
26
|
+
2) Directory to the VOCdevkit folder of the **already downloaded** dataset (i.e. .../VOCdevkit <- /VOC2012)
|
27
|
+
3) Directory to the folder one level up from the VOCdevkit folder,
|
28
|
+
data **may** or **may not** be already downloaded (i.e. ... <- /VOCdevkit/VOC2012)
|
29
|
+
4) Directory to where you would like the dataset to be downloaded
|
30
|
+
image_set : "train", "val", "test", or "base", default "train"
|
31
|
+
If "test", then dataset year must be "2007" or "2012". Note that the 2012 test set does not contain annotations.
|
32
|
+
If "base", then the combined dataset of "train" and "val" is returned.
|
33
|
+
year : "2007", "2008", "2009", "2010", "2011" or "2012", default "2012"
|
34
|
+
The dataset year.
|
35
|
+
transforms : Transform, Sequence[Transform] or None, default None
|
36
|
+
Transform(s) to apply to the data.
|
37
|
+
download : bool, default False
|
38
|
+
If True, downloads the dataset from the internet and puts it in root directory.
|
39
|
+
Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
|
40
|
+
verbose : bool, default False
|
41
|
+
If True, outputs print statements.
|
42
|
+
|
43
|
+
Attributes
|
44
|
+
----------
|
45
|
+
path : pathlib.Path
|
46
|
+
Location of the folder containing the data.
|
47
|
+
year : "2007", "2008", "2009", "2010", "2011" or "2012"
|
48
|
+
The selected dataset year.
|
49
|
+
image_set : "train", "val", "test" or "base"
|
50
|
+
The selected image set from the dataset.
|
51
|
+
index2label : dict[int, str]
|
52
|
+
Dictionary which translates from class integers to the associated class strings.
|
53
|
+
label2index : dict[str, int]
|
54
|
+
Dictionary which translates from class strings to the associated class integers.
|
55
|
+
metadata : DatasetMetadata
|
56
|
+
Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
|
57
|
+
transforms : Sequence[Transform]
|
58
|
+
The transforms to be applied to the data.
|
59
|
+
size : int
|
60
|
+
The size of the dataset.
|
61
|
+
|
62
|
+
Note
|
63
|
+
----
|
64
|
+
Data License: `Flickr Terms of Use <http://www.flickr.com/terms.gne?legacy=1>`_
|
65
|
+
"""
|
maite_datasets/py.typed
ADDED
File without changes
|
@@ -0,0 +1,91 @@
|
|
1
|
+
Metadata-Version: 2.4
|
2
|
+
Name: maite-datasets
|
3
|
+
Version: 0.0.1
|
4
|
+
Summary: A collection of Image Classification and Object Detection task datasets conforming to the MAITE protocol.
|
5
|
+
Author-email: Andrew Weng <andrew.weng@ariacoustics.com>, Ryan Wood <ryan.wood@ariacoustics.com>, Shaun Jullens <shaun.jullens@ariacoustics.com>
|
6
|
+
License-Expression: MIT
|
7
|
+
License-File: LICENSE
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
9
|
+
Classifier: Framework :: Pytest
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
11
|
+
Classifier: Operating System :: OS Independent
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
17
|
+
Requires-Python: >=3.9
|
18
|
+
Requires-Dist: defusedxml>=0.7.1
|
19
|
+
Requires-Dist: numpy>=1.24.2
|
20
|
+
Requires-Dist: pillow>=10.3.0
|
21
|
+
Requires-Dist: requests>=2.32.3
|
22
|
+
Requires-Dist: typing-extensions>=4.12
|
23
|
+
Provides-Extra: tqdm
|
24
|
+
Requires-Dist: tqdm>=4.66; extra == 'tqdm'
|
25
|
+
Description-Content-Type: text/markdown
|
26
|
+
|
27
|
+
# MAITE Datasets
|
28
|
+
|
29
|
+
MAITE Datasets are a collection of public datasets wrapped in a [MAITE](https://mit-ll-ai-technology.github.io/maite/) compliant format.
|
30
|
+
|
31
|
+
## Installation
|
32
|
+
|
33
|
+
To install and use `maite-datasets` you can use pip:
|
34
|
+
|
35
|
+
```bash
|
36
|
+
pip install maite-datasets
|
37
|
+
```
|
38
|
+
|
39
|
+
For status bar indicators when downloading, you can include the extra `tqdm` when installing:
|
40
|
+
|
41
|
+
```bash
|
42
|
+
pip install maite-datasets[tqdm]
|
43
|
+
```
|
44
|
+
|
45
|
+
## Available Datasets
|
46
|
+
|
47
|
+
| Task | Dataset | Description |
|
48
|
+
|----------------|------------------|---------------------------------------------------------------------|
|
49
|
+
| Classification | CIFAR10 | [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) dataset. |
|
50
|
+
| Classification | MNIST | A dataset of hand-written digits. |
|
51
|
+
| Classification | Ships | A dataset that focuses on identifying ships from satellite images. |
|
52
|
+
| Detection | AntiUAVDetection | A UAV detection dataset in natural images with varying backgrounds. |
|
53
|
+
| Detection | MILCO | A side-scan sonar dataset focused on mine-like object detection. |
|
54
|
+
| Detection | Seadrone | A UAV dataset focused on open water object detection. |
|
55
|
+
| Detection | VOCDetection | [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) dataset. |
|
56
|
+
|
57
|
+
## Usage
|
58
|
+
|
59
|
+
Here is an example of how to import MNIST for usage with your workflow.
|
60
|
+
|
61
|
+
```python
|
62
|
+
>>> from maite_datasets.image_classification import MNIST
|
63
|
+
|
64
|
+
>>> mnist = MNIST(root="data", download=True)
|
65
|
+
>>> print(mnist)
|
66
|
+
MNIST Dataset
|
67
|
+
-------------
|
68
|
+
Corruption: None
|
69
|
+
Transforms: []
|
70
|
+
Image_set: train
|
71
|
+
Metadata: {'id': 'MNIST_train', 'index2label': {0: 'zero', 1: 'one', 2: 'two', 3: 'three', 4: 'four', 5: 'five', 6: 'six', 7: 'seven', 8: 'eight', 9: 'nine'}, 'split': 'train'}
|
72
|
+
Path: /home/user/maite-datasets/data/mnist
|
73
|
+
Size: 60000
|
74
|
+
|
75
|
+
>>> print("tuple("+", ".join([str(type(t)) for t in mnist[0]])+")")
|
76
|
+
tuple(<class 'numpy.ndarray'>, <class 'numpy.ndarray'>, <class 'dict'>)
|
77
|
+
```
|
78
|
+
|
79
|
+
## Additional Information
|
80
|
+
|
81
|
+
For more information on the MAITE protocol, check out their [documentation](https://mit-ll-ai-technology.github.io/maite/).
|
82
|
+
|
83
|
+
## Acknowledgement
|
84
|
+
|
85
|
+
### CDAO Funding Acknowledgement
|
86
|
+
|
87
|
+
This material is based upon work supported by the Chief Digital and Artificial
|
88
|
+
Intelligence Office under Contract No. W519TC-23-9-2033. The views and
|
89
|
+
conclusions contained herein are those of the author(s) and should not be
|
90
|
+
interpreted as necessarily representing the official policies or endorsements,
|
91
|
+
either expressed or implied, of the U.S. Government.
|