maite-datasets 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,510 @@
1
+ from __future__ import annotations
2
+
3
+ __all__ = []
4
+
5
+ import os
6
+ import shutil
7
+ from pathlib import Path
8
+ from typing import Any, Literal, Sequence, TypeVar
9
+
10
+ import numpy as np
11
+ from defusedxml.ElementTree import parse
12
+ from numpy.typing import NDArray
13
+
14
+ from maite_datasets._base import (
15
+ BaseDataset,
16
+ BaseODDataset,
17
+ DataLocation,
18
+ _ensure_exists,
19
+ _TArray,
20
+ _TTarget,
21
+ )
22
+ from maite_datasets._mixin._numpy import BaseDatasetNumpyMixin
23
+ from maite_datasets._protocols import Transform
24
+ from maite_datasets._types import ObjectDetectionTarget
25
+
26
+ VOCClassStringMap = Literal[
27
+ "aeroplane",
28
+ "bicycle",
29
+ "bird",
30
+ "boat",
31
+ "bottle",
32
+ "bus",
33
+ "car",
34
+ "cat",
35
+ "chair",
36
+ "cow",
37
+ "diningtable",
38
+ "dog",
39
+ "horse",
40
+ "motorbike",
41
+ "person",
42
+ "pottedplant",
43
+ "sheep",
44
+ "sofa",
45
+ "train",
46
+ "tvmonitor",
47
+ ]
48
+ TVOCClassMap = TypeVar(
49
+ "TVOCClassMap", VOCClassStringMap, int, list[VOCClassStringMap], list[int]
50
+ )
51
+
52
+
53
+ class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
54
+ _resources = [
55
+ DataLocation(
56
+ url="https://data.brainchip.com/dataset-mirror/voc/VOCtrainval_11-May-2012.tar",
57
+ filename="VOCtrainval_11-May-2012.tar",
58
+ md5=False,
59
+ checksum="e14f763270cf193d0b5f74b169f44157a4b0c6efa708f4dd0ff78ee691763bcb",
60
+ ),
61
+ DataLocation(
62
+ url="http://host.robots.ox.ac.uk/pascal/VOC/voc2011/VOCtrainval_25-May-2011.tar",
63
+ filename="VOCtrainval_25-May-2011.tar",
64
+ md5=False,
65
+ checksum="0a7f5f5d154f7290ec65ec3f78b72ef72c6d93ff6d79acd40dc222a9ee5248ba",
66
+ ),
67
+ DataLocation(
68
+ url="http://host.robots.ox.ac.uk/pascal/VOC/voc2010/VOCtrainval_03-May-2010.tar",
69
+ filename="VOCtrainval_03-May-2010.tar",
70
+ md5=False,
71
+ checksum="1af4189cbe44323ab212bff7afbc7d0f55a267cc191eb3aac911037887e5c7d4",
72
+ ),
73
+ DataLocation(
74
+ url="http://host.robots.ox.ac.uk/pascal/VOC/voc2009/VOCtrainval_11-May-2009.tar",
75
+ filename="VOCtrainval_11-May-2009.tar",
76
+ md5=False,
77
+ checksum="11cbe1741fb5bdadbbca3c08e9ec62cd95c14884845527d50847bc2cf57e7fd6",
78
+ ),
79
+ DataLocation(
80
+ url="http://host.robots.ox.ac.uk/pascal/VOC/voc2008/VOCtrainval_14-Jul-2008.tar",
81
+ filename="VOCtrainval_14-Jul-2008.tar",
82
+ md5=False,
83
+ checksum="7f0ca53c1b5a838fbe946965fc106c6e86832183240af5c88e3f6c306318d42e",
84
+ ),
85
+ DataLocation(
86
+ url="https://data.brainchip.com/dataset-mirror/voc/VOCtrainval_06-Nov-2007.tar",
87
+ filename="VOCtrainval_06-Nov-2007.tar",
88
+ md5=False,
89
+ checksum="7d8cd951101b0957ddfd7a530bdc8a94f06121cfc1e511bb5937e973020c7508",
90
+ ),
91
+ DataLocation(
92
+ url="https://data.brainchip.com/dataset-mirror/voc/VOC2012test.tar",
93
+ filename="VOC2012test.tar",
94
+ md5=False,
95
+ checksum="f08582b1935816c5eab3bbb1eb6d06201a789eaa173cdf1cf400c26f0cac2fb3",
96
+ ),
97
+ DataLocation(
98
+ url="https://data.brainchip.com/dataset-mirror/voc/VOCtest_06-Nov-2007.tar",
99
+ filename="VOCtest_06-Nov-2007.tar",
100
+ md5=False,
101
+ checksum="6836888e2e01dca84577a849d339fa4f73e1e4f135d312430c4856b5609b4892",
102
+ ),
103
+ ]
104
+ _base2007: tuple[int, int] = (5, 7)
105
+ _base2012: tuple[int, int] = (0, 6)
106
+
107
+ index2label: dict[int, str] = {
108
+ 0: "aeroplane",
109
+ 1: "bicycle",
110
+ 2: "bird",
111
+ 3: "boat",
112
+ 4: "bottle",
113
+ 5: "bus",
114
+ 6: "car",
115
+ 7: "cat",
116
+ 8: "chair",
117
+ 9: "cow",
118
+ 10: "diningtable",
119
+ 11: "dog",
120
+ 12: "horse",
121
+ 13: "motorbike",
122
+ 14: "person",
123
+ 15: "pottedplant",
124
+ 16: "sheep",
125
+ 17: "sofa",
126
+ 18: "train",
127
+ 19: "tvmonitor",
128
+ }
129
+
130
+ def __init__(
131
+ self,
132
+ root: str | Path,
133
+ image_set: Literal["train", "val", "test", "base"] = "train",
134
+ year: Literal["2007", "2008", "2009", "2010", "2011", "2012"] = "2012",
135
+ transforms: Transform[_TArray] | Sequence[Transform[_TArray]] | None = None,
136
+ download: bool = False,
137
+ verbose: bool = False,
138
+ ) -> None:
139
+ self.year = year
140
+ self._resource_index = self._get_year_image_set_index(year, image_set)
141
+ super().__init__(
142
+ root,
143
+ image_set,
144
+ transforms,
145
+ download,
146
+ verbose,
147
+ )
148
+
149
+ def _get_dataset_dir(self) -> Path:
150
+ """Overrides the base function to determine correct dataset directory for VOC class"""
151
+ return self._find_main_VOC_dir(self._root)
152
+
153
+ def _find_main_VOC_dir(self, base: Path) -> Path:
154
+ """
155
+ Determine the correct dataset directory for VOC detection and segmentation classes.
156
+ Handles various directory structure possibilities and validates existence.
157
+ """
158
+
159
+ # VOCdataset directory possibilities
160
+ dataset_dir = base if base.stem.lower() == "vocdataset" else base / "vocdataset"
161
+
162
+ # Define possible directory structures based on patterns
163
+ # 1. Root is already the specific VOC year directory
164
+ # 2. Root is the VOCdevkit directory
165
+ # 3. Standard structure
166
+ # 4. Special case for year 2011
167
+ # 5. Within VOCdataset directory
168
+ # 6. Special case for year 2011 within VOCdataset
169
+ possible_paths = [
170
+ base if base.stem == f"VOC{self.year}" else None,
171
+ base / f"VOC{self.year}" if base.stem == "VOCdevkit" else None,
172
+ base / "VOCdevkit" / f"VOC{self.year}",
173
+ base / "TrainVal" / "VOCdevkit" / f"VOC{self.year}"
174
+ if self.year == "2011"
175
+ else None,
176
+ dataset_dir / "VOCdevkit" / f"VOC{self.year}",
177
+ dataset_dir / "TrainVal" / "VOCdevkit" / f"VOC{self.year}"
178
+ if self.year == "2011"
179
+ else None,
180
+ ]
181
+
182
+ # Filter out None values and check each path
183
+ for path in filter(None, possible_paths):
184
+ if path.exists():
185
+ return path
186
+
187
+ # If no existing path is found, create and return the dataset directory
188
+ if not dataset_dir.exists():
189
+ dataset_dir.mkdir(parents=True, exist_ok=True)
190
+
191
+ return dataset_dir
192
+
193
+ def _get_year_image_set_index(self, year: str, image_set: str) -> int:
194
+ """Function to ensure that the correct resource file is accessed"""
195
+ if year == "2007" and image_set == "test":
196
+ return -1
197
+ if year == "2012" and image_set == "test":
198
+ return -2
199
+ if year != "2007" and image_set == "test":
200
+ raise ValueError(
201
+ f"The only test sets available are for the years 2007 and 2012, not {year}. "
202
+ "Either select the year 2007 or 2012, or use a different image_set."
203
+ )
204
+ return 2012 - int(year)
205
+
206
+ def _update_path(self) -> None:
207
+ """Update the path to the new folder structure"""
208
+ if self.year == "2011" and self.path.stem.lower() == "vocdataset":
209
+ self.path: Path = self.path / "TrainVal" / "VOCdevkit" / f"VOC{self.year}"
210
+ elif self.path.stem.lower() == "vocdataset":
211
+ self.path: Path = self.path / "VOCdevkit" / f"VOC{self.year}"
212
+
213
+ def _load_data_exception(self) -> tuple[list[str], list[str], dict[str, Any]]:
214
+ """Adjust how the directory is created for the 2007 and 2012 test set"""
215
+ filepaths: list[str] = []
216
+ targets: list[str] = []
217
+ datum_metadata: dict[str, list[Any]] = {}
218
+ tmp_path: Path = self._root / "tmp_directory_for_download"
219
+ tmp_path.mkdir(exist_ok=True)
220
+ resource_idx = self._base2007 if self.year == "2007" else self._base2012
221
+
222
+ # Determine if text files exist
223
+ train_file = self.path / "ImageSets" / "Main" / "trainval.txt"
224
+ test_file = self.path / "ImageSets" / "Main" / "test.txt"
225
+ train_exists = train_file.exists()
226
+ test_exists = test_file.exists()
227
+
228
+ if self.image_set == "base":
229
+ if not train_exists and not test_exists:
230
+ _ensure_exists(
231
+ *self._resources[resource_idx[0]],
232
+ self.path,
233
+ self._root,
234
+ self._download,
235
+ self._verbose,
236
+ )
237
+ self._update_path()
238
+ _ensure_exists(
239
+ *self._resources[resource_idx[1]],
240
+ tmp_path,
241
+ self._root,
242
+ self._download,
243
+ self._verbose,
244
+ )
245
+ self._merge_voc_directories(tmp_path)
246
+
247
+ elif train_exists and not test_exists:
248
+ _ensure_exists(
249
+ *self._resources[resource_idx[1]],
250
+ tmp_path,
251
+ self._root,
252
+ self._download,
253
+ self._verbose,
254
+ )
255
+ self._merge_voc_directories(tmp_path)
256
+
257
+ elif not train_exists and test_exists:
258
+ _ensure_exists(
259
+ *self._resources[resource_idx[0]],
260
+ tmp_path,
261
+ self._root,
262
+ self._download,
263
+ self._verbose,
264
+ )
265
+ self._merge_voc_directories(tmp_path)
266
+
267
+ # Code to determine what is needed in each category
268
+ metadata_list: list[dict[str, Any]] = []
269
+
270
+ for img_set in ["test", "base"]:
271
+ self.image_set = img_set
272
+ resource_filepaths, resource_targets, resource_metadata = (
273
+ self._load_data_inner()
274
+ )
275
+ filepaths.extend(resource_filepaths)
276
+ targets.extend(resource_targets)
277
+ metadata_list.append(resource_metadata)
278
+
279
+ # Combine metadata from all resources
280
+ for data_dict in metadata_list:
281
+ for key, val in data_dict.items():
282
+ str_key = str(key) # Ensure key is string
283
+ if str_key not in datum_metadata:
284
+ datum_metadata[str_key] = []
285
+ datum_metadata[str_key].extend(val)
286
+
287
+ else:
288
+ self._resource = self._resources[resource_idx[1]]
289
+
290
+ if train_exists and not test_exists:
291
+ _ensure_exists(
292
+ *self._resource, tmp_path, self._root, self._download, self._verbose
293
+ )
294
+ self._merge_voc_directories(tmp_path)
295
+
296
+ resource_filepaths, resource_targets, resource_metadata = (
297
+ self._load_try_and_update()
298
+ )
299
+ filepaths.extend(resource_filepaths)
300
+ targets.extend(resource_targets)
301
+ datum_metadata.update(resource_metadata)
302
+
303
+ return filepaths, targets, datum_metadata
304
+
305
+ def _merge_voc_directories(self, source_dir: Path) -> None:
306
+ """Merge two VOC directories, handling file conflicts intelligently."""
307
+ base: Path = self._find_main_VOC_dir(source_dir)
308
+ # Create all subdirectories in target if they don't exist
309
+ for dirpath, dirnames, filenames in os.walk(base):
310
+ # Convert to Path objects
311
+ source_path = Path(dirpath)
312
+
313
+ # Get the relative path from source_dir
314
+ rel_path = source_path.relative_to(base)
315
+
316
+ # Create the corresponding target path
317
+ target_path = self.path / rel_path
318
+ target_path.mkdir(parents=True, exist_ok=True)
319
+
320
+ # Copy all files
321
+ for filename in filenames:
322
+ source_file = source_path / filename
323
+ target_file = target_path / filename
324
+
325
+ # File doesn't exist in target, just move it
326
+ if not target_file.exists():
327
+ shutil.move(source_file, target_file)
328
+ else:
329
+ # File exists in both assume they're identical and skip
330
+ pass
331
+
332
+ shutil.rmtree(source_dir)
333
+
334
+ def _load_try_and_update(self) -> tuple[list[str], list[str], dict[str, Any]]:
335
+ """Test if data needs to be downloaded and update path if it does"""
336
+ if self._verbose:
337
+ print(f"Determining if {self._resource.filename} needs to be downloaded.")
338
+
339
+ try:
340
+ result = self._load_data_inner()
341
+ if self._verbose:
342
+ print("No download needed, loaded data successfully.")
343
+ except FileNotFoundError:
344
+ _ensure_exists(
345
+ *self._resource, self.path, self._root, self._download, self._verbose
346
+ )
347
+ self._update_path()
348
+ result = self._load_data_inner()
349
+ return result
350
+
351
+ def _load_data(self) -> tuple[list[str], list[str], dict[str, Any]]:
352
+ """
353
+ Function to determine if data can be accessed or if it needs to be downloaded and/or extracted.
354
+ """
355
+ # Exception - test sets
356
+ year_set_bool = (self.image_set == "test" or self.image_set == "base") and (
357
+ self.year == "2012" or self.year == "2007"
358
+ )
359
+ if year_set_bool:
360
+ return self._load_data_exception()
361
+
362
+ return self._load_try_and_update()
363
+
364
+ def _get_image_sets(self) -> dict[str, list[str]]:
365
+ """Function to create the list of images in each image set"""
366
+ image_folder = self.path / "JPEGImages"
367
+ image_set_list = (
368
+ ["train", "val", "trainval"] if self.image_set != "test" else ["test"]
369
+ )
370
+ image_sets = {}
371
+ for image_set in image_set_list:
372
+ text_file = self.path / "ImageSets" / "Main" / (image_set + ".txt")
373
+ selected_images: list[str] = []
374
+ with open(text_file) as f:
375
+ for line in f.readlines():
376
+ out = line.strip()
377
+ selected_images.append(str(image_folder / (out + ".jpg")))
378
+
379
+ name = "base" if image_set == "trainval" else image_set
380
+ image_sets[name] = selected_images
381
+ return image_sets
382
+
383
+ def _load_data_inner(self) -> tuple[list[str], list[str], dict[str, Any]]:
384
+ """Function to load in the file paths for the data, annotations and segmentation masks"""
385
+ file_meta = {"year": [], "image_id": [], "mask_path": []}
386
+ ann_folder = self.path / "Annotations"
387
+ seg_folder = self.path / "SegmentationClass"
388
+
389
+ # Load in the image sets
390
+ image_sets = self._get_image_sets()
391
+
392
+ # Get the data, annotations and metadata
393
+ annotations = []
394
+ data = image_sets[self.image_set]
395
+ for entry in data:
396
+ file_name = Path(entry).name
397
+ file_stem = Path(entry).stem
398
+ if self.year != "2007":
399
+ # Remove file extension and split by "_"
400
+ parts = file_stem.split("_")
401
+ file_meta["year"].append(parts[0])
402
+ file_meta["image_id"].append(parts[1])
403
+ else:
404
+ file_meta["year"].append(self.year)
405
+ file_meta["image_id"].append(file_stem)
406
+ file_meta["mask_path"].append(str(seg_folder / file_name))
407
+ annotations.append(str(ann_folder / file_stem) + ".xml")
408
+
409
+ return data, annotations, file_meta
410
+
411
+ def _read_annotations(
412
+ self, annotation: str
413
+ ) -> tuple[list[list[float]], list[int], dict[str, Any]]:
414
+ boxes: list[list[float]] = []
415
+ label_str = []
416
+ if not Path(annotation).exists():
417
+ return boxes, label_str, {}
418
+ root = parse(annotation).getroot()
419
+ if root is None:
420
+ raise ValueError(f"Unable to parse {annotation}")
421
+ additional_meta: dict[str, Any] = {
422
+ "folder": root.findtext("folder", default=""),
423
+ "filename": root.findtext("filename", default=""),
424
+ "database": root.findtext("source/database", default=""),
425
+ "annotation_source": root.findtext("source/annotation", default=""),
426
+ "image_source": root.findtext("source/image", default=""),
427
+ "image_width": int(root.findtext("size/width", default="-1")),
428
+ "image_height": int(root.findtext("size/height", default="-1")),
429
+ "image_depth": int(root.findtext("size/depth", default="-1")),
430
+ "segmented": int(root.findtext("segmented", default="-1")),
431
+ "pose": [],
432
+ "truncated": [],
433
+ "difficult": [],
434
+ }
435
+ for obj in root.findall("object"):
436
+ label_str.append(obj.findtext("name", default=""))
437
+ additional_meta["pose"].append(obj.findtext("pose", default=""))
438
+ additional_meta["truncated"].append(
439
+ int(obj.findtext("truncated", default="-1"))
440
+ )
441
+ additional_meta["difficult"].append(
442
+ int(obj.findtext("difficult", default="-1"))
443
+ )
444
+ boxes.append(
445
+ [
446
+ float(obj.findtext("bndbox/xmin", default="0")),
447
+ float(obj.findtext("bndbox/ymin", default="0")),
448
+ float(obj.findtext("bndbox/xmax", default="0")),
449
+ float(obj.findtext("bndbox/ymax", default="0")),
450
+ ]
451
+ )
452
+ labels = [self.label2index[label] for label in label_str]
453
+ return boxes, labels, additional_meta
454
+
455
+
456
+ class VOCDetection(
457
+ BaseVOCDataset[
458
+ NDArray[np.number[Any]], ObjectDetectionTarget[NDArray[np.number[Any]]]
459
+ ],
460
+ BaseODDataset[NDArray[np.number[Any]], list[str], str],
461
+ BaseDatasetNumpyMixin,
462
+ ):
463
+ """
464
+ `Pascal VOC <http://host.robots.ox.ac.uk/pascal/VOC/>`_ Detection Dataset.
465
+
466
+ Parameters
467
+ ----------
468
+ root : str or pathlib.Path
469
+ Because of the structure of the PASCAL VOC datasets, the root needs to be one of 4 folders.
470
+ 1) Directory containing the year of the **already downloaded** dataset (i.e. .../VOCdevkit/VOC2012 <-)
471
+ 2) Directory to the VOCdevkit folder of the **already downloaded** dataset (i.e. .../VOCdevkit <- /VOC2012)
472
+ 3) Directory to the folder one level up from the VOCdevkit folder,
473
+ data **may** or **may not** be already downloaded (i.e. ... <- /VOCdevkit/VOC2012)
474
+ 4) Directory to where you would like the dataset to be downloaded
475
+ image_set : "train", "val", "test", or "base", default "train"
476
+ If "test", then dataset year must be "2007" or "2012". Note that the 2012 test set does not contain annotations.
477
+ If "base", then the combined dataset of "train" and "val" is returned.
478
+ year : "2007", "2008", "2009", "2010", "2011" or "2012", default "2012"
479
+ The dataset year.
480
+ transforms : Transform, Sequence[Transform] or None, default None
481
+ Transform(s) to apply to the data.
482
+ download : bool, default False
483
+ If True, downloads the dataset from the internet and puts it in root directory.
484
+ Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
485
+ verbose : bool, default False
486
+ If True, outputs print statements.
487
+
488
+ Attributes
489
+ ----------
490
+ path : pathlib.Path
491
+ Location of the folder containing the data.
492
+ year : "2007", "2008", "2009", "2010", "2011" or "2012"
493
+ The selected dataset year.
494
+ image_set : "train", "val", "test" or "base"
495
+ The selected image set from the dataset.
496
+ index2label : dict[int, str]
497
+ Dictionary which translates from class integers to the associated class strings.
498
+ label2index : dict[str, int]
499
+ Dictionary which translates from class strings to the associated class integers.
500
+ metadata : DatasetMetadata
501
+ Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
502
+ transforms : Sequence[Transform]
503
+ The transforms to be applied to the data.
504
+ size : int
505
+ The size of the dataset.
506
+
507
+ Note
508
+ ----
509
+ Data License: `Flickr Terms of Use <http://www.flickr.com/terms.gne?legacy=1>`_
510
+ """
@@ -0,0 +1,65 @@
1
+ from __future__ import annotations
2
+
3
+ __all__ = []
4
+
5
+ from torch import Tensor
6
+
7
+ from maite_datasets._base import BaseODDataset
8
+ from maite_datasets._types import ObjectDetectionTarget
9
+ from maite_datasets._mixin._torch import BaseDatasetTorchMixin
10
+ from maite_datasets.object_detection._voc import BaseVOCDataset
11
+
12
+
13
+ class VOCDetectionTorch(
14
+ BaseVOCDataset[Tensor, ObjectDetectionTarget[Tensor]],
15
+ BaseODDataset[Tensor, list[str], str],
16
+ BaseDatasetTorchMixin,
17
+ ):
18
+ """
19
+ `Pascal VOC <http://host.robots.ox.ac.uk/pascal/VOC/>`_ Detection Dataset as PyTorch tensors.
20
+
21
+ Parameters
22
+ ----------
23
+ root : str or pathlib.Path
24
+ Because of the structure of the PASCAL VOC datasets, the root needs to be one of 4 folders.
25
+ 1) Directory containing the year of the **already downloaded** dataset (i.e. .../VOCdevkit/VOC2012 <-)
26
+ 2) Directory to the VOCdevkit folder of the **already downloaded** dataset (i.e. .../VOCdevkit <- /VOC2012)
27
+ 3) Directory to the folder one level up from the VOCdevkit folder,
28
+ data **may** or **may not** be already downloaded (i.e. ... <- /VOCdevkit/VOC2012)
29
+ 4) Directory to where you would like the dataset to be downloaded
30
+ image_set : "train", "val", "test", or "base", default "train"
31
+ If "test", then dataset year must be "2007" or "2012". Note that the 2012 test set does not contain annotations.
32
+ If "base", then the combined dataset of "train" and "val" is returned.
33
+ year : "2007", "2008", "2009", "2010", "2011" or "2012", default "2012"
34
+ The dataset year.
35
+ transforms : Transform, Sequence[Transform] or None, default None
36
+ Transform(s) to apply to the data.
37
+ download : bool, default False
38
+ If True, downloads the dataset from the internet and puts it in root directory.
39
+ Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
40
+ verbose : bool, default False
41
+ If True, outputs print statements.
42
+
43
+ Attributes
44
+ ----------
45
+ path : pathlib.Path
46
+ Location of the folder containing the data.
47
+ year : "2007", "2008", "2009", "2010", "2011" or "2012"
48
+ The selected dataset year.
49
+ image_set : "train", "val", "test" or "base"
50
+ The selected image set from the dataset.
51
+ index2label : dict[int, str]
52
+ Dictionary which translates from class integers to the associated class strings.
53
+ label2index : dict[str, int]
54
+ Dictionary which translates from class strings to the associated class integers.
55
+ metadata : DatasetMetadata
56
+ Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
57
+ transforms : Sequence[Transform]
58
+ The transforms to be applied to the data.
59
+ size : int
60
+ The size of the dataset.
61
+
62
+ Note
63
+ ----
64
+ Data License: `Flickr Terms of Use <http://www.flickr.com/terms.gne?legacy=1>`_
65
+ """
File without changes
@@ -0,0 +1,91 @@
1
+ Metadata-Version: 2.4
2
+ Name: maite-datasets
3
+ Version: 0.0.1
4
+ Summary: A collection of Image Classification and Object Detection task datasets conforming to the MAITE protocol.
5
+ Author-email: Andrew Weng <andrew.weng@ariacoustics.com>, Ryan Wood <ryan.wood@ariacoustics.com>, Shaun Jullens <shaun.jullens@ariacoustics.com>
6
+ License-Expression: MIT
7
+ License-File: LICENSE
8
+ Classifier: Development Status :: 4 - Beta
9
+ Classifier: Framework :: Pytest
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 3 :: Only
13
+ Classifier: Programming Language :: Python :: 3.9
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Requires-Python: >=3.9
18
+ Requires-Dist: defusedxml>=0.7.1
19
+ Requires-Dist: numpy>=1.24.2
20
+ Requires-Dist: pillow>=10.3.0
21
+ Requires-Dist: requests>=2.32.3
22
+ Requires-Dist: typing-extensions>=4.12
23
+ Provides-Extra: tqdm
24
+ Requires-Dist: tqdm>=4.66; extra == 'tqdm'
25
+ Description-Content-Type: text/markdown
26
+
27
+ # MAITE Datasets
28
+
29
+ MAITE Datasets are a collection of public datasets wrapped in a [MAITE](https://mit-ll-ai-technology.github.io/maite/) compliant format.
30
+
31
+ ## Installation
32
+
33
+ To install and use `maite-datasets` you can use pip:
34
+
35
+ ```bash
36
+ pip install maite-datasets
37
+ ```
38
+
39
+ For status bar indicators when downloading, you can include the extra `tqdm` when installing:
40
+
41
+ ```bash
42
+ pip install maite-datasets[tqdm]
43
+ ```
44
+
45
+ ## Available Datasets
46
+
47
+ | Task | Dataset | Description |
48
+ |----------------|------------------|---------------------------------------------------------------------|
49
+ | Classification | CIFAR10 | [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) dataset. |
50
+ | Classification | MNIST | A dataset of hand-written digits. |
51
+ | Classification | Ships | A dataset that focuses on identifying ships from satellite images. |
52
+ | Detection | AntiUAVDetection | A UAV detection dataset in natural images with varying backgrounds. |
53
+ | Detection | MILCO | A side-scan sonar dataset focused on mine-like object detection. |
54
+ | Detection | Seadrone | A UAV dataset focused on open water object detection. |
55
+ | Detection | VOCDetection | [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) dataset. |
56
+
57
+ ## Usage
58
+
59
+ Here is an example of how to import MNIST for usage with your workflow.
60
+
61
+ ```python
62
+ >>> from maite_datasets.image_classification import MNIST
63
+
64
+ >>> mnist = MNIST(root="data", download=True)
65
+ >>> print(mnist)
66
+ MNIST Dataset
67
+ -------------
68
+ Corruption: None
69
+ Transforms: []
70
+ Image_set: train
71
+ Metadata: {'id': 'MNIST_train', 'index2label': {0: 'zero', 1: 'one', 2: 'two', 3: 'three', 4: 'four', 5: 'five', 6: 'six', 7: 'seven', 8: 'eight', 9: 'nine'}, 'split': 'train'}
72
+ Path: /home/user/maite-datasets/data/mnist
73
+ Size: 60000
74
+
75
+ >>> print("tuple("+", ".join([str(type(t)) for t in mnist[0]])+")")
76
+ tuple(<class 'numpy.ndarray'>, <class 'numpy.ndarray'>, <class 'dict'>)
77
+ ```
78
+
79
+ ## Additional Information
80
+
81
+ For more information on the MAITE protocol, check out their [documentation](https://mit-ll-ai-technology.github.io/maite/).
82
+
83
+ ## Acknowledgement
84
+
85
+ ### CDAO Funding Acknowledgement
86
+
87
+ This material is based upon work supported by the Chief Digital and Artificial
88
+ Intelligence Office under Contract No. W519TC-23-9-2033. The views and
89
+ conclusions contained herein are those of the author(s) and should not be
90
+ interpreted as necessarily representing the official policies or endorsements,
91
+ either expressed or implied, of the U.S. Government.