maite-datasets 0.0.1__tar.gz

maite_datasets-0.0.1/.gitignore

@@ -0,0 +1,13 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+# Test output
+output/

maite_datasets-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 ARiA
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

maite_datasets-0.0.1/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: maite-datasets
+Version: 0.0.1
+Summary: A collection of Image Classification and Object Detection task datasets conforming to the MAITE protocol.
+Author-email: Andrew Weng <andrew.weng@ariacoustics.com>, Ryan Wood <ryan.wood@ariacoustics.com>, Shaun Jullens <shaun.jullens@ariacoustics.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Pytest
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Requires-Dist: defusedxml>=0.7.1
+Requires-Dist: numpy>=1.24.2
+Requires-Dist: pillow>=10.3.0
+Requires-Dist: requests>=2.32.3
+Requires-Dist: typing-extensions>=4.12
+Provides-Extra: tqdm
+Requires-Dist: tqdm>=4.66; extra == 'tqdm'
+Description-Content-Type: text/markdown
+
+# MAITE Datasets
+
+MAITE Datasets are a collection of public datasets wrapped in a [MAITE](https://mit-ll-ai-technology.github.io/maite/) compliant format.
+
+## Installation
+
+To install and use `maite-datasets` you can use pip:
+
+```bash
+pip install maite-datasets
+```
+
+For status bar indicators when downloading, you can include the extra `tqdm` when installing:
+
+```bash
+pip install maite-datasets[tqdm]
+```
+
+## Available Datasets
+
+| Task           | Dataset          | Description                                                         |
+|----------------|------------------|---------------------------------------------------------------------|
+| Classification | CIFAR10          | [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) dataset.     |
+| Classification | MNIST            | A dataset of hand-written digits.                                   |
+| Classification | Ships            | A dataset that focuses on identifying ships from satellite images.  |
+| Detection      | AntiUAVDetection | A UAV detection dataset in natural images with varying backgrounds. |
+| Detection      | MILCO            | A side-scan sonar dataset focused on mine-like object detection.    |
+| Detection      | Seadrone         | A UAV dataset focused on open water object detection.               |
+| Detection      | VOCDetection     | [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) dataset.      |
+
+## Usage
+
+Here is an example of how to import MNIST for usage with your workflow.
+
+```python
+>>> from maite_datasets.image_classification import MNIST
+
+>>> mnist = MNIST(root="data", download=True)
+>>> print(mnist)
+MNIST Dataset
+-------------
+    Corruption: None
+    Transforms: []
+    Image_set: train
+    Metadata: {'id': 'MNIST_train', 'index2label': {0: 'zero', 1: 'one', 2: 'two', 3: 'three', 4: 'four', 5: 'five', 6: 'six', 7: 'seven', 8: 'eight', 9: 'nine'}, 'split': 'train'}
+    Path: /home/user/maite-datasets/data/mnist
+    Size: 60000
+
+>>> print("tuple("+", ".join([str(type(t)) for t in mnist[0]])+")")
+tuple(<class 'numpy.ndarray'>, <class 'numpy.ndarray'>, <class 'dict'>)
+```
+
+## Additional Information
+
+For more information on the MAITE protocol, check out their [documentation](https://mit-ll-ai-technology.github.io/maite/).
+
+## Acknowledgement
+
+### CDAO Funding Acknowledgement
+
+This material is based upon work supported by the Chief Digital and Artificial
+Intelligence Office under Contract No. W519TC-23-9-2033. The views and
+conclusions contained herein are those of the author(s) and should not be
+interpreted as necessarily representing the official policies or endorsements,
+either expressed or implied, of the U.S. Government.

maite_datasets-0.0.1/README.md

@@ -0,0 +1,65 @@
+# MAITE Datasets
+
+MAITE Datasets are a collection of public datasets wrapped in a [MAITE](https://mit-ll-ai-technology.github.io/maite/) compliant format.
+
+## Installation
+
+To install and use `maite-datasets` you can use pip:
+
+```bash
+pip install maite-datasets
+```
+
+For status bar indicators when downloading, you can include the extra `tqdm` when installing:
+
+```bash
+pip install maite-datasets[tqdm]
+```
+
+## Available Datasets
+
+| Task           | Dataset          | Description                                                         |
+|----------------|------------------|---------------------------------------------------------------------|
+| Classification | CIFAR10          | [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) dataset.     |
+| Classification | MNIST            | A dataset of hand-written digits.                                   |
+| Classification | Ships            | A dataset that focuses on identifying ships from satellite images.  |
+| Detection      | AntiUAVDetection | A UAV detection dataset in natural images with varying backgrounds. |
+| Detection      | MILCO            | A side-scan sonar dataset focused on mine-like object detection.    |
+| Detection      | Seadrone         | A UAV dataset focused on open water object detection.               |
+| Detection      | VOCDetection     | [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) dataset.      |
+
+## Usage
+
+Here is an example of how to import MNIST for usage with your workflow.
+
+```python
+>>> from maite_datasets.image_classification import MNIST
+
+>>> mnist = MNIST(root="data", download=True)
+>>> print(mnist)
+MNIST Dataset
+-------------
+    Corruption: None
+    Transforms: []
+    Image_set: train
+    Metadata: {'id': 'MNIST_train', 'index2label': {0: 'zero', 1: 'one', 2: 'two', 3: 'three', 4: 'four', 5: 'five', 6: 'six', 7: 'seven', 8: 'eight', 9: 'nine'}, 'split': 'train'}
+    Path: /home/user/maite-datasets/data/mnist
+    Size: 60000
+
+>>> print("tuple("+", ".join([str(type(t)) for t in mnist[0]])+")")
+tuple(<class 'numpy.ndarray'>, <class 'numpy.ndarray'>, <class 'dict'>)
+```
+
+## Additional Information
+
+For more information on the MAITE protocol, check out their [documentation](https://mit-ll-ai-technology.github.io/maite/).
+
+## Acknowledgement
+
+### CDAO Funding Acknowledgement
+
+This material is based upon work supported by the Chief Digital and Artificial
+Intelligence Office under Contract No. W519TC-23-9-2033. The views and
+conclusions contained herein are those of the author(s) and should not be
+interpreted as necessarily representing the official policies or endorsements,
+either expressed or implied, of the U.S. Government.

maite_datasets-0.0.1/pyproject.toml

@@ -0,0 +1,115 @@
+[project]
+name = "maite-datasets"
+description = "A collection of Image Classification and Object Detection task datasets conforming to the MAITE protocol."
+readme = "README.md"
+requires-python = ">=3.9"
+dynamic = ["version"]
+dependencies = [
+    "defusedxml>=0.7.1",
+    "numpy>=1.24.2",
+    "pillow>=10.3.0",
+    "requests>=2.32.3",
+    "typing-extensions>=4.12",
+]
+license = "MIT"
+authors = [
+  { name = "Andrew Weng", email = "andrew.weng@ariacoustics.com" },
+  { name = "Ryan Wood", email = "ryan.wood@ariacoustics.com" },
+  { name = "Shaun Jullens", email = "shaun.jullens@ariacoustics.com" },
+]
+classifiers = [
+   "Development Status :: 4 - Beta",
+   "Framework :: Pytest",
+   "Operating System :: OS Independent",
+   "License :: OSI Approved :: MIT License",
+   "Programming Language :: Python :: 3 :: Only",
+   "Programming Language :: Python :: 3.9",
+   "Programming Language :: Python :: 3.10",
+   "Programming Language :: Python :: 3.11",
+   "Programming Language :: Python :: 3.12",
+]
+
+[project.optional-dependencies]
+tqdm = [
+    "tqdm>=4.66",
+]
+
+[dependency-groups]
+base = [
+  "nox[uv]>=2025.5.1",
+  "torch>=2.2.0",
+  "uv>=0.7.8",
+]
+lint = [
+  "ruff>=0.11",
+  "codespell[toml]>=2.3",
+]
+test = [
+  { include-group = "base" },
+  "pytest>=8.3",
+  "pytest-cov>=6.1",
+  "coverage[toml]>=7.6",
+]
+type = [
+  { include-group = "base" },
+  "pyright[nodejs]>=1.1.400",
+]
+dev = [
+  { include-group = "base" },
+  { include-group = "lint" },
+  { include-group = "test" },
+  { include-group = "type" },
+]
+
+[tool.uv.sources]
+torch = [
+  { index = "pytorch-cpu" },
+]
+
+[[tool.uv.index]]
+name = "pytorch-cpu"
+url = "https://download.pytorch.org/whl/cpu"
+explicit = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = [
+  "--pythonwarnings=ignore::DeprecationWarning",
+]
+markers = [
+  "required: marks tests for required features",
+  "optional: marks tests for optional features",
+  "year: marks tests that need a specified dataset year",
+]
+
+[tool.coverage.run]
+source = ["src/maite_datasets"]
+branch = true
+
+[tool.coverage.report]
+exclude_also = [
+  "raise NotImplementedError",
+  ": \\.\\.\\.",
+]
+include = ["*/src/maite_datasets/*"]
+fail_under = 90
+
+[tool.codespell]
+skip = './*env*,./output,uv.lock'
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/maite_datasets"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/src",
+    "/LICENSE",
+    "/README.md",
+]
+
+[tool.hatch.version]
+source = "vcs"
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"

maite_datasets-0.0.1/src/maite_datasets/__init__.py

@@ -0,0 +1 @@
+"""Module for MAITE compliant Computer Vision datasets."""

maite_datasets-0.0.1/src/maite_datasets/_base.py

@@ -0,0 +1,254 @@
+from __future__ import annotations
+
+__all__ = []
+
+from abc import abstractmethod
+from pathlib import Path
+from typing import Any, Generic, Iterator, Literal, NamedTuple, Sequence, TypeVar, cast
+
+import numpy as np
+
+from maite_datasets._fileio import _ensure_exists
+from maite_datasets._protocols import Array, Transform
+from maite_datasets._types import (
+    AnnotatedDataset,
+    DatasetMetadata,
+    DatumMetadata,
+    ImageClassificationDataset,
+    ObjectDetectionDataset,
+    ObjectDetectionTarget,
+)
+
+_TArray = TypeVar("_TArray", bound=Array)
+_TTarget = TypeVar("_TTarget")
+_TRawTarget = TypeVar(
+    "_TRawTarget",
+    Sequence[int],
+    Sequence[str],
+    Sequence[tuple[list[int], list[list[float]]]],
+)
+_TAnnotation = TypeVar("_TAnnotation", int, str, tuple[list[int], list[list[float]]])
+
+
+def _to_datum_metadata(index: int, metadata: dict[str, Any]) -> DatumMetadata:
+    _id = metadata.pop("id", index)
+    return DatumMetadata(id=_id, **metadata)
+
+
+class DataLocation(NamedTuple):
+    url: str
+    filename: str
+    md5: bool
+    checksum: str
+
+
+class BaseDatasetMixin(Generic[_TArray]):
+    index2label: dict[int, str]
+
+    def _as_array(self, raw: list[Any]) -> _TArray: ...
+    def _one_hot_encode(self, value: int | list[int]) -> _TArray: ...
+    def _read_file(self, path: str) -> _TArray: ...
+
+
+class BaseDataset(
+    AnnotatedDataset[tuple[_TArray, _TTarget, DatumMetadata]],
+    Generic[_TArray, _TTarget, _TRawTarget, _TAnnotation],
+):
+    """
+    Base class for internet downloaded datasets.
+    """
+
+    # Each subclass should override the attributes below.
+    # Each resource tuple must contain:
+    #    'url': str, the URL to download from
+    #    'filename': str, the name of the file once downloaded
+    #    'md5': boolean, True if it's the checksum value is md5
+    #    'checksum': str, the associated checksum for the downloaded file
+    _resources: list[DataLocation]
+    _resource_index: int = 0
+    index2label: dict[int, str]
+
+    def __init__(
+        self,
+        root: str | Path,
+        image_set: Literal["train", "val", "test", "operational", "base"] = "train",
+        transforms: Transform[_TArray] | Sequence[Transform[_TArray]] | None = None,
+        download: bool = False,
+        verbose: bool = False,
+    ) -> None:
+        self._root: Path = (
+            root.absolute() if isinstance(root, Path) else Path(root).absolute()
+        )
+        transforms = transforms if transforms is not None else []
+        self.transforms: Sequence[Transform[_TArray]] = (
+            transforms if isinstance(transforms, Sequence) else [transforms]
+        )
+        self.image_set = image_set
+        self._verbose = verbose
+
+        # Internal Attributes
+        self._download = download
+        self._filepaths: list[str]
+        self._targets: _TRawTarget
+        self._datum_metadata: dict[str, list[Any]]
+        self._resource: DataLocation = self._resources[self._resource_index]
+        self._label2index = {v: k for k, v in self.index2label.items()}
+
+        self.metadata: DatasetMetadata = DatasetMetadata(
+            id=self._unique_id(),
+            index2label=self.index2label,
+            split=self.image_set,
+        )
+
+        # Load the data
+        self.path: Path = self._get_dataset_dir()
+        self._filepaths, self._targets, self._datum_metadata = self._load_data()
+        self.size: int = len(self._filepaths)
+
+    def __str__(self) -> str:
+        nt = "\n    "
+        title = f"{self.__class__.__name__} Dataset"
+        sep = "-" * len(title)
+        attrs = [
+            f"{k.capitalize()}: {v}"
+            for k, v in self.__dict__.items()
+            if not k.startswith("_")
+        ]
+        return f"{title}\n{sep}{nt}{nt.join(attrs)}"
+
+    @property
+    def label2index(self) -> dict[str, int]:
+        return self._label2index
+
+    def __iter__(self) -> Iterator[tuple[_TArray, _TTarget, DatumMetadata]]:
+        for i in range(len(self)):
+            yield self[i]
+
+    def _get_dataset_dir(self) -> Path:
+        # Create a designated folder for this dataset (named after the class)
+        if self._root.stem.lower() == self.__class__.__name__.lower():
+            dataset_dir: Path = self._root
+        else:
+            dataset_dir: Path = self._root / self.__class__.__name__.lower()
+        if not dataset_dir.exists():
+            dataset_dir.mkdir(parents=True, exist_ok=True)
+        return dataset_dir
+
+    def _unique_id(self) -> str:
+        return f"{self.__class__.__name__}_{self.image_set}"
+
+    def _load_data(self) -> tuple[list[str], _TRawTarget, dict[str, Any]]:
+        """
+        Function to determine if data can be accessed or if it needs to be downloaded and/or extracted.
+        """
+        if self._verbose:
+            print(f"Determining if {self._resource.filename} needs to be downloaded.")
+
+        try:
+            result = self._load_data_inner()
+            if self._verbose:
+                print("No download needed, loaded data successfully.")
+        except FileNotFoundError:
+            _ensure_exists(
+                *self._resource, self.path, self._root, self._download, self._verbose
+            )
+            result = self._load_data_inner()
+        return result
+
+    @abstractmethod
+    def _load_data_inner(self) -> tuple[list[str], _TRawTarget, dict[str, Any]]: ...
+
+    def _transform(self, image: _TArray) -> _TArray:
+        """Function to transform the image prior to returning based on parameters passed in."""
+        for transform in self.transforms:
+            image = transform(image)
+        return image
+
+    def __len__(self) -> int:
+        return self.size
+
+
+class BaseICDataset(
+    BaseDataset[_TArray, _TArray, list[int], int],
+    BaseDatasetMixin[_TArray],
+    ImageClassificationDataset[_TArray],
+):
+    """
+    Base class for image classification datasets.
+    """
+
+    def __getitem__(self, index: int) -> tuple[_TArray, _TArray, DatumMetadata]:
+        """
+        Args
+        ----
+        index : int
+            Value of the desired data point
+
+        Returns
+        -------
+        tuple[TArray, TArray, DatumMetadata]
+            Image, target, datum_metadata - where target is one-hot encoding of class.
+        """
+        # Get the associated label and score
+        label = self._targets[index]
+        score = self._one_hot_encode(label)
+        # Get the image
+        img = self._read_file(self._filepaths[index])
+        img = self._transform(img)
+
+        img_metadata = {key: val[index] for key, val in self._datum_metadata.items()}
+
+        return img, score, _to_datum_metadata(index, img_metadata)
+
+
+class BaseODDataset(
+    BaseDataset[_TArray, ObjectDetectionTarget[_TArray], _TRawTarget, _TAnnotation],
+    BaseDatasetMixin[_TArray],
+    ObjectDetectionDataset[_TArray],
+):
+    """
+    Base class for object detection datasets.
+    """
+
+    _bboxes_per_size: bool = False
+
+    def __getitem__(
+        self, index: int
+    ) -> tuple[_TArray, ObjectDetectionTarget[_TArray], DatumMetadata]:
+        """
+        Args
+        ----
+        index : int
+            Value of the desired data point
+
+        Returns
+        -------
+        tuple[TArray, ObjectDetectionTarget[TArray], DatumMetadata]
+            Image, target, datum_metadata - target.boxes returns boxes in x0, y0, x1, y1 format
+        """
+        # Grab the bounding boxes and labels from the annotations
+        annotation = cast(_TAnnotation, self._targets[index])
+        boxes, labels, additional_metadata = self._read_annotations(annotation)
+        # Get the image
+        img = self._read_file(self._filepaths[index])
+        img_size = img.shape
+        img = self._transform(img)
+        # Adjust labels if necessary
+        if self._bboxes_per_size and boxes:
+            boxes = boxes * np.array(
+                [[img_size[1], img_size[2], img_size[1], img_size[2]]]
+            )
+        # Create the Object Detection Target
+        target = ObjectDetectionTarget(
+            self._as_array(boxes), self._as_array(labels), self._one_hot_encode(labels)
+        )
+
+        img_metadata = {key: val[index] for key, val in self._datum_metadata.items()}
+        img_metadata = img_metadata | additional_metadata
+
+        return img, target, _to_datum_metadata(index, img_metadata)
+
+    @abstractmethod
+    def _read_annotations(
+        self, annotation: _TAnnotation
+    ) -> tuple[list[list[float]], list[int], dict[str, Any]]: ...