maite-datasets 0.0.5__tar.gz → 0.0.6__tar.gz

{maite_datasets-0.0.5 → maite_datasets-0.0.6}/.gitignore

@@ -9,6 +9,10 @@ wheels/
 # Virtual environments
 .venv
 
+# Downloaded data
+.data
+
 # Test output
 .nox/
-output/
+output/
+.coverage

{maite_datasets-0.0.5 → maite_datasets-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: maite-datasets
-Version: 0.0.5
+Version: 0.0.6
 Summary: A collection of Image Classification and Object Detection task datasets conforming to the MAITE protocol.
 Author-email: Andrew Weng <andrew.weng@ariacoustics.com>, Ryan Wood <ryan.wood@ariacoustics.com>, Shaun Jullens <shaun.jullens@ariacoustics.com>
 License-Expression: MIT
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.9
 Requires-Dist: defusedxml>=0.7.1
+Requires-Dist: maite<0.9,>=0.7
 Requires-Dist: numpy>=1.24.2
 Requires-Dist: pillow>=10.3.0
 Requires-Dist: requests>=2.32.3
@@ -42,7 +43,7 @@ For status bar indicators when downloading, you can include the extra `tqdm` whe
 pip install maite-datasets[tqdm]
 ```
 
-## Available Datasets
+## Available Downloadable Datasets
 
 | Task           | Dataset          | Description                                                         |
 |----------------|------------------|---------------------------------------------------------------------|
@@ -54,7 +55,7 @@ pip install maite-datasets[tqdm]
 | Detection      | Seadrone         | A UAV dataset focused on open water object detection.               |
 | Detection      | VOCDetection     | [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) dataset.      |
 
-## Usage
+### Usage
 
 Here is an example of how to import MNIST for usage with your workflow.
 
@@ -76,6 +77,58 @@ MNIST Dataset
 tuple(<class 'numpy.ndarray'>, <class 'numpy.ndarray'>, <class 'dict'>)
 ```
 
+## Dataset Wrappers
+
+Wrappers provide a way to convert datasets to allow usage of tools within specific backend frameworks.
+
+`TorchvisionWrapper` is a convenience class that wraps any of the datasets and provides the capability to apply
+`torchvision` transforms to the dataset.
+
+**NOTE:** `TorchvisionWrapper` requires _torch_ and _torchvision_ to be installed.
+
+```python
+>>> from maite_datasets.object_detection import MILCO
+
+>>> milco = MILCO(root="data", download=True)
+>>> print(milco)
+MILCO Dataset
+-------------
+    Transforms: []
+    Image Set: train
+    Metadata: {'id': 'MILCO_train', 'index2label': {0: 'MILCO', 1: 'NOMBO'}, 'split': 'train'}
+    Path: /home/user/maite-datasets/data/milco
+    Size: 261
+
+>>> print(f"type={milco[0][0].__class__.__name__}, shape={milco[0][0].shape}")
+type=ndarray, shape=(3, 1024, 1024)
+
+>>> print(milco[0][1].boxes[0])
+[ 75. 217. 130. 247.]
+
+>>> from maite_datasets.wrappers import TorchvisionWrapper
+>>> from torchvision.transforms.v2 import Resize
+
+>>> milco_torch = TorchvisionWrapper(milco, transforms=Resize(224))
+>>> print(milco_torch)
+Torchvision Wrapped MILCO Dataset
+---------------------------
+    Transforms: Resize(size=[224], interpolation=InterpolationMode.BILINEAR, antialias=True)
+
+MILCO Dataset
+-------------
+    Transforms: []
+    Image Set: train
+    Metadata: {'id': 'MILCO_train', 'index2label': {0: 'MILCO', 1: 'NOMBO'}, 'split': 'train'}
+    Path: /home/user/maite-datasets/data/milco
+    Size: 261
+
+>>> print(f"type={milco_torch[0][0].__class__.__name__}, shape={milco_torch[0][0].shape}")
+type=Image, shape=torch.Size([3, 224, 224])
+
+>>> print(milco_torch[0][1].boxes[0])
+tensor([16.4062, 47.4688, 28.4375, 54.0312], dtype=torch.float64)
+```
+
 ## Additional Information
 
 For more information on the MAITE protocol, check out their [documentation](https://mit-ll-ai-technology.github.io/maite/).

{maite_datasets-0.0.5 → maite_datasets-0.0.6}/README.md

@@ -16,7 +16,7 @@ For status bar indicators when downloading, you can include the extra `tqdm` whe
 pip install maite-datasets[tqdm]
 ```
 
-## Available Datasets
+## Available Downloadable Datasets
 
 | Task           | Dataset          | Description                                                         |
 |----------------|------------------|---------------------------------------------------------------------|
@@ -28,7 +28,7 @@ pip install maite-datasets[tqdm]
 | Detection      | Seadrone         | A UAV dataset focused on open water object detection.               |
 | Detection      | VOCDetection     | [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) dataset.      |
 
-## Usage
+### Usage
 
 Here is an example of how to import MNIST for usage with your workflow.
 
@@ -50,6 +50,58 @@ MNIST Dataset
 tuple(<class 'numpy.ndarray'>, <class 'numpy.ndarray'>, <class 'dict'>)
 ```
 
+## Dataset Wrappers
+
+Wrappers provide a way to convert datasets to allow usage of tools within specific backend frameworks.
+
+`TorchvisionWrapper` is a convenience class that wraps any of the datasets and provides the capability to apply
+`torchvision` transforms to the dataset.
+
+**NOTE:** `TorchvisionWrapper` requires _torch_ and _torchvision_ to be installed.
+
+```python
+>>> from maite_datasets.object_detection import MILCO
+
+>>> milco = MILCO(root="data", download=True)
+>>> print(milco)
+MILCO Dataset
+-------------
+    Transforms: []
+    Image Set: train
+    Metadata: {'id': 'MILCO_train', 'index2label': {0: 'MILCO', 1: 'NOMBO'}, 'split': 'train'}
+    Path: /home/user/maite-datasets/data/milco
+    Size: 261
+
+>>> print(f"type={milco[0][0].__class__.__name__}, shape={milco[0][0].shape}")
+type=ndarray, shape=(3, 1024, 1024)
+
+>>> print(milco[0][1].boxes[0])
+[ 75. 217. 130. 247.]
+
+>>> from maite_datasets.wrappers import TorchvisionWrapper
+>>> from torchvision.transforms.v2 import Resize
+
+>>> milco_torch = TorchvisionWrapper(milco, transforms=Resize(224))
+>>> print(milco_torch)
+Torchvision Wrapped MILCO Dataset
+---------------------------
+    Transforms: Resize(size=[224], interpolation=InterpolationMode.BILINEAR, antialias=True)
+
+MILCO Dataset
+-------------
+    Transforms: []
+    Image Set: train
+    Metadata: {'id': 'MILCO_train', 'index2label': {0: 'MILCO', 1: 'NOMBO'}, 'split': 'train'}
+    Path: /home/user/maite-datasets/data/milco
+    Size: 261
+
+>>> print(f"type={milco_torch[0][0].__class__.__name__}, shape={milco_torch[0][0].shape}")
+type=Image, shape=torch.Size([3, 224, 224])
+
+>>> print(milco_torch[0][1].boxes[0])
+tensor([16.4062, 47.4688, 28.4375, 54.0312], dtype=torch.float64)
+```
+
 ## Additional Information
 
 For more information on the MAITE protocol, check out their [documentation](https://mit-ll-ai-technology.github.io/maite/).

{maite_datasets-0.0.5 → maite_datasets-0.0.6}/pyproject.toml

@@ -6,6 +6,7 @@ requires-python = ">=3.9"
 dynamic = ["version"]
 dependencies = [
     "defusedxml>=0.7.1",
+    "maite>=0.7,<0.9",
     "numpy>=1.24.2",
     "pillow>=10.3.0",
     "requests>=2.32.3",
@@ -38,7 +39,9 @@ tqdm = [
 base = [
   "nox[uv]>=2025.5.1",
   "torch>=2.2.0",
-  "uv>=0.7.8",
+  "torchvision>=0.17.0",
+  "tqdm>=4.66",
+  "uv>=0.8.0",
 ]
 lint = [
   "ruff>=0.11",
@@ -59,12 +62,12 @@ dev = [
   { include-group = "lint" },
   { include-group = "test" },
   { include-group = "type" },
+  "ipykernel>=6.30.0",
 ]
 
 [tool.uv.sources]
-torch = [
-  { index = "pytorch-cpu" },
-]
+torch = [{ index = "pytorch-cpu" }]
+torchvision = [{ index = "pytorch-cpu" }]
 
 [[tool.uv.index]]
 name = "pytorch-cpu"
@@ -108,6 +111,14 @@ line-length = 120
 indent-width = 4
 target-version = "py39"
 
+[tool.ruff.lint]
+select = ["A", "ANN", "C4", "C90", "E", "F", "I", "NPY", "S", "SIM", "RET", "RUF100", "UP"]
+ignore = ["ANN401", "NPY002"]
+fixable = ["ALL"]
+unfixable = []
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+per-file-ignores = { "!src/*" = ["ANN", "S", "RET"]}
+
 [tool.ruff.lint.isort]
 known-first-party = ["maite_datasets"]
 

{maite_datasets-0.0.5 → maite_datasets-0.0.6}/src/maite_datasets/__init__.py

@@ -1,11 +1,9 @@
 """Module for MAITE compliant Computer Vision datasets."""
 
 from maite_datasets._builder import to_image_classification_dataset, to_object_detection_dataset
-from maite_datasets._collate import collate_as_torch, collate_as_numpy, collate_as_list
+from maite_datasets._collate import collate_as_list, collate_as_numpy, collate_as_torch
+from maite_datasets._reader import create_dataset_reader
 from maite_datasets._validate import validate_dataset
-from maite_datasets._reader._factory import create_dataset_reader
-from maite_datasets._reader._coco import COCODatasetReader
-from maite_datasets._reader._yolo import YOLODatasetReader
 
 __all__ = [
     "collate_as_list",
@@ -15,6 +13,4 @@ __all__ = [
     "to_image_classification_dataset",
     "to_object_detection_dataset",
     "validate_dataset",
-    "COCODatasetReader",
-    "YOLODatasetReader",
 ]

{maite_datasets-0.0.5 → maite_datasets-0.0.6}/src/maite_datasets/_base.py

@@ -2,23 +2,24 @@ from __future__ import annotations
 
 __all__ = []
 
+import inspect
+import warnings
 from abc import abstractmethod
+from collections import namedtuple
+from collections.abc import Iterator, Sequence
 from pathlib import Path
-from typing import Any, Generic, Iterator, Literal, NamedTuple, Sequence, TypeVar, cast
+from typing import Any, Callable, Generic, Literal, NamedTuple, TypeVar, cast
 
 import numpy as np
+from maite.protocols import DatasetMetadata, DatumMetadata
+from numpy.typing import NDArray
+from PIL import Image
 
 from maite_datasets._fileio import _ensure_exists
-from maite_datasets._protocols import Array, Transform
-from maite_datasets._types import (
-    AnnotatedDataset,
-    DatasetMetadata,
-    DatumMetadata,
-    ImageClassificationDataset,
-    ObjectDetectionDataset,
-    ObjectDetectionTarget,
-)
+from maite_datasets.protocols import Array
 
+_T = TypeVar("_T")
+_T_co = TypeVar("_T_co", covariant=True)
 _TArray = TypeVar("_TArray", bound=Array)
 _TTarget = TypeVar("_TTarget")
 _TRawTarget = TypeVar(
@@ -30,16 +31,7 @@ _TRawTarget = TypeVar(
 _TAnnotation = TypeVar("_TAnnotation", int, str, tuple[list[int], list[list[float]]])
 
 
-def _to_datum_metadata(index: int, metadata: dict[str, Any]) -> DatumMetadata:
-    _id = metadata.pop("id", index)
-    return DatumMetadata(id=_id, **metadata)
-
-
-class DataLocation(NamedTuple):
-    url: str
-    filename: str
-    md5: bool
-    checksum: str
+ObjectDetectionTarget = namedtuple("ObjectDetectionTarget", ["boxes", "labels", "scores"])
 
 
 class BaseDatasetMixin(Generic[_TArray]):
@@ -50,8 +42,99 @@ class BaseDatasetMixin(Generic[_TArray]):
     def _read_file(self, path: str) -> _TArray: ...
 
 
-class BaseDataset(
-    AnnotatedDataset[tuple[_TArray, _TTarget, DatumMetadata]],
+class Dataset(Generic[_T_co]):
+    """Abstract generic base class for PyTorch style Dataset"""
+
+    def __getitem__(self, index: int) -> _T_co: ...
+    def __add__(self, other: Dataset[_T_co]) -> Dataset[_T_co]: ...
+
+
+class BaseDataset(Dataset[tuple[_TArray, _TTarget, DatumMetadata]]):
+    metadata: DatasetMetadata
+
+    def __init__(
+        self,
+        transforms: Callable[[_TArray], _TArray]
+        | Callable[
+            [tuple[_TArray, _TTarget, DatumMetadata]],
+            tuple[_TArray, _TTarget, DatumMetadata],
+        ]
+        | Sequence[
+            Callable[[_TArray], _TArray]
+            | Callable[
+                [tuple[_TArray, _TTarget, DatumMetadata]],
+                tuple[_TArray, _TTarget, DatumMetadata],
+            ]
+        ]
+        | None,
+    ) -> None:
+        self.transforms: Sequence[
+            Callable[
+                [tuple[_TArray, _TTarget, DatumMetadata]],
+                tuple[_TArray, _TTarget, DatumMetadata],
+            ]
+        ] = []
+        transforms = transforms if isinstance(transforms, Sequence) else [transforms] if transforms else []
+        for transform in transforms:
+            sig = inspect.signature(transform)
+            if len(sig.parameters) != 1:
+                warnings.warn(f"Dropping unrecognized transform: {str(transform)}")
+            elif "tuple" in str(sig.parameters.values()):
+                transform = cast(
+                    Callable[
+                        [tuple[_TArray, _TTarget, DatumMetadata]],
+                        tuple[_TArray, _TTarget, DatumMetadata],
+                    ],
+                    transform,
+                )
+                self.transforms.append(transform)
+            else:
+                transform = cast(Callable[[_TArray], _TArray], transform)
+                self.transforms.append(self._wrap_transform(transform))
+
+    def _wrap_transform(
+        self, transform: Callable[[_TArray], _TArray]
+    ) -> Callable[
+        [tuple[_TArray, _TTarget, DatumMetadata]],
+        tuple[_TArray, _TTarget, DatumMetadata],
+    ]:
+        def wrapper(
+            datum: tuple[_TArray, _TTarget, DatumMetadata],
+        ) -> tuple[_TArray, _TTarget, DatumMetadata]:
+            image, target, metadata = datum
+            return (transform(image), target, metadata)
+
+        return wrapper
+
+    def _transform(self, datum: tuple[_TArray, _TTarget, DatumMetadata]) -> tuple[_TArray, _TTarget, DatumMetadata]:
+        """Function to transform the image prior to returning based on parameters passed in."""
+        for transform in self.transforms:
+            datum = transform(datum)
+        return datum
+
+    def __len__(self) -> int: ...
+
+    def __str__(self) -> str:
+        nt = "\n    "
+        title = f"{self.__class__.__name__.replace('Dataset', '')} Dataset"
+        sep = "-" * len(title)
+        attrs = [
+            f"{' '.join(w.capitalize() for w in k.split('_'))}: {v}"
+            for k, v in self.__dict__.items()
+            if not k.startswith("_")
+        ]
+        return f"{title}\n{sep}{nt}{nt.join(attrs)}"
+
+
+class DataLocation(NamedTuple):
+    url: str
+    filename: str
+    md5: bool
+    checksum: str
+
+
+class BaseDownloadedDataset(
+    BaseDataset[_TArray, _TTarget],
     Generic[_TArray, _TTarget, _TRawTarget, _TAnnotation],
 ):
     """
@@ -72,13 +155,24 @@ class BaseDataset(
         self,
         root: str | Path,
         image_set: Literal["train", "val", "test", "operational", "base"] = "train",
-        transforms: Transform[_TArray] | Sequence[Transform[_TArray]] | None = None,
+        transforms: Callable[[_TArray], _TArray]
+        | Callable[
+            [tuple[_TArray, _TTarget, DatumMetadata]],
+            tuple[_TArray, _TTarget, DatumMetadata],
+        ]
+        | Sequence[
+            Callable[[_TArray], _TArray]
+            | Callable[
+                [tuple[_TArray, _TTarget, DatumMetadata]],
+                tuple[_TArray, _TTarget, DatumMetadata],
+            ]
+        ]
+        | None = None,
         download: bool = False,
         verbose: bool = False,
     ) -> None:
+        super().__init__(transforms)
         self._root: Path = root.absolute() if isinstance(root, Path) else Path(root).absolute()
-        transforms = transforms if transforms is not None else []
-        self.transforms: Sequence[Transform[_TArray]] = transforms if isinstance(transforms, Sequence) else [transforms]
         self.image_set = image_set
         self._verbose = verbose
 
@@ -91,9 +185,11 @@ class BaseDataset(
         self._label2index = {v: k for k, v in self.index2label.items()}
 
         self.metadata: DatasetMetadata = DatasetMetadata(
-            id=self._unique_id(),
-            index2label=self.index2label,
-            split=self.image_set,
+            **{
+                "id": self._unique_id(),
+                "index2label": self.index2label,
+                "split": self.image_set,
+            }
         )
 
         # Load the data
@@ -101,13 +197,6 @@ class BaseDataset(
         self._filepaths, self._targets, self._datum_metadata = self._load_data()
         self.size: int = len(self._filepaths)
 
-    def __str__(self) -> str:
-        nt = "\n    "
-        title = f"{self.__class__.__name__} Dataset"
-        sep = "-" * len(title)
-        attrs = [f"{k.capitalize()}: {v}" for k, v in self.__dict__.items() if not k.startswith("_")]
-        return f"{title}\n{sep}{nt}{nt.join(attrs)}"
-
     @property
     def label2index(self) -> dict[str, int]:
         return self._label2index
@@ -148,20 +237,18 @@ class BaseDataset(
     @abstractmethod
     def _load_data_inner(self) -> tuple[list[str], _TRawTarget, dict[str, Any]]: ...
 
-    def _transform(self, image: _TArray) -> _TArray:
-        """Function to transform the image prior to returning based on parameters passed in."""
-        for transform in self.transforms:
-            image = transform(image)
-        return image
+    def _to_datum_metadata(self, index: int, metadata: dict[str, Any]) -> DatumMetadata:
+        _id = metadata.pop("id", index)
+        return DatumMetadata(id=_id, **metadata)
 
     def __len__(self) -> int:
         return self.size
 
 
 class BaseICDataset(
-    BaseDataset[_TArray, _TArray, list[int], int],
+    BaseDownloadedDataset[_TArray, _TArray, list[int], int],
     BaseDatasetMixin[_TArray],
-    ImageClassificationDataset[_TArray],
+    BaseDataset[_TArray, _TArray],
 ):
     """
     Base class for image classification datasets.
@@ -184,17 +271,16 @@ class BaseICDataset(
         score = self._one_hot_encode(label)
         # Get the image
         img = self._read_file(self._filepaths[index])
-        img = self._transform(img)
 
         img_metadata = {key: val[index] for key, val in self._datum_metadata.items()}
 
-        return img, score, _to_datum_metadata(index, img_metadata)
+        return self._transform((img, score, self._to_datum_metadata(index, img_metadata)))
 
 
 class BaseODDataset(
-    BaseDataset[_TArray, ObjectDetectionTarget[_TArray], _TRawTarget, _TAnnotation],
+    BaseDownloadedDataset[_TArray, ObjectDetectionTarget, _TRawTarget, _TAnnotation],
     BaseDatasetMixin[_TArray],
-    ObjectDetectionDataset[_TArray],
+    BaseDataset[_TArray, ObjectDetectionTarget],
 ):
     """
     Base class for object detection datasets.
@@ -202,7 +288,7 @@ class BaseODDataset(
 
     _bboxes_per_size: bool = False
 
-    def __getitem__(self, index: int) -> tuple[_TArray, ObjectDetectionTarget[_TArray], DatumMetadata]:
+    def __getitem__(self, index: int) -> tuple[_TArray, ObjectDetectionTarget, DatumMetadata]:
         """
         Args
         ----
@@ -211,7 +297,7 @@ class BaseODDataset(
 
         Returns
         -------
-        tuple[TArray, ObjectDetectionTarget[TArray], DatumMetadata]
+        tuple[TArray, ObjectDetectionTarget, DatumMetadata]
             Image, target, datum_metadata - target.boxes returns boxes in x0, y0, x1, y1 format
         """
         # Grab the bounding boxes and labels from the annotations
@@ -220,17 +306,49 @@ class BaseODDataset(
         # Get the image
         img = self._read_file(self._filepaths[index])
         img_size = img.shape
-        img = self._transform(img)
         # Adjust labels if necessary
         if self._bboxes_per_size and boxes:
-            boxes = boxes * np.array([[img_size[1], img_size[2], img_size[1], img_size[2]]])
+            boxes = boxes * np.asarray([[img_size[1], img_size[2], img_size[1], img_size[2]]])
         # Create the Object Detection Target
         target = ObjectDetectionTarget(self._as_array(boxes), self._as_array(labels), self._one_hot_encode(labels))
 
         img_metadata = {key: val[index] for key, val in self._datum_metadata.items()}
         img_metadata = img_metadata | additional_metadata
 
-        return img, target, _to_datum_metadata(index, img_metadata)
+        return self._transform((img, target, self._to_datum_metadata(index, img_metadata)))
 
     @abstractmethod
     def _read_annotations(self, annotation: _TAnnotation) -> tuple[list[list[float]], list[int], dict[str, Any]]: ...
+
+
+NumpyArray = NDArray[np.floating[Any]] | NDArray[np.integer[Any]]
+
+
+class BaseDatasetNumpyMixin(BaseDatasetMixin[NumpyArray]):
+    def _as_array(self, raw: list[Any]) -> NumpyArray:
+        return np.asarray(raw)
+
+    def _one_hot_encode(self, value: int | list[int]) -> NumpyArray:
+        if isinstance(value, int):
+            encoded = np.zeros(len(self.index2label))
+            encoded[value] = 1
+        else:
+            encoded = np.zeros((len(value), len(self.index2label)))
+            encoded[np.arange(len(value)), value] = 1
+        return encoded
+
+    def _read_file(self, path: str) -> NumpyArray:
+        return np.array(Image.open(path)).transpose(2, 0, 1)
+
+
+NumpyImageTransform = Callable[[NumpyArray], NumpyArray]
+NumpyImageClassificationDatumTransform = Callable[
+    [tuple[NumpyArray, NumpyArray, DatumMetadata]],
+    tuple[NumpyArray, NumpyArray, DatumMetadata],
+]
+NumpyObjectDetectionDatumTransform = Callable[
+    [tuple[NumpyArray, ObjectDetectionTarget, DatumMetadata]],
+    tuple[NumpyArray, ObjectDetectionTarget, DatumMetadata],
+]
+NumpyImageClassificationTransform = NumpyImageTransform | NumpyImageClassificationDatumTransform
+NumpyObjectDetectionTransform = NumpyImageTransform | NumpyObjectDetectionDatumTransform