maite-datasets 0.0.1__tar.gz → 0.0.3__tar.gz

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/.gitignore

@@ -10,4 +10,5 @@ wheels/
 .venv
 
 # Test output
+.nox/
 output/

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: maite-datasets
-Version: 0.0.1
+Version: 0.0.3
 Summary: A collection of Image Classification and Object Detection task datasets conforming to the MAITE protocol.
 Author-email: Andrew Weng <andrew.weng@ariacoustics.com>, Ryan Wood <ryan.wood@ariacoustics.com>, Shaun Jullens <shaun.jullens@ariacoustics.com>
 License-Expression: MIT

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/pyproject.toml

@@ -97,6 +97,20 @@ fail_under = 90
 [tool.codespell]
 skip = './*env*,./output,uv.lock'
 
+[tool.ruff]
+exclude = [
+  ".github",
+  ".vscode",
+  "*env*",
+  ".nox",
+]
+line-length = 120
+indent-width = 4
+target-version = "py39"
+
+[tool.ruff.lint.isort]
+known-first-party = ["maite_datasets"]
+
 [tool.hatch.build.targets.wheel]
 packages = ["src/maite_datasets"]
 

maite_datasets-0.0.3/src/maite_datasets/__init__.py

@@ -0,0 +1,14 @@
+"""Module for MAITE compliant Computer Vision datasets."""
+
+from maite_datasets._builder import to_image_classification_dataset, to_object_detection_dataset
+from maite_datasets._collate import collate_as_torch, collate_as_numpy, collate_as_list
+from maite_datasets._validate import validate_dataset
+
+__all__ = [
+    "collate_as_list",
+    "collate_as_numpy",
+    "collate_as_torch",
+    "to_image_classification_dataset",
+    "to_object_detection_dataset",
+    "validate_dataset",
+]

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/_base.py

@@ -76,13 +76,9 @@ class BaseDataset(
         download: bool = False,
         verbose: bool = False,
     ) -> None:
-        self._root: Path = (
-            root.absolute() if isinstance(root, Path) else Path(root).absolute()
-        )
+        self._root: Path = root.absolute() if isinstance(root, Path) else Path(root).absolute()
         transforms = transforms if transforms is not None else []
-        self.transforms: Sequence[Transform[_TArray]] = (
-            transforms if isinstance(transforms, Sequence) else [transforms]
-        )
+        self.transforms: Sequence[Transform[_TArray]] = transforms if isinstance(transforms, Sequence) else [transforms]
         self.image_set = image_set
         self._verbose = verbose
 
@@ -109,11 +105,7 @@ class BaseDataset(
         nt = "\n    "
         title = f"{self.__class__.__name__} Dataset"
         sep = "-" * len(title)
-        attrs = [
-            f"{k.capitalize()}: {v}"
-            for k, v in self.__dict__.items()
-            if not k.startswith("_")
-        ]
+        attrs = [f"{k.capitalize()}: {v}" for k, v in self.__dict__.items() if not k.startswith("_")]
         return f"{title}\n{sep}{nt}{nt.join(attrs)}"
 
     @property
@@ -149,9 +141,7 @@ class BaseDataset(
             if self._verbose:
                 print("No download needed, loaded data successfully.")
         except FileNotFoundError:
-            _ensure_exists(
-                *self._resource, self.path, self._root, self._download, self._verbose
-            )
+            _ensure_exists(*self._resource, self.path, self._root, self._download, self._verbose)
             result = self._load_data_inner()
         return result
 
@@ -212,9 +202,7 @@ class BaseODDataset(
 
     _bboxes_per_size: bool = False
 
-    def __getitem__(
-        self, index: int
-    ) -> tuple[_TArray, ObjectDetectionTarget[_TArray], DatumMetadata]:
+    def __getitem__(self, index: int) -> tuple[_TArray, ObjectDetectionTarget[_TArray], DatumMetadata]:
         """
         Args
         ----
@@ -235,13 +223,9 @@ class BaseODDataset(
         img = self._transform(img)
         # Adjust labels if necessary
         if self._bboxes_per_size and boxes:
-            boxes = boxes * np.array(
-                [[img_size[1], img_size[2], img_size[1], img_size[2]]]
-            )
+            boxes = boxes * np.array([[img_size[1], img_size[2], img_size[1], img_size[2]]])
         # Create the Object Detection Target
-        target = ObjectDetectionTarget(
-            self._as_array(boxes), self._as_array(labels), self._one_hot_encode(labels)
-        )
+        target = ObjectDetectionTarget(self._as_array(boxes), self._as_array(labels), self._one_hot_encode(labels))
 
         img_metadata = {key: val[index] for key, val in self._datum_metadata.items()}
         img_metadata = img_metadata | additional_metadata
@@ -249,6 +233,4 @@ class BaseODDataset(
         return img, target, _to_datum_metadata(index, img_metadata)
 
     @abstractmethod
-    def _read_annotations(
-        self, annotation: _TAnnotation
-    ) -> tuple[list[list[float]], list[int], dict[str, Any]]: ...
+    def _read_annotations(self, annotation: _TAnnotation) -> tuple[list[list[float]], list[int], dict[str, Any]]: ...

maite_datasets-0.0.3/src/maite_datasets/_builder.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+import numpy as np
+
+__all__ = []
+
+from typing import (
+    Any,
+    Generic,
+    Iterable,
+    Literal,
+    Sequence,
+    SupportsFloat,
+    SupportsInt,
+    TypeVar,
+    cast,
+)
+
+from maite_datasets._protocols import (
+    Array,
+    ArrayLike,
+    DatasetMetadata,
+    ImageClassificationDataset,
+    ObjectDetectionDataset,
+)
+
+
+def _ensure_id(index: int, metadata: dict[str, Any]) -> dict[str, Any]:
+    return {"id": index, **metadata} if "id" not in metadata else metadata
+
+
+def _validate_data(
+    datum_type: Literal["ic", "od"],
+    images: Array | Sequence[Array],
+    labels: Array | Sequence[int] | Sequence[Array] | Sequence[Sequence[int]],
+    bboxes: Array | Sequence[Array] | Sequence[Sequence[Array]] | Sequence[Sequence[Sequence[float]]] | None,
+    metadata: Sequence[dict[str, Any]] | dict[str, Sequence[Any]] | None,
+) -> None:
+    # Validate inputs
+    dataset_len = len(images)
+
+    if not isinstance(images, (Sequence, Array)) or len(images[0].shape) != 3:
+        raise ValueError("Images must be a sequence or array of 3 dimensional arrays (H, W, C).")
+    if len(labels) != dataset_len:
+        raise ValueError(f"Number of labels ({len(labels)}) does not match number of images ({dataset_len}).")
+    if bboxes is not None and len(bboxes) != dataset_len:
+        raise ValueError(f"Number of bboxes ({len(bboxes)}) does not match number of images ({dataset_len}).")
+    if metadata is not None and (
+        len(metadata) != dataset_len
+        if isinstance(metadata, Sequence)
+        else any(
+            not isinstance(metadatum, Sequence) or len(metadatum) != dataset_len for metadatum in metadata.values()
+        )
+    ):
+        raise ValueError(f"Number of metadata ({len(metadata)}) does not match number of images ({dataset_len}).")
+
+    if datum_type == "ic":
+        if not isinstance(labels, (Sequence, Array)) or not isinstance(labels[0], (int, SupportsInt)):
+            raise TypeError("Labels must be a sequence of integers for image classification.")
+    elif datum_type == "od":
+        if (
+            not isinstance(labels, (Sequence, Array))
+            or not isinstance(labels[0], (Sequence, Array))
+            or not isinstance(cast(Sequence[Any], labels[0])[0], (int, SupportsInt))
+        ):
+            raise TypeError("Labels must be a sequence of sequences of integers for object detection.")
+        if (
+            bboxes is None
+            or not isinstance(bboxes, (Sequence, Array))
+            or not isinstance(bboxes[0], (Sequence, Array))
+            or not isinstance(bboxes[0][0], (Sequence, Array))
+            or not isinstance(bboxes[0][0][0], (float, SupportsFloat))
+            or not len(bboxes[0][0]) == 4
+        ):
+            raise TypeError("Boxes must be a sequence of sequences of (x0, y0, x1, y1) for object detection.")
+    else:
+        raise ValueError(f"Unknown datum type '{datum_type}'. Must be 'ic' or 'od'.")
+
+
+def _listify_metadata(
+    metadata: Sequence[dict[str, Any]] | dict[str, Sequence[Any]] | None,
+) -> Sequence[dict[str, Any]] | None:
+    if isinstance(metadata, dict):
+        return [{k: v[i] for k, v in metadata.items()} for i in range(len(next(iter(metadata.values()))))]
+    return metadata
+
+
+def _find_max(arr: ArrayLike) -> Any:
+    if not isinstance(arr, (bytes, str)) and isinstance(arr, (Iterable, Sequence, Array)):
+        nested = [x for x in [_find_max(x) for x in arr] if x is not None]
+        return max(nested) if len(nested) > 0 else None
+    return arr
+
+
+_TLabels = TypeVar("_TLabels", Sequence[int], Sequence[Sequence[int]])
+
+
+class BaseAnnotatedDataset(Generic[_TLabels]):
+    def __init__(
+        self,
+        datum_type: Literal["ic", "od"],
+        images: Array | Sequence[Array],
+        labels: _TLabels,
+        metadata: Sequence[dict[str, Any]] | None,
+        classes: Sequence[str] | None,
+        name: str | None = None,
+    ) -> None:
+        self._classes = classes if classes is not None else [str(i) for i in range(_find_max(labels) + 1)]
+        self._index2label = dict(enumerate(self._classes))
+        self._images = images
+        self._labels = labels
+        self._metadata = metadata
+        self._id = name or f"{len(self._images)}_image_{len(self._index2label)}_class_{datum_type}_dataset"
+
+    @property
+    def metadata(self) -> DatasetMetadata:
+        return DatasetMetadata(id=self._id, index2label=self._index2label)
+
+    def __len__(self) -> int:
+        return len(self._images)
+
+
+class CustomImageClassificationDataset(BaseAnnotatedDataset[Sequence[int]], ImageClassificationDataset):
+    def __init__(
+        self,
+        images: Array | Sequence[Array],
+        labels: Array | Sequence[int],
+        metadata: Sequence[dict[str, Any]] | None,
+        classes: Sequence[str] | None,
+        name: str | None = None,
+    ) -> None:
+        super().__init__(
+            "ic",
+            images,
+            np.asarray(labels).tolist() if isinstance(labels, Array) else labels,
+            metadata,
+            classes,
+        )
+        if name is not None:
+            self.__name__ = name
+            self.__class__.__name__ = name
+            self.__class__.__qualname__ = name
+
+    def __getitem__(self, idx: int, /) -> tuple[Array, Array, dict[str, Any]]:
+        one_hot = [0.0] * len(self._index2label)
+        one_hot[self._labels[idx]] = 1.0
+        return (
+            self._images[idx],
+            np.asarray(one_hot),
+            _ensure_id(idx, self._metadata[idx] if self._metadata is not None else {}),
+        )
+
+
+class CustomObjectDetectionDataset(BaseAnnotatedDataset[Sequence[Sequence[int]]], ObjectDetectionDataset):
+    class ObjectDetectionTarget:
+        def __init__(
+            self,
+            labels: Sequence[int],
+            bboxes: Sequence[Sequence[float]],
+            class_count: int,
+        ) -> None:
+            self._labels = labels
+            self._bboxes = bboxes
+            one_hot = [[0.0] * class_count] * len(labels)
+            for i, label in enumerate(labels):
+                one_hot[i][label] = 1.0
+            self._scores = one_hot
+
+        @property
+        def labels(self) -> Sequence[int]:
+            return self._labels
+
+        @property
+        def boxes(self) -> Sequence[Sequence[float]]:
+            return self._bboxes
+
+        @property
+        def scores(self) -> Sequence[Sequence[float]]:
+            return self._scores
+
+    def __init__(
+        self,
+        images: Array | Sequence[Array],
+        labels: Array | Sequence[Array] | Sequence[Sequence[int]],
+        bboxes: Array | Sequence[Array] | Sequence[Sequence[Array]] | Sequence[Sequence[Sequence[float]]],
+        metadata: Sequence[dict[str, Any]] | None,
+        classes: Sequence[str] | None,
+        name: str | None = None,
+    ) -> None:
+        super().__init__(
+            "od",
+            images,
+            [np.asarray(label).tolist() if isinstance(label, Array) else label for label in labels],
+            metadata,
+            classes,
+        )
+        if name is not None:
+            self.__name__ = name
+            self.__class__.__name__ = name
+            self.__class__.__qualname__ = name
+        self._bboxes = [
+            [np.asarray(box).tolist() if isinstance(box, Array) else box for box in bbox] for bbox in bboxes
+        ]
+
+    @property
+    def metadata(self) -> DatasetMetadata:
+        return DatasetMetadata(id=self._id, index2label=self._index2label)
+
+    def __getitem__(self, idx: int, /) -> tuple[Array, ObjectDetectionTarget, dict[str, Any]]:
+        return (
+            self._images[idx],
+            self.ObjectDetectionTarget(self._labels[idx], self._bboxes[idx], len(self._classes)),
+            _ensure_id(idx, self._metadata[idx] if self._metadata is not None else {}),
+        )
+
+
+def to_image_classification_dataset(
+    images: Array | Sequence[Array],
+    labels: Array | Sequence[int],
+    metadata: Sequence[dict[str, Any]] | dict[str, Sequence[Any]] | None,
+    classes: Sequence[str] | None,
+    name: str | None = None,
+) -> ImageClassificationDataset:
+    """
+    Helper function to create custom ImageClassificationDataset classes.
+
+    Parameters
+    ----------
+    images : Array | Sequence[Array]
+        The images to use in the dataset.
+    labels : Array | Sequence[int]
+        The labels to use in the dataset.
+    metadata : Sequence[dict[str, Any]] | dict[str, Sequence[Any]] | None
+        The metadata to use in the dataset.
+    classes : Sequence[str] | None
+        The classes to use in the dataset.
+
+    Returns
+    -------
+    ImageClassificationDataset
+    """
+    _validate_data("ic", images, labels, None, metadata)
+    return CustomImageClassificationDataset(images, labels, _listify_metadata(metadata), classes, name)
+
+
+def to_object_detection_dataset(
+    images: Array | Sequence[Array],
+    labels: Array | Sequence[Array] | Sequence[Sequence[int]],
+    bboxes: Array | Sequence[Array] | Sequence[Sequence[Array]] | Sequence[Sequence[Sequence[float]]],
+    metadata: Sequence[dict[str, Any]] | dict[str, Sequence[Any]] | None,
+    classes: Sequence[str] | None,
+    name: str | None = None,
+) -> ObjectDetectionDataset:
+    """
+    Helper function to create custom ObjectDetectionDataset classes.
+
+    Parameters
+    ----------
+    images : Array | Sequence[Array]
+        The images to use in the dataset.
+    labels : Array | Sequence[Array] | Sequence[Sequence[int]]
+        The labels to use in the dataset.
+    bboxes : Array | Sequence[Array] | Sequence[Sequence[Array]] | Sequence[Sequence[Sequence[float]]]
+        The bounding boxes (x0,y0,x1,y0) to use in the dataset.
+    metadata : Sequence[dict[str, Any]] | dict[str, Sequence[Any]] | None
+        The metadata to use in the dataset.
+    classes : Sequence[str] | None
+        The classes to use in the dataset.
+
+    Returns
+    -------
+    ObjectDetectionDataset
+    """
+    _validate_data("od", images, labels, bboxes, metadata)
+    return CustomObjectDetectionDataset(images, labels, bboxes, _listify_metadata(metadata), classes, name)

maite_datasets-0.0.3/src/maite_datasets/_collate.py

@@ -0,0 +1,112 @@
+"""
+Collate functions used with a PyTorch DataLoader to load data from MAITE compliant datasets.
+"""
+
+from __future__ import annotations
+
+__all__ = []
+
+from collections.abc import Iterable, Sequence
+from typing import Any, TypeVar, TYPE_CHECKING
+
+import numpy as np
+from numpy.typing import NDArray
+
+if TYPE_CHECKING:
+    import torch
+
+from maite_datasets._protocols import ArrayLike
+
+T_in = TypeVar("T_in")
+T_tgt = TypeVar("T_tgt")
+T_md = TypeVar("T_md")
+
+
+def collate_as_list(
+    batch_data_as_singles: Iterable[tuple[T_in, T_tgt, T_md]],
+) -> tuple[Sequence[T_in], Sequence[T_tgt], Sequence[T_md]]:
+    """
+    A collate function that takes a batch of individual data points in the format
+    (input, target, metadata) and returns three lists: the input batch, the target batch,
+    and the metadata batch. This is useful for loading data with torch.utils.data.DataLoader
+    when the target and metadata are not tensors.
+
+    Parameters
+    ----------
+    batch_data_as_singles : An iterable of (input, target, metadata) tuples.
+
+    Returns
+    -------
+    tuple[Sequence[T_in], Sequence[T_tgt], Sequence[T_md]]
+        A tuple of three lists: the input batch, the target batch, and the metadata batch.
+    """
+    input_batch: list[T_in] = []
+    target_batch: list[T_tgt] = []
+    metadata_batch: list[T_md] = []
+    for input_datum, target_datum, metadata_datum in batch_data_as_singles:
+        input_batch.append(input_datum)
+        target_batch.append(target_datum)
+        metadata_batch.append(metadata_datum)
+
+    return input_batch, target_batch, metadata_batch
+
+
+def collate_as_numpy(
+    batch_data_as_singles: Iterable[tuple[ArrayLike, T_tgt, T_md]],
+) -> tuple[NDArray[Any], Sequence[T_tgt], Sequence[T_md]]:
+    """
+    A collate function that takes a batch of individual data points in the format
+    (input, target, metadata) and returns the batched input as a single NumPy array with two
+    lists: the target batch, and the metadata batch. The inputs must be homogeneous arrays.
+
+    Parameters
+    ----------
+    batch_data_as_singles : An iterable of (ArrayLike, target, metadata) tuples.
+
+    Returns
+    -------
+    tuple[NDArray[Any], Sequence[T_tgt], Sequence[T_md]]
+        A tuple of a NumPy array and two lists: the input batch, the target batch, and the metadata batch.
+    """
+    input_batch: list[NDArray[Any]] = []
+    target_batch: list[T_tgt] = []
+    metadata_batch: list[T_md] = []
+    for input_datum, target_datum, metadata_datum in batch_data_as_singles:
+        input_batch.append(np.asarray(input_datum))
+        target_batch.append(target_datum)
+        metadata_batch.append(metadata_datum)
+
+    return np.stack(input_batch) if input_batch else np.array([]), target_batch, metadata_batch
+
+
+def collate_as_torch(
+    batch_data_as_singles: Iterable[tuple[ArrayLike, T_tgt, T_md]],
+) -> tuple[torch.Tensor, Sequence[T_tgt], Sequence[T_md]]:
+    """
+    A collate function that takes a batch of individual data points in the format
+    (input, target, metadata) and returns the batched input as a single torch Tensor with two
+    lists: the target batch, and the metadata batch. The inputs must be homogeneous arrays.
+
+    Parameters
+    ----------
+    batch_data_as_singles : An iterable of (ArrayLike, target, metadata) tuples.
+
+    Returns
+    -------
+    tuple[torch.Tensor, Sequence[T_tgt], Sequence[T_md]]
+        A tuple of a torch Tensor and two lists: the input batch, the target batch, and the metadata batch.
+    """
+    try:
+        import torch
+    except ImportError:
+        raise ImportError("PyTorch is not installed. Please install it to use this function.")
+
+    input_batch: list[torch.Tensor] = []
+    target_batch: list[T_tgt] = []
+    metadata_batch: list[T_md] = []
+    for input_datum, target_datum, metadata_datum in batch_data_as_singles:
+        input_batch.append(torch.as_tensor(input_datum))
+        target_batch.append(target_datum)
+        metadata_batch.append(metadata_datum)
+
+    return torch.stack(input_batch) if input_batch else torch.tensor([]), target_batch, metadata_batch

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/_fileio.py

@@ -23,9 +23,7 @@ def _print(text: str, verbose: bool) -> None:
         print(text)
 
 
-def _validate_file(
-    fpath: Path | str, file_md5: str, md5: bool = False, chunk_size: int = 65535
-) -> bool:
+def _validate_file(fpath: Path | str, file_md5: str, md5: bool = False, chunk_size: int = 65535) -> bool:
     hasher = hashlib.md5(usedforsecurity=False) if md5 else hashlib.sha256()
     with open(fpath, "rb") as fpath_file:
         while chunk := fpath_file.read(chunk_size):
@@ -33,28 +31,20 @@ def _validate_file(
     return hasher.hexdigest() == file_md5
 
 
-def _download_dataset(
-    url: str, file_path: Path, timeout: int = 60, verbose: bool = False
-) -> None:
+def _download_dataset(url: str, file_path: Path, timeout: int = 60, verbose: bool = False) -> None:
     """Download a single resource from its URL to the `data_folder`."""
     error_msg = "URL fetch failure on {}: {} -- {}"
     try:
         response = requests.get(url, stream=True, timeout=timeout)
         response.raise_for_status()
     except requests.exceptions.HTTPError as e:
-        raise RuntimeError(
-            f"{error_msg.format(url, e.response.status_code, e.response.reason)}"
-        ) from e
+        raise RuntimeError(f"{error_msg.format(url, e.response.status_code, e.response.reason)}") from e
     except requests.exceptions.RequestException as e:
         raise ValueError(f"{error_msg.format(url, 'Unknown error', str(e))}") from e
 
     total_size = int(response.headers.get("content-length", 0))
     block_size = 8192  # 8 KB
-    progress_bar = (
-        None
-        if tqdm is None
-        else tqdm(total=total_size, unit="iB", unit_scale=True, disable=not verbose)
-    )
+    progress_bar = None if tqdm is None else tqdm(total=total_size, unit="iB", unit_scale=True, disable=not verbose)
 
     with open(file_path, "wb") as f:
         for chunk in response.iter_content(block_size):
@@ -72,9 +62,7 @@ def _extract_zip_archive(file_path: Path, extract_to: Path) -> None:
             zip_ref.extractall(extract_to)  # noqa: S202
             file_path.unlink()
     except zipfile.BadZipFile:
-        raise FileNotFoundError(
-            f"{file_path.name} is not a valid zip file, skipping extraction."
-        )
+        raise FileNotFoundError(f"{file_path.name} is not a valid zip file, skipping extraction.")
 
 
 def _extract_tar_archive(file_path: Path, extract_to: Path) -> None:
@@ -84,9 +72,7 @@ def _extract_tar_archive(file_path: Path, extract_to: Path) -> None:
             tar_ref.extractall(extract_to)  # noqa: S202
             file_path.unlink()
     except tarfile.TarError:
-        raise FileNotFoundError(
-            f"{file_path.name} is not a valid tar file, skipping extraction."
-        )
+        raise FileNotFoundError(f"{file_path.name} is not a valid tar file, skipping extraction.")
 
 
 def _extract_archive(
@@ -135,11 +121,7 @@ def _ensure_exists(
         file_ext = file_path.suffixes[0]
         compression = True
 
-    check_path = (
-        alternate_path
-        if alternate_path.exists() and not file_path.exists()
-        else file_path
-    )
+    check_path = alternate_path if alternate_path.exists() and not file_path.exists() else file_path
 
     # Download file if it doesn't exist.
     if not check_path.exists() and download:
@@ -147,9 +129,7 @@ def _ensure_exists(
         _download_dataset(url, check_path, verbose=verbose)
 
         if not _validate_file(check_path, checksum, md5):
-            raise Exception(
-                "File checksum mismatch. Remove current file and retry download."
-            )
+            raise Exception("File checksum mismatch. Remove current file and retry download.")
 
         # If the file is a zip, tar or tgz extract it into the designated folder.
         if file_ext in ARCHIVE_ENDINGS:
@@ -164,9 +144,7 @@ def _ensure_exists(
         )
     else:
         if not _validate_file(check_path, checksum, md5):
-            raise Exception(
-                "File checksum mismatch. Remove current file and retry download."
-            )
+            raise Exception("File checksum mismatch. Remove current file and retry download.")
         _print(f"{filename} already exists, skipping download.", verbose)
 
         if file_ext in ARCHIVE_ENDINGS:

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/_protocols.py

@@ -174,9 +174,7 @@ class ObjectDetectionTarget(Protocol):
     def scores(self) -> ArrayLike: ...
 
 
-ObjectDetectionDatum: TypeAlias = tuple[
-    ArrayLike, ObjectDetectionTarget, Mapping[str, Any]
-]
+ObjectDetectionDatum: TypeAlias = tuple[ArrayLike, ObjectDetectionTarget, Mapping[str, Any]]
 """
 Type alias for an object detection datum tuple.
 

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/_types.py

@@ -37,9 +37,7 @@ class AnnotatedDataset(Dataset[_TDatum]):
     def __len__(self) -> int: ...
 
 
-class ImageClassificationDataset(
-    AnnotatedDataset[tuple[_TArray, _TArray, DatumMetadata]]
-): ...
+class ImageClassificationDataset(AnnotatedDataset[tuple[_TArray, _TArray, DatumMetadata]]): ...
 
 
 @dataclass
@@ -49,6 +47,4 @@ class ObjectDetectionTarget(Generic[_TArray]):
     scores: _TArray
 
 
-class ObjectDetectionDataset(
-    AnnotatedDataset[tuple[_TArray, ObjectDetectionTarget[_TArray], DatumMetadata]]
-): ...
+class ObjectDetectionDataset(AnnotatedDataset[tuple[_TArray, ObjectDetectionTarget[_TArray], DatumMetadata]]): ...

maite_datasets-0.0.3/src/maite_datasets/_validate.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+__all__ = []
+
+import numpy as np
+from collections.abc import Sequence, Sized
+from typing import Any, Literal
+
+from maite_datasets._protocols import Array, ObjectDetectionTarget
+
+
+class ValidationMessages:
+    DATASET_SIZED = "Dataset must be sized."
+    DATASET_INDEXABLE = "Dataset must be indexable."
+    DATASET_NONEMPTY = "Dataset must be non-empty."
+    DATASET_METADATA = "Dataset must have a 'metadata' attribute."
+    DATASET_METADATA_TYPE = "Dataset metadata must be a dictionary."
+    DATASET_METADATA_FORMAT = "Dataset metadata must contain an 'id' key."
+    DATUM_TYPE = "Dataset datum must be a tuple."
+    DATUM_FORMAT = "Dataset datum must contain 3 elements: image, target, metadata."
+    DATUM_IMAGE_TYPE = "Images must be 3-dimensional arrays."
+    DATUM_IMAGE_FORMAT = "Images must be in CHW format."
+    DATUM_TARGET_IC_TYPE = "ImageClassificationDataset targets must be one-dimensional arrays."
+    DATUM_TARGET_IC_FORMAT = "ImageClassificationDataset targets must be one-hot encoded or pseudo-probabilities."
+    DATUM_TARGET_OD_TYPE = "ObjectDetectionDataset targets must be have 'boxes', 'labels' and 'scores'."
+    DATUM_TARGET_OD_LABELS_TYPE = "ObjectDetectionTarget labels must be one-dimensional (N,) arrays."
+    DATUM_TARGET_OD_BOXES_TYPE = "ObjectDetectionTarget boxes must be two-dimensional (N, 4) arrays in xxyy format."
+    DATUM_TARGET_OD_SCORES_TYPE = "ObjectDetectionTarget scores must be one (N,) or two-dimensional (N, M) arrays."
+    DATUM_TARGET_TYPE = "Target is not a valid ImageClassification or ObjectDetection target type."
+    DATUM_METADATA_TYPE = "Datum metadata must be a dictionary."
+    DATUM_METADATA_FORMAT = "Datum metadata must contain an 'id' key."
+
+
+def _validate_dataset_type(dataset: Any) -> list[str]:
+    issues = []
+    is_sized = isinstance(dataset, Sized)
+    is_indexable = hasattr(dataset, "__getitem__")
+    if not is_sized:
+        issues.append(ValidationMessages.DATASET_SIZED)
+    if not is_indexable:
+        issues.append(ValidationMessages.DATASET_INDEXABLE)
+    if is_sized and len(dataset) == 0:
+        issues.append(ValidationMessages.DATASET_NONEMPTY)
+    return issues
+
+
+def _validate_dataset_metadata(dataset: Any) -> list[str]:
+    issues = []
+    if not hasattr(dataset, "metadata"):
+        issues.append(ValidationMessages.DATASET_METADATA)
+    metadata = getattr(dataset, "metadata", None)
+    if not isinstance(metadata, dict):
+        issues.append(ValidationMessages.DATASET_METADATA_TYPE)
+    if not isinstance(metadata, dict) or "id" not in metadata:
+        issues.append(ValidationMessages.DATASET_METADATA_FORMAT)
+    return issues
+
+
+def _validate_datum_type(datum: Any) -> list[str]:
+    issues = []
+    if not isinstance(datum, tuple):
+        issues.append(ValidationMessages.DATUM_TYPE)
+    if datum is None or isinstance(datum, Sized) and len(datum) != 3:
+        issues.append(ValidationMessages.DATUM_FORMAT)
+    return issues
+
+
+def _validate_datum_image(image: Any) -> list[str]:
+    issues = []
+    if not isinstance(image, Array) or len(image.shape) != 3:
+        issues.append(ValidationMessages.DATUM_IMAGE_TYPE)
+    if (
+        not isinstance(image, Array)
+        or len(image.shape) == 3
+        and (image.shape[0] > image.shape[1] or image.shape[0] > image.shape[2])
+    ):
+        issues.append(ValidationMessages.DATUM_IMAGE_FORMAT)
+    return issues
+
+
+def _validate_datum_target_ic(target: Any) -> list[str]:
+    issues = []
+    if not isinstance(target, Array) or len(target.shape) != 1:
+        issues.append(ValidationMessages.DATUM_TARGET_IC_TYPE)
+    if target is None or sum(target) > 1 + 1e-6 or sum(target) < 1 - 1e-6:
+        issues.append(ValidationMessages.DATUM_TARGET_IC_FORMAT)
+    return issues
+
+
+def _validate_datum_target_od(target: Any) -> list[str]:
+    issues = []
+    if not isinstance(target, ObjectDetectionTarget):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_TYPE)
+    od_target: ObjectDetectionTarget | None = target if isinstance(target, ObjectDetectionTarget) else None
+    if od_target is None or len(np.asarray(od_target.labels).shape) != 1:
+        issues.append(ValidationMessages.DATUM_TARGET_OD_LABELS_TYPE)
+    if (
+        od_target is None
+        or len(np.asarray(od_target.boxes).shape) != 2
+        or (len(np.asarray(od_target.boxes).shape) == 2 and np.asarray(od_target.boxes).shape[1] != 4)
+    ):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_BOXES_TYPE)
+    if od_target is None or len(np.asarray(od_target.scores).shape) not in (1, 2):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_SCORES_TYPE)
+    return issues
+
+
+def _detect_target_type(target: Any) -> Literal["ic", "od", "auto"]:
+    if isinstance(target, Array):
+        return "ic"
+    if isinstance(target, ObjectDetectionTarget):
+        return "od"
+    return "auto"
+
+
+def _validate_datum_target(target: Any, target_type: Literal["ic", "od", "auto"]) -> list[str]:
+    issues = []
+    target_type = _detect_target_type(target) if target_type == "auto" else target_type
+    if target_type == "ic":
+        issues.extend(_validate_datum_target_ic(target))
+    elif target_type == "od":
+        issues.extend(_validate_datum_target_od(target))
+    else:
+        issues.append(ValidationMessages.DATUM_TARGET_TYPE)
+    return issues
+
+
+def _validate_datum_metadata(metadata: Any) -> list[str]:
+    issues = []
+    if metadata is None or not isinstance(metadata, dict):
+        issues.append(ValidationMessages.DATUM_METADATA_TYPE)
+    if metadata is None or isinstance(metadata, dict) and "id" not in metadata:
+        issues.append(ValidationMessages.DATUM_METADATA_FORMAT)
+    return issues
+
+
+def validate_dataset(dataset: Any, dataset_type: Literal["ic", "od", "auto"] = "auto") -> None:
+    """
+    Validate a dataset for compliance with MAITE protocol.
+
+    Parameters
+    ----------
+    dataset: Any
+        Dataset to validate.
+    dataset_type: "ic", "od", or "auto", default "auto"
+        Dataset type, if known.
+
+    Raises
+    ------
+    ValueError
+        Raises exception if dataset is invalid with a list of validation issues.
+    """
+    issues = []
+    issues.extend(_validate_dataset_type(dataset))
+    datum = None if issues else dataset[0]  # type: ignore
+    issues.extend(_validate_dataset_metadata(dataset))
+    issues.extend(_validate_datum_type(datum))
+
+    is_seq = isinstance(datum, Sequence)
+    datum_len = len(datum) if is_seq else 0
+    image = datum[0] if is_seq and datum_len > 0 else None
+    target = datum[1] if is_seq and datum_len > 1 else None
+    metadata = datum[2] if is_seq and datum_len > 2 else None
+    issues.extend(_validate_datum_image(image))
+    issues.extend(_validate_datum_target(target, dataset_type))
+    issues.extend(_validate_datum_metadata(metadata))
+
+    if issues:
+        raise ValueError("Dataset validation issues found:\n - " + "\n - ".join(issues))

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/image_classification/_cifar10.py

@@ -24,9 +24,7 @@ CIFARClassStringMap = Literal[
     "ship",
     "truck",
 ]
-TCIFARClassMap = TypeVar(
-    "TCIFARClassMap", CIFARClassStringMap, int, list[CIFARClassStringMap], list[int]
-)
+TCIFARClassMap = TypeVar("TCIFARClassMap", CIFARClassStringMap, int, list[CIFARClassStringMap], list[int])
 
 
 class CIFAR10(BaseICDataset[NDArray[np.number[Any]]], BaseDatasetNumpyMixin):
@@ -91,9 +89,7 @@ class CIFAR10(BaseICDataset[NDArray[np.number[Any]]], BaseDatasetNumpyMixin):
         self,
         root: str | Path,
         image_set: Literal["train", "test", "base"] = "train",
-        transforms: Transform[NDArray[np.number[Any]]]
-        | Sequence[Transform[NDArray[np.number[Any]]]]
-        | None = None,
+        transforms: Transform[NDArray[np.number[Any]]] | Sequence[Transform[NDArray[np.number[Any]]]] | None = None,
         download: bool = False,
         verbose: bool = False,
     ) -> None:
@@ -105,9 +101,7 @@ class CIFAR10(BaseICDataset[NDArray[np.number[Any]]], BaseDatasetNumpyMixin):
             verbose,
         )
 
-    def _load_bin_data(
-        self, data_folder: list[Path]
-    ) -> tuple[list[str], list[int], dict[str, Any]]:
+    def _load_bin_data(self, data_folder: list[Path]) -> tuple[list[str], list[int], dict[str, Any]]:
         batch_nums = np.zeros(60000, dtype=np.uint8)
         all_labels = np.zeros(60000, dtype=np.uint8)
         all_images = np.zeros((60000, 3, 32, 32), dtype=np.uint8)
@@ -115,9 +109,7 @@ class CIFAR10(BaseICDataset[NDArray[np.number[Any]]], BaseDatasetNumpyMixin):
         for batch_file in data_folder:
             # Get batch parameters
             batch_type = "test" if "test" in batch_file.stem else "train"
-            batch_num = (
-                5 if batch_type == "test" else int(batch_file.stem.split("_")[-1]) - 1
-            )
+            batch_num = 5 if batch_type == "test" else int(batch_file.stem.split("_")[-1]) - 1
 
             # Load data
             batch_images, batch_labels = self._unpack_batch_files(batch_file)
@@ -193,9 +185,7 @@ class CIFAR10(BaseICDataset[NDArray[np.number[Any]]], BaseDatasetNumpyMixin):
             {"batch_num": batch_nums.tolist()},
         )
 
-    def _unpack_batch_files(
-        self, file_path: Path
-    ) -> tuple[NDArray[np.uint8], NDArray[np.uint8]]:
+    def _unpack_batch_files(self, file_path: Path) -> tuple[NDArray[np.uint8], NDArray[np.uint8]]:
         # Load pickle data with latin1 encoding
         with file_path.open("rb") as f:
             buffer = np.frombuffer(f.read(), dtype=np.uint8)

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/image_classification/_mnist.py

@@ -12,12 +12,8 @@ from maite_datasets._base import BaseICDataset, DataLocation
 from maite_datasets._mixin._numpy import BaseDatasetNumpyMixin
 from maite_datasets._protocols import Transform
 
-MNISTClassStringMap = Literal[
-    "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"
-]
-TMNISTClassMap = TypeVar(
-    "TMNISTClassMap", MNISTClassStringMap, int, list[MNISTClassStringMap], list[int]
-)
+MNISTClassStringMap = Literal["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"]
+TMNISTClassMap = TypeVar("TMNISTClassMap", MNISTClassStringMap, int, list[MNISTClassStringMap], list[int])
 CorruptionStringMap = Literal[
     "identity",
     "shot_noise",
@@ -122,9 +118,7 @@ class MNIST(BaseICDataset[NDArray[np.number[Any]]], BaseDatasetNumpyMixin):
         root: str | Path,
         image_set: Literal["train", "test", "base"] = "train",
         corruption: CorruptionStringMap | None = None,
-        transforms: Transform[NDArray[np.number[Any]]]
-        | Sequence[Transform[NDArray[np.number[Any]]]]
-        | None = None,
+        transforms: Transform[NDArray[np.number[Any]]] | Sequence[Transform[NDArray[np.number[Any]]]] | None = None,
         download: bool = False,
         verbose: bool = False,
     ) -> None:
@@ -182,18 +176,12 @@ class MNIST(BaseICDataset[NDArray[np.number[Any]]], BaseDatasetNumpyMixin):
 
         return data, labels
 
-    def _grab_data(
-        self, path: Path
-    ) -> tuple[NDArray[np.number[Any]], NDArray[np.uintp]]:
+    def _grab_data(self, path: Path) -> tuple[NDArray[np.number[Any]], NDArray[np.uintp]]:
         """Function to load in the data numpy array"""
         with np.load(path, allow_pickle=True) as data_array:
             if self.image_set == "base":
-                data = np.concatenate(
-                    [data_array["x_train"], data_array["x_test"]], axis=0
-                )
-                labels = np.concatenate(
-                    [data_array["y_train"], data_array["y_test"]], axis=0
-                ).astype(np.uintp)
+                data = np.concatenate([data_array["x_train"], data_array["x_test"]], axis=0)
+                labels = np.concatenate([data_array["y_train"], data_array["y_test"]], axis=0).astype(np.uintp)
             else:
                 data, labels = (
                     data_array[f"x_{self.image_set}"],

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/image_classification/_ships.py

@@ -76,9 +76,7 @@ class Ships(BaseICDataset[NDArray[np.number[Any]]], BaseDatasetNumpyMixin):
     def __init__(
         self,
         root: str | Path,
-        transforms: Transform[NDArray[np.number[Any]]]
-        | Sequence[Transform[NDArray[np.number[Any]]]]
-        | None = None,
+        transforms: Transform[NDArray[np.number[Any]]] | Sequence[Transform[NDArray[np.number[Any]]]] | None = None,
         download: bool = False,
         verbose: bool = False,
     ) -> None:

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/object_detection/_antiuav.py

@@ -14,9 +14,7 @@ from maite_datasets._mixin._numpy import BaseDatasetNumpyMixin
 from maite_datasets._protocols import Transform
 
 
-class AntiUAVDetection(
-    BaseODDataset[NDArray[np.number[Any]], list[str], str], BaseDatasetNumpyMixin
-):
+class AntiUAVDetection(BaseODDataset[NDArray[np.number[Any]], list[str], str], BaseDatasetNumpyMixin):
     """
     A UAV detection dataset focused on detecting UAVs in natural images against large variation in backgrounds.
 
@@ -103,9 +101,7 @@ class AntiUAVDetection(
         self,
         root: str | Path,
         image_set: Literal["train", "val", "test", "base"] = "train",
-        transforms: Transform[NDArray[np.number[Any]]]
-        | Sequence[Transform[NDArray[np.number[Any]]]]
-        | None = None,
+        transforms: Transform[NDArray[np.number[Any]]] | Sequence[Transform[NDArray[np.number[Any]]]] | None = None,
         download: bool = False,
         verbose: bool = False,
     ) -> None:
@@ -128,9 +124,7 @@ class AntiUAVDetection(
 
             for resource in self._resources:
                 self._resource = resource
-                resource_filepaths, resource_targets, resource_metadata = (
-                    super()._load_data()
-                )
+                resource_filepaths, resource_targets, resource_metadata = super()._load_data()
                 filepaths.extend(resource_filepaths)
                 targets.extend(resource_targets)
                 metadata_list.append(resource_metadata)
@@ -148,9 +142,7 @@ class AntiUAVDetection(
             for resource in self._resources:
                 if self.image_set in resource.filename:
                     self._resource = resource
-                    resource_filepaths, resource_targets, resource_metadata = (
-                        super()._load_data()
-                    )
+                    resource_filepaths, resource_targets, resource_metadata = super()._load_data()
                     filepaths.extend(resource_filepaths)
                     targets.extend(resource_targets)
                     datum_metadata.update(resource_metadata)
@@ -164,17 +156,13 @@ class AntiUAVDetection(
         if not data_folder:
             raise FileNotFoundError
 
-        file_data = {
-            "image_id": [f"{resource_name}_{entry.name}" for entry in data_folder]
-        }
+        file_data = {"image_id": [f"{resource_name}_{entry.name}" for entry in data_folder]}
         data = [str(entry) for entry in data_folder]
         annotations = sorted(str(entry) for entry in (base_dir / "xml").glob("*.xml"))
 
         return data, annotations, file_data
 
-    def _read_annotations(
-        self, annotation: str
-    ) -> tuple[list[list[float]], list[int], dict[str, Any]]:
+    def _read_annotations(self, annotation: str) -> tuple[list[list[float]], list[int], dict[str, Any]]:
         """Function for extracting the info for the label and boxes"""
         boxes: list[list[float]] = []
         labels = []

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/object_detection/_milco.py

@@ -13,9 +13,7 @@ from maite_datasets._mixin._numpy import BaseDatasetNumpyMixin
 from maite_datasets._protocols import Transform
 
 
-class MILCO(
-    BaseODDataset[NDArray[np.number[Any]], list[str], str], BaseDatasetNumpyMixin
-):
+class MILCO(BaseODDataset[NDArray[np.number[Any]], list[str], str], BaseDatasetNumpyMixin):
     """
     A side-scan sonar dataset focused on mine-like object detection.
 
@@ -118,9 +116,7 @@ class MILCO(
         self,
         root: str | Path,
         image_set: Literal["train", "operational", "base"] = "train",
-        transforms: Transform[NDArray[np.number[Any]]]
-        | Sequence[Transform[NDArray[np.number[Any]]]]
-        | None = None,
+        transforms: Transform[NDArray[np.number[Any]]] | Sequence[Transform[NDArray[np.number[Any]]]] | None = None,
         download: bool = False,
         verbose: bool = False,
     ) -> None:
@@ -180,9 +176,7 @@ class MILCO(
 
         return data, annotations, file_data
 
-    def _read_annotations(
-        self, annotation: str
-    ) -> tuple[list[list[float]], list[int], dict[str, Any]]:
+    def _read_annotations(self, annotation: str) -> tuple[list[list[float]], list[int], dict[str, Any]]:
         """Function for extracting the info out of the text files"""
         labels: list[int] = []
         boxes: list[list[float]] = []

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/object_detection/_seadrone.py

@@ -313,9 +313,7 @@ class SeaDrone(
         self,
         root: str | Path,
         image_set: Literal["train", "val", "test", "base"] = "train",
-        transforms: Transform[NDArray[np.number[Any]]]
-        | Sequence[Transform[NDArray[np.number[Any]]]]
-        | None = None,
+        transforms: Transform[NDArray[np.number[Any]]] | Sequence[Transform[NDArray[np.number[Any]]]] | None = None,
         download: bool = False,
         verbose: bool = False,
     ) -> None:
@@ -365,9 +363,7 @@ class SeaDrone(
 
     def _load_data(
         self,
-    ) -> tuple[
-        list[str], list[tuple[list[int], list[list[float]]]], dict[str, list[Any]]
-    ]:
+    ) -> tuple[list[str], list[tuple[list[int], list[list[float]]]], dict[str, list[Any]]]:
         image_sets: dict[str, list[int]] = {
             "train": list(range(20)),
             "val": list(range(20, 24)),
@@ -390,9 +386,7 @@ class SeaDrone(
 
         return filepaths, list(targets), datum_metadata
 
-    def _load_images(
-        self, data_folder: Path, file_data: dict[int, dict[str, Any]]
-    ) -> dict[int, dict[str, Any]]:
+    def _load_images(self, data_folder: Path, file_data: dict[int, dict[str, Any]]) -> dict[int, dict[str, Any]]:
         for entry in data_folder.iterdir():
             if entry.is_file() and entry.suffix == ".jpg":
                 if int(entry.stem) not in file_data:
@@ -441,14 +435,10 @@ class SeaDrone(
             current_file["storage"] = source.get("folder_name", "")
 
             # Handle non-standard file metadata
-            current_file["date_time"] = (
-                file_meta.get("date_time") or meta.get("date_time") or ""
-            )
+            current_file["date_time"] = file_meta.get("date_time") or meta.get("date_time") or ""
             if "frame" in file_meta:
                 frame = file_meta["frame"][:-4]
-                current_file["frame"] = (
-                    int(frame.split("_")[-1]) if "IMG_" in frame else int(frame[3:])
-                )
+                current_file["frame"] = int(frame.split("_")[-1]) if "IMG_" in frame else int(frame[3:])
             elif "frame_no" in source:
                 current_file["frame"] = source["frame_no"]
             else:
@@ -456,9 +446,7 @@ class SeaDrone(
 
             # Grab additional metadata if available
             for output_key, (possible_keys, default) in mappings.items():
-                current_file[output_key] = next(
-                    (meta.get(key) for key in possible_keys if key in meta), default
-                )
+                current_file[output_key] = next((meta.get(key) for key in possible_keys if key in meta), default)
 
         # Retrieve the label and bounding box
         for annotation in result["annotations"]:
@@ -482,9 +470,7 @@ class SeaDrone(
 
         return file_data
 
-    def _restructure_file_data(
-        self, file_data: dict[int, dict[str, Any]]
-    ) -> dict[str, list[Any]]:
+    def _restructure_file_data(self, file_data: dict[int, dict[str, Any]]) -> dict[str, list[Any]]:
         """Restructure file_data from dictionary of dictionaries to a dictionary of lists"""
         # Get the keys from the dictionary
         all_keys = set()
@@ -501,9 +487,7 @@ class SeaDrone(
         # Create the lists
         for file_id, file_dict in file_data.items():
             restructured_data["image_id"].append(file_id)
-            restructured_data["label_box"].append(
-                (file_dict.get("label", []), file_dict.get("box", []))
-            )
+            restructured_data["label_box"].append((file_dict.get("label", []), file_dict.get("box", [])))
             for key in all_keys:
                 restructured_data[key].append(file_dict.get(key, None))
 
@@ -528,12 +512,8 @@ class SeaDrone(
                 json_name = folder
                 if json_name == "test":
                     json_name += "_nogt"
-                annotation_file = (
-                    self.path / "annotations" / f"instances_{json_name}.json"
-                )
-                file_data = self._create_per_image_annotations(
-                    annotation_file, file_data
-                )
+                annotation_file = self.path / "annotations" / f"instances_{json_name}.json"
+                file_data = self._create_per_image_annotations(annotation_file, file_data)
 
         meta_data = self._restructure_file_data(file_data)
         data = meta_data.pop("data_path")

{maite_datasets-0.0.1 → maite_datasets-0.0.3}/src/maite_datasets/object_detection/_voc.py

@@ -45,9 +45,7 @@ VOCClassStringMap = Literal[
     "train",
     "tvmonitor",
 ]
-TVOCClassMap = TypeVar(
-    "TVOCClassMap", VOCClassStringMap, int, list[VOCClassStringMap], list[int]
-)
+TVOCClassMap = TypeVar("TVOCClassMap", VOCClassStringMap, int, list[VOCClassStringMap], list[int])
 
 
 class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
@@ -170,13 +168,9 @@ class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
             base if base.stem == f"VOC{self.year}" else None,
             base / f"VOC{self.year}" if base.stem == "VOCdevkit" else None,
             base / "VOCdevkit" / f"VOC{self.year}",
-            base / "TrainVal" / "VOCdevkit" / f"VOC{self.year}"
-            if self.year == "2011"
-            else None,
+            base / "TrainVal" / "VOCdevkit" / f"VOC{self.year}" if self.year == "2011" else None,
             dataset_dir / "VOCdevkit" / f"VOC{self.year}",
-            dataset_dir / "TrainVal" / "VOCdevkit" / f"VOC{self.year}"
-            if self.year == "2011"
-            else None,
+            dataset_dir / "TrainVal" / "VOCdevkit" / f"VOC{self.year}" if self.year == "2011" else None,
         ]
 
         # Filter out None values and check each path
@@ -269,9 +263,7 @@ class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
 
             for img_set in ["test", "base"]:
                 self.image_set = img_set
-                resource_filepaths, resource_targets, resource_metadata = (
-                    self._load_data_inner()
-                )
+                resource_filepaths, resource_targets, resource_metadata = self._load_data_inner()
                 filepaths.extend(resource_filepaths)
                 targets.extend(resource_targets)
                 metadata_list.append(resource_metadata)
@@ -288,14 +280,10 @@ class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
             self._resource = self._resources[resource_idx[1]]
 
             if train_exists and not test_exists:
-                _ensure_exists(
-                    *self._resource, tmp_path, self._root, self._download, self._verbose
-                )
+                _ensure_exists(*self._resource, tmp_path, self._root, self._download, self._verbose)
                 self._merge_voc_directories(tmp_path)
 
-            resource_filepaths, resource_targets, resource_metadata = (
-                self._load_try_and_update()
-            )
+            resource_filepaths, resource_targets, resource_metadata = self._load_try_and_update()
             filepaths.extend(resource_filepaths)
             targets.extend(resource_targets)
             datum_metadata.update(resource_metadata)
@@ -341,9 +329,7 @@ class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
             if self._verbose:
                 print("No download needed, loaded data successfully.")
         except FileNotFoundError:
-            _ensure_exists(
-                *self._resource, self.path, self._root, self._download, self._verbose
-            )
+            _ensure_exists(*self._resource, self.path, self._root, self._download, self._verbose)
             self._update_path()
             result = self._load_data_inner()
         return result
@@ -364,9 +350,7 @@ class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
     def _get_image_sets(self) -> dict[str, list[str]]:
         """Function to create the list of images in each image set"""
         image_folder = self.path / "JPEGImages"
-        image_set_list = (
-            ["train", "val", "trainval"] if self.image_set != "test" else ["test"]
-        )
+        image_set_list = ["train", "val", "trainval"] if self.image_set != "test" else ["test"]
         image_sets = {}
         for image_set in image_set_list:
             text_file = self.path / "ImageSets" / "Main" / (image_set + ".txt")
@@ -408,9 +392,7 @@ class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
 
         return data, annotations, file_meta
 
-    def _read_annotations(
-        self, annotation: str
-    ) -> tuple[list[list[float]], list[int], dict[str, Any]]:
+    def _read_annotations(self, annotation: str) -> tuple[list[list[float]], list[int], dict[str, Any]]:
         boxes: list[list[float]] = []
         label_str = []
         if not Path(annotation).exists():
@@ -435,12 +417,8 @@ class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
         for obj in root.findall("object"):
             label_str.append(obj.findtext("name", default=""))
             additional_meta["pose"].append(obj.findtext("pose", default=""))
-            additional_meta["truncated"].append(
-                int(obj.findtext("truncated", default="-1"))
-            )
-            additional_meta["difficult"].append(
-                int(obj.findtext("difficult", default="-1"))
-            )
+            additional_meta["truncated"].append(int(obj.findtext("truncated", default="-1")))
+            additional_meta["difficult"].append(int(obj.findtext("difficult", default="-1")))
             boxes.append(
                 [
                     float(obj.findtext("bndbox/xmin", default="0")),
@@ -454,9 +432,7 @@ class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str], str]):
 
 
 class VOCDetection(
-    BaseVOCDataset[
-        NDArray[np.number[Any]], ObjectDetectionTarget[NDArray[np.number[Any]]]
-    ],
+    BaseVOCDataset[NDArray[np.number[Any]], ObjectDetectionTarget[NDArray[np.number[Any]]]],
     BaseODDataset[NDArray[np.number[Any]], list[str], str],
     BaseDatasetNumpyMixin,
 ):

maite_datasets-0.0.1/src/maite_datasets/__init__.py

@@ -1 +0,0 @@
-"""Module for MAITE compliant Computer Vision datasets."""