maite-datasets 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

maite_datasets/__init__.py

@@ -1,5 +1,20 @@
 """Module for MAITE compliant Computer Vision datasets."""
 
 from maite_datasets._builder import to_image_classification_dataset, to_object_detection_dataset
+from maite_datasets._collate import collate_as_torch, collate_as_numpy, collate_as_list
+from maite_datasets._validate import validate_dataset
+from maite_datasets._reader._factory import create_dataset_reader
+from maite_datasets._reader._coco import COCODatasetReader
+from maite_datasets._reader._yolo import YOLODatasetReader
 
-__all__ = ["to_image_classification_dataset", "to_object_detection_dataset"]
+__all__ = [
+    "collate_as_list",
+    "collate_as_numpy",
+    "collate_as_torch",
+    "create_dataset_reader",
+    "to_image_classification_dataset",
+    "to_object_detection_dataset",
+    "validate_dataset",
+    "COCODatasetReader",
+    "YOLODatasetReader",
+]

maite_datasets/_collate.py

@@ -0,0 +1,112 @@
+"""
+Collate functions used with a PyTorch DataLoader to load data from MAITE compliant datasets.
+"""
+
+from __future__ import annotations
+
+__all__ = []
+
+from collections.abc import Iterable, Sequence
+from typing import Any, TypeVar, TYPE_CHECKING
+
+import numpy as np
+from numpy.typing import NDArray
+
+if TYPE_CHECKING:
+    import torch
+
+from maite_datasets._protocols import ArrayLike
+
+T_in = TypeVar("T_in")
+T_tgt = TypeVar("T_tgt")
+T_md = TypeVar("T_md")
+
+
+def collate_as_list(
+    batch_data_as_singles: Iterable[tuple[T_in, T_tgt, T_md]],
+) -> tuple[Sequence[T_in], Sequence[T_tgt], Sequence[T_md]]:
+    """
+    A collate function that takes a batch of individual data points in the format
+    (input, target, metadata) and returns three lists: the input batch, the target batch,
+    and the metadata batch. This is useful for loading data with torch.utils.data.DataLoader
+    when the target and metadata are not tensors.
+
+    Parameters
+    ----------
+    batch_data_as_singles : An iterable of (input, target, metadata) tuples.
+
+    Returns
+    -------
+    tuple[Sequence[T_in], Sequence[T_tgt], Sequence[T_md]]
+        A tuple of three lists: the input batch, the target batch, and the metadata batch.
+    """
+    input_batch: list[T_in] = []
+    target_batch: list[T_tgt] = []
+    metadata_batch: list[T_md] = []
+    for input_datum, target_datum, metadata_datum in batch_data_as_singles:
+        input_batch.append(input_datum)
+        target_batch.append(target_datum)
+        metadata_batch.append(metadata_datum)
+
+    return input_batch, target_batch, metadata_batch
+
+
+def collate_as_numpy(
+    batch_data_as_singles: Iterable[tuple[ArrayLike, T_tgt, T_md]],
+) -> tuple[NDArray[Any], Sequence[T_tgt], Sequence[T_md]]:
+    """
+    A collate function that takes a batch of individual data points in the format
+    (input, target, metadata) and returns the batched input as a single NumPy array with two
+    lists: the target batch, and the metadata batch. The inputs must be homogeneous arrays.
+
+    Parameters
+    ----------
+    batch_data_as_singles : An iterable of (ArrayLike, target, metadata) tuples.
+
+    Returns
+    -------
+    tuple[NDArray[Any], Sequence[T_tgt], Sequence[T_md]]
+        A tuple of a NumPy array and two lists: the input batch, the target batch, and the metadata batch.
+    """
+    input_batch: list[NDArray[Any]] = []
+    target_batch: list[T_tgt] = []
+    metadata_batch: list[T_md] = []
+    for input_datum, target_datum, metadata_datum in batch_data_as_singles:
+        input_batch.append(np.asarray(input_datum))
+        target_batch.append(target_datum)
+        metadata_batch.append(metadata_datum)
+
+    return np.stack(input_batch) if input_batch else np.array([]), target_batch, metadata_batch
+
+
+def collate_as_torch(
+    batch_data_as_singles: Iterable[tuple[ArrayLike, T_tgt, T_md]],
+) -> tuple[torch.Tensor, Sequence[T_tgt], Sequence[T_md]]:
+    """
+    A collate function that takes a batch of individual data points in the format
+    (input, target, metadata) and returns the batched input as a single torch Tensor with two
+    lists: the target batch, and the metadata batch. The inputs must be homogeneous arrays.
+
+    Parameters
+    ----------
+    batch_data_as_singles : An iterable of (ArrayLike, target, metadata) tuples.
+
+    Returns
+    -------
+    tuple[torch.Tensor, Sequence[T_tgt], Sequence[T_md]]
+        A tuple of a torch Tensor and two lists: the input batch, the target batch, and the metadata batch.
+    """
+    try:
+        import torch
+    except ImportError:
+        raise ImportError("PyTorch is not installed. Please install it to use this function.")
+
+    input_batch: list[torch.Tensor] = []
+    target_batch: list[T_tgt] = []
+    metadata_batch: list[T_md] = []
+    for input_datum, target_datum, metadata_datum in batch_data_as_singles:
+        input_batch.append(torch.as_tensor(input_datum))
+        target_batch.append(target_datum)
+        metadata_batch.append(metadata_datum)
+
+    return torch.stack(input_batch) if input_batch else torch.tensor([]), target_batch, metadata_batch

maite_datasets/_protocols.py

@@ -1,14 +1,14 @@
 """
-Common type protocols used for interoperability with MAITE.
+Common type protocols used for interoperability.
 """
 
+from collections.abc import Iterator
 import sys
 from typing import (
     Any,
     Generic,
-    Iterator,
-    Mapping,
     Protocol,
+    TypeAlias,
     TypedDict,
     TypeVar,
     runtime_checkable,
@@ -36,29 +36,10 @@ See Also
 @runtime_checkable
 class Array(Protocol):
     """
-    Protocol for array objects providing interoperability with DataEval.
+    Protocol for interoperable array objects.
 
     Supports common array representations with popular libraries like
     PyTorch, Tensorflow and JAX, as well as NumPy arrays.
-
-    Example
-    -------
-    >>> import numpy as np
-    >>> import torch
-    >>> from maite_datasets._typing import Array
-
-    Create array objects
-
-    >>> ndarray = np.random.random((10, 10))
-    >>> tensor = torch.tensor([1, 2, 3])
-
-    Check type at runtime
-
-    >>> isinstance(ndarray, Array)
-    True
-
-    >>> isinstance(tensor, Array)
-    True
     """
 
     @property
@@ -71,6 +52,7 @@ class Array(Protocol):
 
 _T = TypeVar("_T")
 _T_co = TypeVar("_T_co", covariant=True)
+_T_cn = TypeVar("_T_cn", contravariant=True)
 
 
 class DatasetMetadata(TypedDict, total=False):
@@ -89,6 +71,19 @@ class DatasetMetadata(TypedDict, total=False):
     index2label: NotRequired[ReadOnly[dict[int, str]]]
 
 
+class DatumMetadata(TypedDict, total=False):
+    """
+    Datum level metadata required for all `AnnotatedDataset` classes.
+
+    Attributes
+    ----------
+    id : Required[str]
+        A unique identifier for the datum
+    """
+
+    id: Required[ReadOnly[str]]
+
+
 @runtime_checkable
 class Dataset(Generic[_T_co], Protocol):
     """
@@ -134,7 +129,7 @@ class AnnotatedDataset(Dataset[_T_co], Generic[_T_co], Protocol):
 # ========== IMAGE CLASSIFICATION DATASETS ==========
 
 
-ImageClassificationDatum: TypeAlias = tuple[ArrayLike, ArrayLike, Mapping[str, Any]]
+ImageClassificationDatum: TypeAlias = tuple[ArrayLike, ArrayLike, DatumMetadata]
 """
 Type alias for an image classification datum tuple.
 
@@ -174,7 +169,7 @@ class ObjectDetectionTarget(Protocol):
     def scores(self) -> ArrayLike: ...
 
 
-ObjectDetectionDatum: TypeAlias = tuple[ArrayLike, ObjectDetectionTarget, Mapping[str, Any]]
+ObjectDetectionDatum: TypeAlias = tuple[ArrayLike, ObjectDetectionTarget, DatumMetadata]
 """
 Type alias for an object detection datum tuple.
 

maite_datasets/_reader/__init__.py

@@ -0,0 +1,6 @@
+"""
+Dataset readers for common computer vision dataset formats.
+
+This module provides standardized readers that for loading datasets
+from directory structures.
+"""

maite_datasets/_reader/_base.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+import logging
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from maite_datasets._protocols import ArrayLike, ObjectDetectionDataset
+
+_logger = logging.getLogger(__name__)
+
+
+class _ObjectDetectionTarget:
+    """Internal implementation of ObjectDetectionTarget protocol."""
+
+    def __init__(self, boxes: ArrayLike, labels: ArrayLike, scores: ArrayLike) -> None:
+        self._boxes = np.asarray(boxes)
+        self._labels = np.asarray(labels)
+        self._scores = np.asarray(scores)
+
+    @property
+    def boxes(self) -> ArrayLike:
+        return self._boxes
+
+    @property
+    def labels(self) -> ArrayLike:
+        return self._labels
+
+    @property
+    def scores(self) -> ArrayLike:
+        return self._scores
+
+
+class BaseDatasetReader(ABC):
+    """
+    Abstract base class for object detection dataset readers.
+
+    Provides common functionality for dataset path handling, validation,
+    and dataset creation while allowing format-specific implementations.
+
+    Parameters
+    ----------
+    dataset_path : str or Path
+        Root directory containing dataset files
+    dataset_id : str or None, default None
+        Dataset identifier. If None, uses dataset_path name
+    """
+
+    def __init__(self, dataset_path: str | Path, dataset_id: str | None = None) -> None:
+        self.dataset_path = Path(dataset_path)
+        self._dataset_id = dataset_id or self.dataset_path.name
+
+        # Basic path validation
+        if not self.dataset_path.exists():
+            raise FileNotFoundError(f"Dataset path not found: {self.dataset_path}")
+
+        # Format-specific initialization
+        self._initialize_format_specific()
+
+    @abstractmethod
+    def _initialize_format_specific(self) -> None:
+        """Initialize format-specific components (annotations, classes, etc.)."""
+        pass
+
+    @abstractmethod
+    def _create_dataset_implementation(self) -> ObjectDetectionDataset:
+        """Create the format-specific dataset implementation."""
+        pass
+
+    @abstractmethod
+    def _validate_format_specific(self) -> tuple[list[str], dict[str, Any]]:
+        """Validate format-specific structure and return issues and stats."""
+        pass
+
+    @property
+    @abstractmethod
+    def index2label(self) -> dict[int, str]:
+        """Mapping from class index to class name."""
+        pass
+
+    def _validate_images_directory(self) -> tuple[list[str], dict[str, Any]]:
+        """Validate images directory and return issues and stats."""
+        issues = []
+        stats = {}
+
+        images_path = self.dataset_path / "images"
+        if not images_path.exists():
+            issues.append("Missing images/ directory")
+            return issues, stats
+
+        image_files = []
+        for ext in [".jpg", ".jpeg", ".png", ".bmp"]:
+            image_files.extend(images_path.glob(f"*{ext}"))
+            image_files.extend(images_path.glob(f"*{ext.upper()}"))
+
+        stats["num_images"] = len(image_files)
+        if len(image_files) == 0:
+            issues.append("No image files found in images/ directory")
+
+        return issues, stats
+
+    def validate_structure(self) -> dict[str, Any]:
+        """
+        Validate dataset directory structure and return diagnostic information.
+
+        Returns
+        -------
+        dict[str, Any]
+            Validation results containing:
+            - is_valid: bool indicating if structure is valid
+            - issues: list of validation issues found
+            - stats: dict with dataset statistics
+        """
+        # Validate images directory (common to all formats)
+        issues, stats = self._validate_images_directory()
+
+        # Format-specific validation
+        format_issues, format_stats = self._validate_format_specific()
+        issues.extend(format_issues)
+        stats.update(format_stats)
+
+        return {"is_valid": len(issues) == 0, "issues": issues, "stats": stats}
+
+    def get_dataset(self) -> ObjectDetectionDataset:
+        """
+        Get dataset conforming to MAITE ObjectDetectionDataset protocol.
+
+        Returns
+        -------
+        ObjectDetectionDataset
+            Dataset instance with MAITE-compatible interface
+        """
+        return self._create_dataset_implementation()

maite_datasets/_reader/_coco.py

@@ -0,0 +1,287 @@
+"""Dataset reader for COCO detection format."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+from PIL import Image
+
+from maite_datasets._protocols import DatasetMetadata, DatumMetadata, ObjectDetectionDataset, ObjectDetectionDatum
+from maite_datasets._reader._base import _ObjectDetectionTarget, BaseDatasetReader
+
+_logger = logging.getLogger(__name__)
+
+
+class COCODatasetReader(BaseDatasetReader):
+    """
+    COCO format dataset reader conforming to MAITE protocols.
+
+    Reads COCO format object detection datasets from disk and provides
+    MAITE-compatible interface.
+
+    Directory Structure Requirements
+    --------------------------------
+    ```
+    dataset_root/
+    ├── images/
+    │   ├── image1.jpg
+    │   ├── image2.jpg
+    │   └── ...
+    ├── annotations.json  # COCO format annotation file
+    └── classes.txt       # Optional: one class name per line
+    ```
+
+    COCO Format Specifications
+    --------------------------
+    annotations.json structure:
+    ```json
+    {
+      "images": [
+        {
+          "id": 1,
+          "file_name": "image1.jpg",
+          "width": 640,
+          "height": 480
+        }
+      ],
+      "annotations": [
+        {
+          "id": 1,
+          "image_id": 1,
+          "category_id": 1,
+          "bbox": [100, 50, 200, 150],  // [x, y, width, height]
+          "area": 30000
+        }
+      ],
+      "categories": [
+        {
+          "id": 1,
+          "name": "person"
+        }
+      ]
+    }
+    ```
+
+    classes.txt format (optional, one class per line, ordered by index):
+    ```
+    person
+    bicycle
+    car
+    motorcycle
+    ```
+
+    Parameters
+    ----------
+    dataset_path : str or Path
+        Root directory containing COCO dataset files
+    annotation_file : str, default "annotations.json"
+        Name of COCO annotation JSON file
+    images_dir : str, default "images"
+        Name of directory containing images
+    classes_file : str or None, default "classes.txt"
+        Optional file containing class names (one per line)
+        If None, uses category names from COCO annotations
+    dataset_id : str or None, default None
+        Dataset identifier. If None, uses dataset_path name
+
+    Notes
+    -----
+    COCO annotations should follow standard COCO format with:
+    - "images": list of image metadata
+    - "annotations": list of bounding box annotations
+    - "categories": list of category definitions
+
+    Bounding boxes are converted from COCO format (x, y, width, height)
+    to MAITE format (x1, y1, x2, y2).
+    """
+
+    def __init__(
+        self,
+        dataset_path: str | Path,
+        annotation_file: str = "annotations.json",
+        images_dir: str = "images",
+        classes_file: str | None = "classes.txt",
+        dataset_id: str | None = None,
+    ) -> None:
+        self.annotation_file = annotation_file
+        self.images_dir = images_dir
+        self.classes_file = classes_file
+
+        # Initialize base class
+        super().__init__(dataset_path, dataset_id)
+
+    def _initialize_format_specific(self) -> None:
+        """Initialize COCO-specific components."""
+        self.images_path = self.dataset_path / self.images_dir
+        self.annotation_path = self.dataset_path / self.annotation_file
+        self.classes_path = self.dataset_path / self.classes_file if self.classes_file else None
+
+        if not self.annotation_path.exists():
+            raise FileNotFoundError(f"Annotation file not found: {self.annotation_path}")
+        if not self.images_path.exists():
+            raise FileNotFoundError(f"Images directory not found: {self.images_path}")
+
+        self._load_annotations()
+
+    @property
+    def index2label(self) -> dict[int, str]:
+        """Mapping from class index to class name."""
+        return self._index2label
+
+    def _create_dataset_implementation(self) -> ObjectDetectionDataset:
+        """Create COCO dataset implementation."""
+        return _COCODataset(self)
+
+    def _validate_format_specific(self) -> tuple[list[str], dict[str, Any]]:
+        """Validate COCO format specific files and structure."""
+        issues = []
+        stats = {}
+
+        annotation_path = self.dataset_path / self.annotation_file
+        if not annotation_path.exists():
+            issues.append(f"Missing {self.annotation_file} file")
+            return issues, stats
+
+        try:
+            with open(annotation_path) as f:
+                coco_data = json.load(f)
+        except json.JSONDecodeError as e:
+            issues.append(f"Invalid JSON in {self.annotation_file}: {e}")
+            return issues, stats
+
+        # Check required keys
+        required_keys = ["images", "annotations", "categories"]
+        for key in required_keys:
+            if key not in coco_data:
+                issues.append(f"Missing required key '{key}' in {self.annotation_file}")
+            else:
+                stats[f"num_{key}"] = len(coco_data[key])
+
+        # Check optional classes.txt
+        if self.classes_file:
+            classes_path = self.dataset_path / self.classes_file
+            if classes_path.exists():
+                try:
+                    with open(classes_path) as f:
+                        class_lines = [line.strip() for line in f if line.strip()]
+                    stats["num_class_names"] = len(class_lines)
+                except Exception as e:
+                    issues.append(f"Error reading {self.classes_file}: {e}")
+
+        return issues, stats
+
+    def _load_annotations(self) -> None:
+        """Load and parse COCO annotations."""
+        with open(self.annotation_path) as f:
+            self.coco_data = json.load(f)
+
+        # Build mappings
+        self.image_id_to_info = {img["id"]: img for img in self.coco_data["images"]}
+        self.category_id_to_idx = {cat["id"]: idx for idx, cat in enumerate(self.coco_data["categories"])}
+
+        # Group annotations by image
+        self.image_id_to_annotations: dict[int, list[dict[str, Any]]] = {}
+        for ann in self.coco_data["annotations"]:
+            img_id = ann["image_id"]
+            if img_id not in self.image_id_to_annotations:
+                self.image_id_to_annotations[img_id] = []
+            self.image_id_to_annotations[img_id].append(ann)
+
+        # Load class names
+        if self.classes_path and self.classes_path.exists():
+            with open(self.classes_path) as f:
+                class_names = [line.strip() for line in f if line.strip()]
+        else:
+            class_names = [cat["name"] for cat in self.coco_data["categories"]]
+
+        self._index2label = {idx: name for idx, name in enumerate(class_names)}
+
+
+class _COCODataset:
+    """Internal COCO dataset implementation."""
+
+    def __init__(self, reader: COCODatasetReader) -> None:
+        self.reader = reader
+        self.image_ids = list(reader.image_id_to_info.keys())
+
+    @property
+    def metadata(self) -> DatasetMetadata:
+        return DatasetMetadata(
+            id=self.reader._dataset_id,
+            index2label=self.reader.index2label,
+        )
+
+    def __len__(self) -> int:
+        return len(self.image_ids)
+
+    def __getitem__(self, index: int) -> ObjectDetectionDatum:
+        image_id = self.image_ids[index]
+        image_info = self.reader.image_id_to_info[image_id]
+
+        # Load image
+        image_path = self.reader.images_path / image_info["file_name"]
+        image = np.array(Image.open(image_path).convert("RGB"))
+        image = np.transpose(image, (2, 0, 1))  # Convert to CHW format
+
+        # Get annotations for this image
+        annotations = self.reader.image_id_to_annotations.get(image_id, [])
+
+        if annotations:
+            boxes = []
+            labels = []
+            annotation_metadata = []
+
+            for ann in annotations:
+                # Convert COCO bbox (x, y, w, h) to (x1, y1, x2, y2)
+                x, y, w, h = ann["bbox"]
+                boxes.append([x, y, x + w, y + h])
+
+                # Map category_id to class index
+                cat_idx = self.reader.category_id_to_idx[ann["category_id"]]
+                labels.append(cat_idx)
+
+                # Collect annotation metadata
+                ann_meta = {
+                    "annotation_id": ann["id"],
+                    "category_id": ann["category_id"],
+                    "area": ann.get("area", 0),
+                    "iscrowd": ann.get("iscrowd", 0),
+                }
+                # Add any additional fields from annotation
+                for key, value in ann.items():
+                    if key not in ["id", "image_id", "category_id", "bbox", "area", "iscrowd"]:
+                        ann_meta[f"ann_{key}"] = value
+                annotation_metadata.append(ann_meta)
+
+            boxes = np.array(boxes, dtype=np.float32)
+            labels = np.array(labels, dtype=np.int64)
+            scores = np.ones(len(labels), dtype=np.float32)  # Ground truth scores
+        else:
+            # Empty annotations
+            boxes = np.empty((0, 4), dtype=np.float32)
+            labels = np.empty(0, dtype=np.int64)
+            scores = np.empty(0, dtype=np.float32)
+            annotation_metadata = []
+
+        target = _ObjectDetectionTarget(boxes, labels, scores)
+
+        # Create comprehensive datum metadata
+        datum_metadata = DatumMetadata(
+            id=f"{self.reader._dataset_id}_{image_id}",
+            # Image-level metadata
+            coco_image_id=image_id,
+            file_name=image_info["file_name"],
+            width=image_info["width"],
+            height=image_info["height"],
+            # Optional COCO image fields
+            **{key: value for key, value in image_info.items() if key not in ["id", "file_name", "width", "height"]},
+            # Annotation metadata
+            annotations=annotation_metadata,
+            num_annotations=len(annotations),
+        )
+
+        return image, target, datum_metadata

maite_datasets/_reader/_factory.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from maite_datasets._reader._base import BaseDatasetReader
+from maite_datasets._reader._yolo import YOLODatasetReader
+from maite_datasets._reader._coco import COCODatasetReader
+
+_logger = logging.getLogger(__name__)
+
+
+def create_dataset_reader(dataset_path: str | Path, format_hint: str | None = None) -> BaseDatasetReader:
+    """
+    Factory function to create appropriate dataset reader based on directory structure.
+
+    Parameters
+    ----------
+    dataset_path : str or Path
+        Root directory containing dataset files
+    format_hint : str or None, default None
+        Format hint ("coco" or "yolo"). If None, auto-detects based on file structure
+
+    Returns
+    -------
+    BaseDatasetReader
+        Appropriate reader instance for the detected format
+
+    Raises
+    ------
+    ValueError
+        If format cannot be determined or is unsupported
+    """
+    dataset_path = Path(dataset_path)
+
+    if format_hint:
+        format_hint = format_hint.lower()
+        if format_hint == "coco":
+            return COCODatasetReader(dataset_path)
+        elif format_hint == "yolo":
+            return YOLODatasetReader(dataset_path)
+        else:
+            raise ValueError(f"Unsupported format hint: {format_hint}")
+
+    # Auto-detect format
+    has_annotations_json = (dataset_path / "annotations.json").exists()
+    has_labels_dir = (dataset_path / "labels").exists()
+
+    if has_annotations_json and not has_labels_dir:
+        _logger.info(f"Detected COCO format for {dataset_path}")
+        return COCODatasetReader(dataset_path)
+    elif has_labels_dir and not has_annotations_json:
+        _logger.info(f"Detected YOLO format for {dataset_path}")
+        return YOLODatasetReader(dataset_path)
+    elif has_annotations_json and has_labels_dir:
+        raise ValueError(
+            f"Ambiguous format in {dataset_path}: both annotations.json and labels/ exist. "
+            "Use format_hint parameter to specify format."
+        )
+    else:
+        raise ValueError(
+            f"Cannot detect dataset format in {dataset_path}. "
+            "Expected either annotations.json (COCO) or labels/ directory (YOLO)."
+        )

maite_datasets/_reader/_yolo.py

@@ -0,0 +1,312 @@
+"""Dataset reader for YOLO detection format."""
+
+from __future__ import annotations
+
+__all__ = []
+
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+from PIL import Image
+
+from maite_datasets._protocols import DatasetMetadata, DatumMetadata, ObjectDetectionDataset, ObjectDetectionDatum
+from maite_datasets._reader._base import _ObjectDetectionTarget, BaseDatasetReader
+
+
+class YOLODatasetReader(BaseDatasetReader):
+    """
+    YOLO format dataset reader conforming to MAITE protocols.
+
+    Reads YOLO format object detection datasets from disk and provides
+    MAITE-compatible interface.
+
+    Directory Structure Requirements
+    --------------------------------
+    ```
+    dataset_root/
+    ├── images/
+    │   ├── image1.jpg
+    │   ├── image2.jpg
+    │   └── ...
+    ├── labels/
+    │   ├── image1.txt    # YOLO format annotations
+    │   ├── image2.txt
+    │   └── ...
+    ├── classes.txt       # Required: one class name per line
+    └── data.yaml         # Optional: dataset metadata
+    ```
+
+    YOLO Format Specifications
+    --------------------------
+    Label file format (one line per object):
+    ```
+    class_id center_x center_y width height
+    0 0.5 0.3 0.2 0.4
+    1 0.7 0.8 0.1 0.2
+    ```
+    All YOLO coordinates are normalized to [0, 1] relative to image dimensions.
+
+    classes.txt format (required, one class per line, ordered by index):
+    ```
+    person
+    bicycle
+    car
+    motorcycle
+    ```
+
+    Parameters
+    ----------
+    dataset_path : str or Path
+        Root directory containing YOLO dataset files
+    images_dir : str, default "images"
+        Name of directory containing images
+    labels_dir : str, default "labels"
+        Name of directory containing YOLO label files
+    classes_file : str, default "classes.txt"
+        File containing class names (one per line)
+    dataset_id : str or None, default None
+        Dataset identifier. If None, uses dataset_path name
+    image_extensions : list[str], default [".jpg", ".jpeg", ".png", ".bmp"]
+        Supported image file extensions
+
+    Notes
+    -----
+    YOLO label files should contain one line per object:
+    `class_id center_x center_y width height`
+
+    All coordinates should be normalized to [0, 1] relative to image dimensions.
+    Coordinates are converted to absolute pixel values and MAITE format (x1, y1, x2, y2).
+    """
+
+    def __init__(
+        self,
+        dataset_path: str | Path,
+        images_dir: str = "images",
+        labels_dir: str = "labels",
+        classes_file: str = "classes.txt",
+        dataset_id: str | None = None,
+        image_extensions: list[str] | None = None,
+    ) -> None:
+        self.images_dir = images_dir
+        self.labels_dir = labels_dir
+        self.classes_file = classes_file
+
+        if image_extensions is None:
+            image_extensions = [".jpg", ".jpeg", ".png", ".bmp"]
+        self.image_extensions = [ext.lower() for ext in image_extensions]
+
+        # Initialize base class
+        super().__init__(dataset_path, dataset_id)
+
+    def _initialize_format_specific(self) -> None:
+        """Initialize YOLO-specific components."""
+        self.images_path = self.dataset_path / self.images_dir
+        self.labels_path = self.dataset_path / self.labels_dir
+        self.classes_path = self.dataset_path / self.classes_file
+
+        if not self.images_path.exists():
+            raise FileNotFoundError(f"Images directory not found: {self.images_path}")
+        if not self.labels_path.exists():
+            raise FileNotFoundError(f"Labels directory not found: {self.labels_path}")
+        if not self.classes_path.exists():
+            raise FileNotFoundError(f"Classes file not found: {self.classes_path}")
+
+        self._load_class_names()
+        self._find_image_files()
+
+    @property
+    def index2label(self) -> dict[int, str]:
+        """Mapping from class index to class name."""
+        return self._index2label
+
+    def _create_dataset_implementation(self) -> ObjectDetectionDataset:
+        """Create YOLO dataset implementation."""
+        return _YOLODataset(self)
+
+    def _validate_format_specific(self) -> tuple[list[str], dict[str, Any]]:
+        """Validate YOLO format specific files and structure."""
+        issues = []
+        stats = {}
+
+        # Check labels directory
+        labels_path = self.dataset_path / self.labels_dir
+        if not labels_path.exists():
+            issues.append(f"Missing {self.labels_dir}/ directory")
+        else:
+            label_files = list(labels_path.glob("*.txt"))
+            stats["num_label_files"] = len(label_files)
+            if len(label_files) == 0:
+                issues.append(f"No label files found in {self.labels_dir}/ directory")
+            else:
+                # Validate label file format (sample check)
+                label_issues = self._validate_yolo_label_format(labels_path)
+                issues.extend(label_issues)
+
+        # Check required classes.txt
+        classes_path = self.dataset_path / self.classes_file
+        if not classes_path.exists():
+            issues.append(f"Missing required {self.classes_file} file")
+        else:
+            try:
+                with open(classes_path) as f:
+                    class_lines = [line.strip() for line in f if line.strip()]
+                stats["num_classes"] = len(class_lines)
+                if len(class_lines) == 0:
+                    issues.append(f"{self.classes_file} is empty")
+            except Exception as e:
+                issues.append(f"Error reading {self.classes_file}: {e}")
+
+        return issues, stats
+
+    def _validate_yolo_label_format(self, labels_path: Path) -> list[str]:
+        """Validate YOLO label file format (sample check)."""
+        issues = []
+        label_files = list(labels_path.glob("*.txt"))
+
+        if not label_files:
+            return issues
+
+        sample_label = label_files[0]
+        try:
+            with open(sample_label) as f:
+                for line_num, line in enumerate(f, 1):
+                    if not line.strip():
+                        continue
+
+                    parts = line.strip().split()
+                    if len(parts) != 5:
+                        issues.append(
+                            f"Invalid YOLO format in {sample_label.name} line {line_num}: "
+                            f"expected 5 values, got {len(parts)}"
+                        )
+                        break
+
+                    try:
+                        coords = [float(x) for x in parts[1:]]
+                        if not all(0 <= coord <= 1 for coord in coords):
+                            issues.append(f"Coordinates out of range [0,1] in {sample_label.name} line {line_num}")
+                            break
+                    except ValueError:
+                        issues.append(f"Invalid numeric values in {sample_label.name} line {line_num}")
+                        break
+        except Exception as e:
+            issues.append(f"Error validating label file {sample_label.name}: {e}")
+
+        return issues
+
+    def _load_class_names(self) -> None:
+        """Load class names from classes file."""
+        with open(self.classes_path) as f:
+            class_names = [line.strip() for line in f if line.strip()]
+        self._index2label = {idx: name for idx, name in enumerate(class_names)}
+
+    def _find_image_files(self) -> None:
+        """Find all valid image files."""
+        self.image_files = []
+        for ext in self.image_extensions:
+            self.image_files.extend(self.images_path.glob(f"*{ext}"))
+        self.image_files.sort()
+
+        if not self.image_files:
+            raise ValueError(f"No image files found in {self.images_path}")
+
+
+class _YOLODataset:
+    """Internal YOLO dataset implementation."""
+
+    def __init__(self, reader: YOLODatasetReader) -> None:
+        self.reader = reader
+
+    @property
+    def metadata(self) -> DatasetMetadata:
+        return DatasetMetadata(
+            id=self.reader._dataset_id,
+            index2label=self.reader.index2label,
+        )
+
+    def __len__(self) -> int:
+        return len(self.reader.image_files)
+
+    def __getitem__(self, index: int) -> ObjectDetectionDatum:
+        image_path = self.reader.image_files[index]
+
+        # Load image
+        image = np.array(Image.open(image_path).convert("RGB"))
+        img_height, img_width = image.shape[:2]
+        image = np.transpose(image, (2, 0, 1))  # Convert to CHW format
+
+        # Load corresponding label file
+        label_path = self.reader.labels_path / f"{image_path.stem}.txt"
+
+        annotation_metadata = []
+        if label_path.exists():
+            boxes = []
+            labels = []
+
+            with open(label_path) as f:
+                for line_num, line in enumerate(f):
+                    if not line.strip():
+                        continue
+
+                    parts = line.strip().split()
+                    if len(parts) != 5:
+                        continue
+
+                    class_id = int(parts[0])
+                    center_x, center_y, width, height = map(float, parts[1:])
+
+                    # Convert normalized YOLO format to absolute pixel coordinates
+                    x1 = (center_x - width / 2) * img_width
+                    y1 = (center_y - height / 2) * img_height
+                    x2 = (center_x + width / 2) * img_width
+                    y2 = (center_y + height / 2) * img_height
+
+                    boxes.append([x1, y1, x2, y2])
+                    labels.append(class_id)
+
+                    # Store original YOLO format coordinates in metadata
+                    ann_meta = {
+                        "line_number": line_num + 1,
+                        "class_id": class_id,
+                        "yolo_center_x": center_x,
+                        "yolo_center_y": center_y,
+                        "yolo_width": width,
+                        "yolo_height": height,
+                        "absolute_bbox": [x1, y1, x2, y2],
+                    }
+                    annotation_metadata.append(ann_meta)
+
+            if boxes:
+                boxes = np.array(boxes, dtype=np.float32)
+                labels = np.array(labels, dtype=np.int64)
+                scores = np.ones(len(labels), dtype=np.float32)  # Ground truth scores
+            else:
+                boxes = np.empty((0, 4), dtype=np.float32)
+                labels = np.empty(0, dtype=np.int64)
+                scores = np.empty(0, dtype=np.float32)
+        else:
+            # No label file - empty annotations
+            boxes = np.empty((0, 4), dtype=np.float32)
+            labels = np.empty(0, dtype=np.int64)
+            scores = np.empty(0, dtype=np.float32)
+
+        target = _ObjectDetectionTarget(boxes, labels, scores)
+
+        # Create comprehensive datum metadata
+        datum_metadata = DatumMetadata(
+            id=f"{self.reader._dataset_id}_{image_path.stem}",
+            # Image-level metadata
+            file_name=image_path.name,
+            file_path=str(image_path),
+            width=img_width,
+            height=img_height,
+            # Label file metadata
+            label_file=label_path.name if label_path.exists() else None,
+            label_file_exists=label_path.exists(),
+            # Annotation metadata
+            annotations=annotation_metadata,
+            num_annotations=len(annotation_metadata),
+        )
+
+        return image, target, datum_metadata

maite_datasets/_validate.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+__all__ = []
+
+import numpy as np
+from collections.abc import Sequence, Sized
+from typing import Any, Literal
+
+from maite_datasets._protocols import Array, ObjectDetectionTarget
+
+
+class ValidationMessages:
+    DATASET_SIZED = "Dataset must be sized."
+    DATASET_INDEXABLE = "Dataset must be indexable."
+    DATASET_NONEMPTY = "Dataset must be non-empty."
+    DATASET_METADATA = "Dataset must have a 'metadata' attribute."
+    DATASET_METADATA_TYPE = "Dataset metadata must be a dictionary."
+    DATASET_METADATA_FORMAT = "Dataset metadata must contain an 'id' key."
+    DATUM_TYPE = "Dataset datum must be a tuple."
+    DATUM_FORMAT = "Dataset datum must contain 3 elements: image, target, metadata."
+    DATUM_IMAGE_TYPE = "Images must be 3-dimensional arrays."
+    DATUM_IMAGE_FORMAT = "Images must be in CHW format."
+    DATUM_TARGET_IC_TYPE = "ImageClassificationDataset targets must be one-dimensional arrays."
+    DATUM_TARGET_IC_FORMAT = "ImageClassificationDataset targets must be one-hot encoded or pseudo-probabilities."
+    DATUM_TARGET_OD_TYPE = "ObjectDetectionDataset targets must be have 'boxes', 'labels' and 'scores'."
+    DATUM_TARGET_OD_LABELS_TYPE = "ObjectDetectionTarget labels must be one-dimensional (N,) arrays."
+    DATUM_TARGET_OD_BOXES_TYPE = "ObjectDetectionTarget boxes must be two-dimensional (N, 4) arrays in xxyy format."
+    DATUM_TARGET_OD_SCORES_TYPE = "ObjectDetectionTarget scores must be one (N,) or two-dimensional (N, M) arrays."
+    DATUM_TARGET_TYPE = "Target is not a valid ImageClassification or ObjectDetection target type."
+    DATUM_METADATA_TYPE = "Datum metadata must be a dictionary."
+    DATUM_METADATA_FORMAT = "Datum metadata must contain an 'id' key."
+
+
+def _validate_dataset_type(dataset: Any) -> list[str]:
+    issues = []
+    is_sized = isinstance(dataset, Sized)
+    is_indexable = hasattr(dataset, "__getitem__")
+    if not is_sized:
+        issues.append(ValidationMessages.DATASET_SIZED)
+    if not is_indexable:
+        issues.append(ValidationMessages.DATASET_INDEXABLE)
+    if is_sized and len(dataset) == 0:
+        issues.append(ValidationMessages.DATASET_NONEMPTY)
+    return issues
+
+
+def _validate_dataset_metadata(dataset: Any) -> list[str]:
+    issues = []
+    if not hasattr(dataset, "metadata"):
+        issues.append(ValidationMessages.DATASET_METADATA)
+    metadata = getattr(dataset, "metadata", None)
+    if not isinstance(metadata, dict):
+        issues.append(ValidationMessages.DATASET_METADATA_TYPE)
+    if not isinstance(metadata, dict) or "id" not in metadata:
+        issues.append(ValidationMessages.DATASET_METADATA_FORMAT)
+    return issues
+
+
+def _validate_datum_type(datum: Any) -> list[str]:
+    issues = []
+    if not isinstance(datum, tuple):
+        issues.append(ValidationMessages.DATUM_TYPE)
+    if datum is None or isinstance(datum, Sized) and len(datum) != 3:
+        issues.append(ValidationMessages.DATUM_FORMAT)
+    return issues
+
+
+def _validate_datum_image(image: Any) -> list[str]:
+    issues = []
+    if not isinstance(image, Array) or len(image.shape) != 3:
+        issues.append(ValidationMessages.DATUM_IMAGE_TYPE)
+    if (
+        not isinstance(image, Array)
+        or len(image.shape) == 3
+        and (image.shape[0] > image.shape[1] or image.shape[0] > image.shape[2])
+    ):
+        issues.append(ValidationMessages.DATUM_IMAGE_FORMAT)
+    return issues
+
+
+def _validate_datum_target_ic(target: Any) -> list[str]:
+    issues = []
+    if not isinstance(target, Array) or len(target.shape) != 1:
+        issues.append(ValidationMessages.DATUM_TARGET_IC_TYPE)
+    if target is None or sum(target) > 1 + 1e-6 or sum(target) < 1 - 1e-6:
+        issues.append(ValidationMessages.DATUM_TARGET_IC_FORMAT)
+    return issues
+
+
+def _validate_datum_target_od(target: Any) -> list[str]:
+    issues = []
+    if not isinstance(target, ObjectDetectionTarget):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_TYPE)
+    od_target: ObjectDetectionTarget | None = target if isinstance(target, ObjectDetectionTarget) else None
+    if od_target is None or len(np.asarray(od_target.labels).shape) != 1:
+        issues.append(ValidationMessages.DATUM_TARGET_OD_LABELS_TYPE)
+    if (
+        od_target is None
+        or len(np.asarray(od_target.boxes).shape) != 2
+        or (len(np.asarray(od_target.boxes).shape) == 2 and np.asarray(od_target.boxes).shape[1] != 4)
+    ):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_BOXES_TYPE)
+    if od_target is None or len(np.asarray(od_target.scores).shape) not in (1, 2):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_SCORES_TYPE)
+    return issues
+
+
+def _detect_target_type(target: Any) -> Literal["ic", "od", "auto"]:
+    if isinstance(target, Array):
+        return "ic"
+    if isinstance(target, ObjectDetectionTarget):
+        return "od"
+    return "auto"
+
+
+def _validate_datum_target(target: Any, target_type: Literal["ic", "od", "auto"]) -> list[str]:
+    issues = []
+    target_type = _detect_target_type(target) if target_type == "auto" else target_type
+    if target_type == "ic":
+        issues.extend(_validate_datum_target_ic(target))
+    elif target_type == "od":
+        issues.extend(_validate_datum_target_od(target))
+    else:
+        issues.append(ValidationMessages.DATUM_TARGET_TYPE)
+    return issues
+
+
+def _validate_datum_metadata(metadata: Any) -> list[str]:
+    issues = []
+    if metadata is None or not isinstance(metadata, dict):
+        issues.append(ValidationMessages.DATUM_METADATA_TYPE)
+    if metadata is None or isinstance(metadata, dict) and "id" not in metadata:
+        issues.append(ValidationMessages.DATUM_METADATA_FORMAT)
+    return issues
+
+
+def validate_dataset(dataset: Any, dataset_type: Literal["ic", "od", "auto"] = "auto") -> None:
+    """
+    Validate a dataset for compliance with MAITE protocol.
+
+    Parameters
+    ----------
+    dataset: Any
+        Dataset to validate.
+    dataset_type: "ic", "od", or "auto", default "auto"
+        Dataset type, if known.
+
+    Raises
+    ------
+    ValueError
+        Raises exception if dataset is invalid with a list of validation issues.
+    """
+    issues = []
+    issues.extend(_validate_dataset_type(dataset))
+    datum = None if issues else dataset[0]  # type: ignore
+    issues.extend(_validate_dataset_metadata(dataset))
+    issues.extend(_validate_datum_type(datum))
+
+    is_seq = isinstance(datum, Sequence)
+    datum_len = len(datum) if is_seq else 0
+    image = datum[0] if is_seq and datum_len > 0 else None
+    target = datum[1] if is_seq and datum_len > 1 else None
+    metadata = datum[2] if is_seq and datum_len > 2 else None
+    issues.extend(_validate_datum_image(image))
+    issues.extend(_validate_datum_target(target, dataset_type))
+    issues.extend(_validate_datum_metadata(metadata))
+
+    if issues:
+        raise ValueError("Dataset validation issues found:\n - " + "\n - ".join(issues))

{maite_datasets-0.0.2.dist-info → maite_datasets-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: maite-datasets
-Version: 0.0.2
+Version: 0.0.4
 Summary: A collection of Image Classification and Object Detection task datasets conforming to the MAITE protocol.
 Author-email: Andrew Weng <andrew.weng@ariacoustics.com>, Ryan Wood <ryan.wood@ariacoustics.com>, Shaun Jullens <shaun.jullens@ariacoustics.com>
 License-Expression: MIT

{maite_datasets-0.0.2.dist-info → maite_datasets-0.0.4.dist-info}/RECORD

@@ -1,13 +1,20 @@
-maite_datasets/__init__.py,sha256=aM16hWPYR5WF0nx2AqTYHbGmibNTBCrYilcDKUs_yPo,235
+maite_datasets/__init__.py,sha256=53LW5bHMAr4uD6w2bvrPxgtROUIzaE-3LR6TR0dDucs,746
 maite_datasets/_base.py,sha256=BiWB_xvL4AtV0jxVjzpcZHuRTb52dTD0CQtu08DzoXA,8195
 maite_datasets/_builder.py,sha256=URhRCedvuqsy88N4lzQrwI-uL1kS1_kavP9fS402sPw,10036
+maite_datasets/_collate.py,sha256=-XuKeeMmOnSB0RgQbz8BjsoqQar9Tsf_qALZxijQ498,4063
 maite_datasets/_fileio.py,sha256=7S-hF3xU60AdcsPsfYR7rjbeGZUlv3JjGEZhGJOxGYU,5622
-maite_datasets/_protocols.py,sha256=uwnI2P-zJnpEHJ0eOJ7dO_7KehwHEtEqR4pYcJiEXNk,5312
+maite_datasets/_protocols.py,sha256=aWrnUM1stZ9VInkBEynod_OdYq2ORSpew7yoF-Zeuig,5247
 maite_datasets/_types.py,sha256=S5DMyiUrkUjV9uM0ysKqxVoi7z5P7B3EPiLI4Fyq9Jc,1147
+maite_datasets/_validate.py,sha256=sP-5lYXkmkiTadJcy_LtEMiZ0m82xR0yELoxWORrZDQ,6904
 maite_datasets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 maite_datasets/_mixin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 maite_datasets/_mixin/_numpy.py,sha256=GEuRyeprH-STh-_zktAp0Tg6NNyMdh1ThyhjW558NOo,860
 maite_datasets/_mixin/_torch.py,sha256=pkN2vMNsDk_h5wnD5899zIHsPtEADbGfmRyI5CdGonI,827
+maite_datasets/_reader/__init__.py,sha256=VzrVOsmztPJV83um8tY5qdqU-HEPP15RlLClGbxTFlQ,164
+maite_datasets/_reader/_base.py,sha256=3_425HLcvfEU9bqQjy9S9gvqXlkPDR471IVXFxBozl0,4289
+maite_datasets/_reader/_coco.py,sha256=YyDrgdXZog_EHViWary5k8bkQd-jNaPm-G1wiN_V5ks,9960
+maite_datasets/_reader/_factory.py,sha256=cI3Cw1yWj4hK2gn6N5bugXzGMcNwcCEkJ4AoynwOZvI,2222
+maite_datasets/_reader/_yolo.py,sha256=abWAXrFFGE00NlIMUb_lAoiXFykGYOAGKGHekhG30Q8,11462
 maite_datasets/image_classification/__init__.py,sha256=pcZojkdsiMoLgY4mKjoQY6WyEwiGYHxNrAGpnvn3zsY,308
 maite_datasets/image_classification/_cifar10.py,sha256=w7BPGZzUV1gXFoYRgxa6VOqKn1EgQi3x1rrA4nEUbeI,8470
 maite_datasets/image_classification/_mnist.py,sha256=6xDWY4qbY1hlcUZKvVZeQMvYbF0vLtaVzOuQUKJkcJU,8248
@@ -18,7 +25,7 @@ maite_datasets/object_detection/_milco.py,sha256=KEU4JFvCxfyMAb4RFMnxTMk_MggdEAV
 maite_datasets/object_detection/_seadrone.py,sha256=w_pSojLzgwdKrUSxaz8r7dPJVKGND6JSYl0S_BKOLH0,271282
 maite_datasets/object_detection/_voc.py,sha256=VuokKaOzI1wSfgG5DC7ufMbRDlG-b6Se3hg4eQzNQbE,19731
 maite_datasets/object_detection/_voc_torch.py,sha256=bjeawnNit7Llcf_cZY_9lcJYoUoAU-Wen6MMT-7QX3k,2917
-maite_datasets-0.0.2.dist-info/METADATA,sha256=O3RGToBWSFhEyi_iAdnc8pqYSVzNRXo_XjIQBOEIEWA,3747
-maite_datasets-0.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-maite_datasets-0.0.2.dist-info/licenses/LICENSE,sha256=6h3J3R-ajGHh_isDSftzS5_jJjB9HH4TaI0vU-VscaY,1082
-maite_datasets-0.0.2.dist-info/RECORD,,
+maite_datasets-0.0.4.dist-info/METADATA,sha256=8-83ACnQAjf9LJgZY25GvIPGL5o5Wi0RA-SEog7jcvU,3747
+maite_datasets-0.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+maite_datasets-0.0.4.dist-info/licenses/LICENSE,sha256=6h3J3R-ajGHh_isDSftzS5_jJjB9HH4TaI0vU-VscaY,1082
+maite_datasets-0.0.4.dist-info/RECORD,,