maite-datasets 0.0.6__py3-none-any.whl → 0.0.8__py3-none-any.whl

maite_datasets/_base.py

@@ -8,20 +8,22 @@ from abc import abstractmethod
 from collections import namedtuple
 from collections.abc import Iterator, Sequence
 from pathlib import Path
-from typing import Any, Callable, Generic, Literal, NamedTuple, TypeVar, cast
+from typing import Any, Callable, Generic, Literal, NamedTuple, Protocol, TypeVar, cast
 
 import numpy as np
 from maite.protocols import DatasetMetadata, DatumMetadata
+from maite.protocols import image_classification as ic
+from maite.protocols import object_detection as od
 from numpy.typing import NDArray
 from PIL import Image
 
 from maite_datasets._fileio import _ensure_exists
 from maite_datasets.protocols import Array
 
-_T = TypeVar("_T")
 _T_co = TypeVar("_T_co", covariant=True)
 _TArray = TypeVar("_TArray", bound=Array)
 _TTarget = TypeVar("_TTarget")
+_TODTarget = TypeVar("_TODTarget", bound=od.ObjectDetectionTarget)
 _TRawTarget = TypeVar(
     "_TRawTarget",
     Sequence[int],
@@ -30,8 +32,7 @@ _TRawTarget = TypeVar(
 )
 _TAnnotation = TypeVar("_TAnnotation", int, str, tuple[list[int], list[list[float]]])
 
-
-ObjectDetectionTarget = namedtuple("ObjectDetectionTarget", ["boxes", "labels", "scores"])
+ObjectDetectionTargetTuple = namedtuple("ObjectDetectionTargetTuple", ["boxes", "labels", "scores"])
 
 
 class BaseDatasetMixin(Generic[_TArray]):
@@ -249,6 +250,7 @@ class BaseICDataset(
     BaseDownloadedDataset[_TArray, _TArray, list[int], int],
     BaseDatasetMixin[_TArray],
     BaseDataset[_TArray, _TArray],
+    ic.Dataset,
 ):
     """
     Base class for image classification datasets.
@@ -278,9 +280,10 @@ class BaseICDataset(
 
 
 class BaseODDataset(
-    BaseDownloadedDataset[_TArray, ObjectDetectionTarget, _TRawTarget, _TAnnotation],
+    BaseDownloadedDataset[_TArray, _TODTarget, _TRawTarget, _TAnnotation],
     BaseDatasetMixin[_TArray],
-    BaseDataset[_TArray, ObjectDetectionTarget],
+    BaseDataset[_TArray, _TODTarget],
+    od.Dataset,
 ):
     """
     Base class for object detection datasets.
@@ -288,7 +291,7 @@ class BaseODDataset(
 
     _bboxes_per_size: bool = False
 
-    def __getitem__(self, index: int) -> tuple[_TArray, ObjectDetectionTarget, DatumMetadata]:
+    def __getitem__(self, index: int) -> tuple[_TArray, _TODTarget, DatumMetadata]:
         """
         Args
         ----
@@ -310,7 +313,9 @@ class BaseODDataset(
         if self._bboxes_per_size and boxes:
             boxes = boxes * np.asarray([[img_size[1], img_size[2], img_size[1], img_size[2]]])
         # Create the Object Detection Target
-        target = ObjectDetectionTarget(self._as_array(boxes), self._as_array(labels), self._one_hot_encode(labels))
+        target = ObjectDetectionTargetTuple(self._as_array(boxes), self._as_array(labels), self._one_hot_encode(labels))
+        # Cast target explicitly to ODTarget as namedtuple does not provide any typing metadata
+        target = cast(_TODTarget, target)
 
         img_metadata = {key: val[index] for key, val in self._datum_metadata.items()}
         img_metadata = img_metadata | additional_metadata
@@ -324,6 +329,15 @@ class BaseODDataset(
 NumpyArray = NDArray[np.floating[Any]] | NDArray[np.integer[Any]]
 
 
+class NumpyObjectDetectionTarget(od.ObjectDetectionTarget, Protocol):
+    @property
+    def boxes(self) -> NumpyArray: ...
+    @property
+    def labels(self) -> NumpyArray: ...
+    @property
+    def scores(self) -> NumpyArray: ...
+
+
 class BaseDatasetNumpyMixin(BaseDatasetMixin[NumpyArray]):
     def _as_array(self, raw: list[Any]) -> NumpyArray:
         return np.asarray(raw)
@@ -347,8 +361,8 @@ NumpyImageClassificationDatumTransform = Callable[
     tuple[NumpyArray, NumpyArray, DatumMetadata],
 ]
 NumpyObjectDetectionDatumTransform = Callable[
-    [tuple[NumpyArray, ObjectDetectionTarget, DatumMetadata]],
-    tuple[NumpyArray, ObjectDetectionTarget, DatumMetadata],
+    [tuple[NumpyArray, NumpyObjectDetectionTarget, DatumMetadata]],
+    tuple[NumpyArray, NumpyObjectDetectionTarget, DatumMetadata],
 ]
 NumpyImageClassificationTransform = NumpyImageTransform | NumpyImageClassificationDatumTransform
 NumpyObjectDetectionTransform = NumpyImageTransform | NumpyObjectDetectionDatumTransform

maite_datasets/adapters/__init__.py

@@ -0,0 +1,3 @@
+from ._huggingface import HFImageClassificationDataset, HFObjectDetectionDataset, from_huggingface
+
+__all__ = ["HFImageClassificationDataset", "HFObjectDetectionDataset", "from_huggingface"]

maite_datasets/adapters/_huggingface.py

@@ -0,0 +1,391 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any, Literal, TypeAlias, overload
+
+import maite.protocols.image_classification as ic
+import maite.protocols.object_detection as od
+import numpy as np
+from maite.protocols import DatasetMetadata, DatumMetadata
+
+from maite_datasets._base import BaseDataset, NumpyArray, ObjectDetectionTargetTuple
+from maite_datasets.protocols import HFArray, HFClassLabel, HFDataset, HFImage, HFList, HFValue
+from maite_datasets.wrappers._torch import TTarget
+
+# Constants for image processing
+MAX_VALID_CHANNELS = 10
+
+FeatureDict: TypeAlias = Mapping[str, Any]
+
+
+@dataclass
+class HFDatasetInfo:
+    image_key: str
+
+
+@dataclass
+class HFImageClassificationDatasetInfo(HFDatasetInfo):
+    label_key: str
+
+
+@dataclass
+class HFObjectDetectionDatasetInfo(HFDatasetInfo):
+    objects_key: str
+    bbox_key: str
+    label_key: str
+
+
+class HFBaseDataset(BaseDataset[NumpyArray, TTarget]):
+    """Base wrapper for Hugging Face datasets, handling common logic."""
+
+    def __init__(self, hf_dataset: HFDataset, image_key: str, known_keys: set[str]) -> None:
+        self.source = hf_dataset
+        self._image_key = image_key
+
+        # Add dataset metadata
+        dataset_info_dict = hf_dataset.info.__dict__
+        if "id" in dataset_info_dict:
+            dataset_info_dict["datasetinfo_id"] = dataset_info_dict.pop("id")
+        self._metadata_id = dataset_info_dict["dataset_name"]
+        self._metadata_dict = dataset_info_dict
+
+        # Pre-validate features and cache metadata keys
+        self._validate_features(hf_dataset.features)
+        self._scalar_meta_keys = self._extract_scalar_meta_keys(hf_dataset.features, known_keys)
+
+        # Cache for image conversions
+        self._image_cache: dict[int, np.ndarray] = {}
+
+    def _validate_features(self, features: FeatureDict) -> None:
+        """Pre-validate all features during initialization."""
+        if self._image_key not in features:
+            raise ValueError(f"Image key '{self._image_key}' not found in dataset features.")
+
+        if not isinstance(features[self._image_key], (HFImage, HFArray)):
+            raise TypeError(f"Image feature '{self._image_key}' must be HFImage or HFArray.")
+
+    def _extract_scalar_meta_keys(self, features: FeatureDict, known_keys: set[str]) -> list[str]:
+        """Extract scalar metadata keys during initialization."""
+        return [key for key, feature in features.items() if key not in known_keys and isinstance(feature, HFValue)]
+
+    def __len__(self) -> int:
+        return len(self.source)
+
+    def _get_base_metadata(self, index: int) -> DatumMetadata:
+        """Extract base metadata for a datum."""
+        item = self.source[index]
+        datum_metadata: DatumMetadata = {"id": index}
+        for key in self._scalar_meta_keys:
+            datum_metadata[key] = item[key]
+        return datum_metadata
+
+    @lru_cache(maxsize=64)  # Cache image conversions
+    def _get_image(self, index: int) -> np.ndarray:
+        """Get and process image with caching and optimized conversions."""
+        # Convert to numpy array only once
+        raw_image = self.source[index][self._image_key]
+        image = np.asarray(raw_image)
+
+        # Handle different image formats efficiently
+        if image.ndim == 2:
+            # Grayscale: HW -> CHW
+            image = image[np.newaxis, ...]  # More efficient than expand_dims
+        elif image.ndim == 3:
+            # Check if we need to transpose from HWC to CHW
+            if image.shape[-1] < image.shape[-3] and image.shape[-1] <= MAX_VALID_CHANNELS:
+                # HWC -> CHW using optimized transpose
+                image = np.transpose(image, (2, 0, 1))
+            elif image.shape[0] > MAX_VALID_CHANNELS:
+                raise ValueError(
+                    f"Image at index {index} has invalid channel configuration. "
+                    f"Expected channels to be less than {MAX_VALID_CHANNELS}, got shape {image.shape}"
+                )
+        else:
+            raise ValueError(
+                f"Image at index {index} has unsupported dimensionality. "
+                f"Expected 2D or 3D, got {image.ndim}D with shape {image.shape}"
+            )
+
+        if image.ndim != 3:
+            raise ValueError(f"Image processing failed for index {index}. Final shape: {image.shape}")
+
+        return image
+
+
+class HFImageClassificationDataset(HFBaseDataset[NumpyArray], ic.Dataset):
+    """Wraps a Hugging Face dataset to comply with the ImageClassificationDataset protocol."""
+
+    def __init__(self, hf_dataset: HFDataset, image_key: str, label_key: str) -> None:
+        super().__init__(hf_dataset, image_key, known_keys={image_key, label_key})
+        self._label_key = label_key
+
+        # Pre-validate label feature
+        label_feature = hf_dataset.features[self._label_key]
+        if not isinstance(label_feature, HFClassLabel):
+            raise TypeError(
+                f"Label feature '{self._label_key}' must be a datasets.ClassLabel, got {type(label_feature).__name__}."
+            )
+
+        self._num_classes: int = label_feature.num_classes
+
+        # Pre-compute one-hot identity matrix for efficient encoding
+        self._one_hot_matrix = np.eye(self._num_classes, dtype=np.float32)
+
+        # Enhanced metadata with validation
+        self.metadata: DatasetMetadata = DatasetMetadata(
+            id=self._metadata_id, index2label=dict(enumerate(label_feature.names)), **self._metadata_dict
+        )
+
+    def __getitem__(self, index: int) -> tuple[NumpyArray, NumpyArray, DatumMetadata]:
+        if not 0 <= index < len(self.source):
+            raise IndexError(f"Index {index} out of range for dataset of size {len(self.source)}")
+
+        # Process image
+        image = self._get_image(index)
+        label_int = self.source[index][self._label_key]
+
+        # Process target
+        if not 0 <= label_int < self._num_classes:
+            raise ValueError(f"Label {label_int} at index {index} is out of range [0, {self._num_classes})")
+        one_hot_label = self._one_hot_matrix[label_int]
+
+        # Process metadata
+        datum_metadata = self._get_base_metadata(index)
+
+        return image, one_hot_label, datum_metadata
+
+
+class HFObjectDetectionDataset(HFBaseDataset[ObjectDetectionTargetTuple], od.Dataset):
+    """Wraps a Hugging Face dataset to comply with the ObjectDetectionDataset protocol."""
+
+    def __init__(self, hf_dataset: HFDataset, image_key: str, objects_key: str, bbox_key: str, label_key: str) -> None:
+        super().__init__(hf_dataset, image_key, known_keys={image_key, objects_key})
+        self._objects_key = objects_key
+        self._bbox_key = bbox_key
+        self._label_key = label_key
+
+        # Pre-validate and extract object features
+        self._object_meta_keys = self._validate_and_extract_object_features(hf_dataset.features)
+
+        # Validate and extract label information
+        label_feature = self._extract_label_feature(hf_dataset.features)
+        self.metadata: DatasetMetadata = DatasetMetadata(
+            id=self._metadata_id, index2label=dict(enumerate(label_feature.names)), **self._metadata_dict
+        )
+
+    def _validate_and_extract_object_features(self, features: FeatureDict) -> list[str]:
+        """Validate objects feature and extract metadata keys."""
+        objects_feature = features[self._objects_key]
+
+        # Determine the structure and get inner features
+        if isinstance(objects_feature, HFList):  # list(dict) case
+            if not isinstance(objects_feature.feature, dict):
+                raise TypeError(f"Objects feature '{self._objects_key}' with list type must contain dict features.")
+            inner_feature_dict = objects_feature.feature
+        elif isinstance(objects_feature, dict):  # dict(list) case
+            inner_feature_dict = objects_feature
+        else:
+            raise TypeError(
+                f"Objects feature '{self._objects_key}' must be a list or dict, got {type(objects_feature).__name__}."
+            )
+
+        # Validate required keys exist
+        required_keys = {self._bbox_key, self._label_key}
+        missing_keys = required_keys - set(inner_feature_dict.keys())
+        if missing_keys:
+            raise ValueError(f"Objects feature '{self._objects_key}' missing required keys: {missing_keys}")
+
+        # Extract object metadata keys
+        known_inner_keys = {self._bbox_key, self._label_key}
+        return [
+            key
+            for key, feature in inner_feature_dict.items()
+            if key not in known_inner_keys and isinstance(feature, (HFValue, HFList))
+        ]
+
+    def _extract_label_feature(self, features: FeatureDict) -> HFClassLabel:
+        """Extract and validate the label feature."""
+        objects_feature = features[self._objects_key]
+
+        inner_features = objects_feature.feature if isinstance(objects_feature, HFList) else objects_feature
+        label_feature_container = inner_features[self._label_key]
+        label_feature = (
+            label_feature_container.feature
+            if isinstance(label_feature_container.feature, HFClassLabel)
+            else label_feature_container
+        )
+
+        if not isinstance(label_feature, HFClassLabel):
+            raise TypeError(
+                f"Label '{self._label_key}' in '{self._objects_key}' must be a ClassLabel, "
+                f"got {type(label_feature).__name__}."
+            )
+
+        return label_feature
+
+    def __getitem__(self, index: int) -> tuple[NumpyArray, ObjectDetectionTargetTuple, DatumMetadata]:
+        if not 0 <= index < len(self.source):
+            raise IndexError(f"Index {index} out of range for dataset of size {len(self.source)}")
+
+        # Process image
+        image = self._get_image(index)
+        objects = self.source[index][self._objects_key]
+
+        # Process target
+        boxes = objects[self._bbox_key]
+        labels = objects[self._label_key]
+        scores = np.zeros_like(labels, dtype=np.float32)
+        target = ObjectDetectionTargetTuple(boxes, labels, scores)
+
+        # Process metadata
+        datum_metadata = self._get_base_metadata(index)
+        self._add_object_metadata(objects, datum_metadata)
+
+        return image, target, datum_metadata
+
+    def _add_object_metadata(self, objects: dict[str, Any], datum_metadata: DatumMetadata) -> None:
+        """Efficiently add object metadata to datum metadata."""
+        if not objects[self._bbox_key]:  # No objects
+            return
+
+        num_objects = len(objects[self._bbox_key])
+
+        for key in self._object_meta_keys:
+            value = objects[key]
+            if isinstance(value, list):
+                if len(value) == num_objects:
+                    datum_metadata[key] = value
+                else:
+                    raise ValueError(
+                        f"Object metadata '{key}' length {len(value)} doesn't match number of objects {num_objects}"
+                    )
+            else:
+                datum_metadata[key] = [value] * num_objects
+
+
+def is_bbox(feature: Any) -> bool:
+    """Check if feature represents bounding box data with proper type validation."""
+    if not isinstance(feature, HFList):
+        return False
+
+    # Handle nested list structure
+    bbox_candidate = feature.feature if isinstance(feature.feature, HFList) else feature
+
+    return (
+        isinstance(bbox_candidate, HFList)
+        and bbox_candidate.length == 4
+        and isinstance(bbox_candidate.feature, HFValue)
+        and any(dtype in bbox_candidate.feature.dtype for dtype in ["float", "int"])
+    )
+
+
+def is_label(feature: Any) -> bool:
+    """Check if feature represents label data with proper type validation."""
+    target_feature = feature.feature if isinstance(feature, HFList) else feature
+    return isinstance(target_feature, HFClassLabel)
+
+
+def find_od_keys(feature: Any) -> tuple[str | None, str | None]:
+    """Helper to find bbox and label keys for object detection with improved logic."""
+    if not ((isinstance(feature, HFList) and isinstance(feature.feature, dict)) or isinstance(feature, dict)):
+        return None, None
+
+    inner_features: FeatureDict = feature.feature if isinstance(feature, HFList) else feature
+
+    bbox_key = label_key = None
+
+    for inner_name, inner_feature in inner_features.items():
+        if bbox_key is None and is_bbox(inner_feature):
+            bbox_key = inner_name
+        if label_key is None and is_label(inner_feature):
+            label_key = inner_name
+
+        # Early exit if both found
+        if bbox_key and label_key:
+            break
+
+    return bbox_key, label_key
+
+
+def get_dataset_info(dataset: HFDataset) -> HFDatasetInfo:
+    """Extract dataset information with improved validation and error messages."""
+    features = dataset.features
+    image_key = label_key = objects_key = bbox_key = None
+
+    # More efficient feature detection
+    for name, feature in features.items():
+        if image_key is None and isinstance(feature, (HFImage, HFArray)):
+            image_key = name
+        elif label_key is None and isinstance(feature, HFClassLabel):
+            label_key = name
+        elif objects_key is None:
+            temp_bbox, temp_label = find_od_keys(feature)
+            if temp_bbox and temp_label:
+                objects_key, bbox_key, label_key = name, temp_bbox, temp_label
+
+    if not image_key:
+        available_features = list(features.keys())
+        raise ValueError(
+            f"No image key found in dataset. Available features: {available_features}. "
+            f"Expected HFImage or HFArray type."
+        )
+
+    # Return appropriate dataset info based on detected features
+    if objects_key and bbox_key and label_key:
+        return HFObjectDetectionDatasetInfo(image_key, objects_key, bbox_key, label_key)
+    if label_key:
+        return HFImageClassificationDatasetInfo(image_key, label_key)
+    return HFDatasetInfo(image_key)
+
+
+@overload
+def from_huggingface(dataset: HFDataset, task: Literal["image_classification"]) -> HFImageClassificationDataset: ...
+
+
+@overload
+def from_huggingface(dataset: HFDataset, task: Literal["object_detection"]) -> HFObjectDetectionDataset: ...
+
+
+@overload
+def from_huggingface(
+    dataset: HFDataset, task: Literal["auto"] = "auto"
+) -> HFObjectDetectionDataset | HFImageClassificationDataset: ...
+
+
+def from_huggingface(
+    dataset: HFDataset, task: Literal["image_classification", "object_detection", "auto"] = "auto"
+) -> HFObjectDetectionDataset | HFImageClassificationDataset:
+    """Create appropriate dataset wrapper with enhanced error handling."""
+    info = get_dataset_info(dataset)
+
+    if isinstance(info, HFImageClassificationDatasetInfo):
+        if task in ("image_classification", "auto"):
+            return HFImageClassificationDataset(dataset, info.image_key, info.label_key)
+        if task == "object_detection":
+            raise ValueError(
+                f"Task mismatch: requested 'object_detection' but dataset appears to be "
+                f"image classification. Detected features: image='{info.image_key}', "
+                f"label='{info.label_key}'"
+            )
+
+    elif isinstance(info, HFObjectDetectionDatasetInfo):
+        if task in ("object_detection", "auto"):
+            return HFObjectDetectionDataset(dataset, info.image_key, info.objects_key, info.bbox_key, info.label_key)
+        if task == "image_classification":
+            raise ValueError(
+                f"Task mismatch: requested 'image_classification' but dataset appears to be "
+                f"object detection. Detected features: image='{info.image_key}', "
+                f"objects='{info.objects_key}'"
+            )
+
+    # Enhanced error message for auto-detection failure
+    available_features = list(dataset.features.keys())
+    feature_types = {k: type(v).__name__ for k, v in dataset.features.items()}
+
+    raise ValueError(
+        f"Could not automatically determine task for requested type '{task}'. "
+        f"Detected info: {info}. Available features: {available_features}. "
+        f"Feature types: {feature_types}. Ensure dataset has proper image and label/objects features."
+    )

maite_datasets/object_detection/_antiuav.py

@@ -13,11 +13,12 @@ from maite_datasets._base import (
     BaseODDataset,
     DataLocation,
     NumpyArray,
+    NumpyObjectDetectionTarget,
     NumpyObjectDetectionTransform,
 )
 
 
-class AntiUAVDetection(BaseODDataset[NumpyArray, list[str], str], BaseDatasetNumpyMixin):
+class AntiUAVDetection(BaseODDataset[NumpyArray, NumpyObjectDetectionTarget, list[str], str], BaseDatasetNumpyMixin):
     """
     A UAV detection dataset focused on detecting UAVs in natural images against large variation in backgrounds.
 

maite_datasets/object_detection/_coco.py

@@ -11,7 +11,7 @@ import numpy as np
 from maite.protocols import DatasetMetadata, DatumMetadata
 from PIL import Image
 
-from maite_datasets._base import BaseDataset, ObjectDetectionTarget
+from maite_datasets._base import BaseDataset, ObjectDetectionTargetTuple
 from maite_datasets._reader import BaseDatasetReader
 
 
@@ -269,7 +269,7 @@ class COCODataset(BaseDataset):
             scores = np.empty(0, dtype=np.float32)
             annotation_metadata = []
 
-        target = ObjectDetectionTarget(boxes, labels, scores)
+        target = ObjectDetectionTargetTuple(boxes, labels, scores)
 
         # Create comprehensive datum metadata
         datum_metadata = DatumMetadata(

maite_datasets/object_detection/_milco.py

@@ -11,11 +11,12 @@ from maite_datasets._base import (
     BaseODDataset,
     DataLocation,
     NumpyArray,
+    NumpyObjectDetectionTarget,
     NumpyObjectDetectionTransform,
 )
 
 
-class MILCO(BaseODDataset[NumpyArray, list[str], str], BaseDatasetNumpyMixin):
+class MILCO(BaseODDataset[NumpyArray, NumpyObjectDetectionTarget, list[str], str], BaseDatasetNumpyMixin):
     """
     A side-scan sonar dataset focused on mine-like object detection.
 

maite_datasets/object_detection/_seadrone.py

@@ -12,6 +12,7 @@ from maite_datasets._base import (
     BaseODDataset,
     DataLocation,
     NumpyArray,
+    NumpyObjectDetectionTarget,
     NumpyObjectDetectionTransform,
 )
 from maite_datasets._fileio import _ensure_exists
@@ -20,6 +21,7 @@ from maite_datasets._fileio import _ensure_exists
 class SeaDrone(
     BaseODDataset[
         NumpyArray,
+        NumpyObjectDetectionTarget,
         list[tuple[list[int], list[list[float]]]],
         tuple[list[int], list[list[float]]],
     ],

maite_datasets/object_detection/_voc.py

@@ -16,8 +16,8 @@ from maite_datasets._base import (
     BaseODDataset,
     DataLocation,
     NumpyArray,
+    NumpyObjectDetectionTarget,
     NumpyObjectDetectionTransform,
-    ObjectDetectionTarget,
     _ensure_exists,
 )
 
@@ -46,7 +46,7 @@ VOCClassStringMap = Literal[
 TVOCClassMap = TypeVar("TVOCClassMap", VOCClassStringMap, int, list[VOCClassStringMap], list[int])
 
 
-class BaseVOCDataset(BaseDownloadedDataset[NumpyArray, ObjectDetectionTarget, list[str], str]):
+class BaseVOCDataset(BaseDownloadedDataset[NumpyArray, NumpyObjectDetectionTarget, list[str], str]):
     _resources = [
         DataLocation(
             url="https://data.brainchip.com/dataset-mirror/voc/VOCtrainval_11-May-2012.tar",
@@ -431,7 +431,7 @@ class BaseVOCDataset(BaseDownloadedDataset[NumpyArray, ObjectDetectionTarget, li
 
 class VOCDetection(
     BaseVOCDataset,
-    BaseODDataset[NumpyArray, list[str], str],
+    BaseODDataset[NumpyArray, NumpyObjectDetectionTarget, list[str], str],
     BaseDatasetNumpyMixin,
 ):
     """

maite_datasets/object_detection/_yolo.py

@@ -12,7 +12,7 @@ import numpy as np
 from maite.protocols import DatasetMetadata, DatumMetadata
 from PIL import Image
 
-from maite_datasets._base import BaseDataset, ObjectDetectionTarget
+from maite_datasets._base import BaseDataset, ObjectDetectionTargetTuple
 from maite_datasets._reader import BaseDatasetReader
 
 
@@ -297,7 +297,7 @@ class YOLODataset(BaseDataset):
             labels = np.empty(0, dtype=np.int64)
             scores = np.empty(0, dtype=np.float32)
 
-        target = ObjectDetectionTarget(boxes, labels, scores)
+        target = ObjectDetectionTargetTuple(boxes, labels, scores)
 
         # Create comprehensive datum metadata
         datum_metadata = DatumMetadata(

maite_datasets/protocols.py

@@ -2,8 +2,8 @@
 Common type protocols used for interoperability.
 """
 
-from collections.abc import Iterator
-from typing import Any, Protocol, runtime_checkable
+from collections.abc import Iterable, Iterator, Mapping, Sequence
+from typing import Any, Generic, Protocol, TypeVar, overload, runtime_checkable
 
 
 @runtime_checkable
@@ -21,3 +21,85 @@ class Array(Protocol):
     def __getitem__(self, key: Any, /) -> Any: ...
     def __iter__(self) -> Iterator[Any]: ...
     def __len__(self) -> int: ...
+
+
+TBoxes = TypeVar("TBoxes", Array, Sequence)
+TLabels = TypeVar("TLabels", Array, Sequence)
+TScores = TypeVar("TScores", Array, Sequence)
+
+
+class GenericObjectDetectionTarget(Generic[TBoxes, TLabels, TScores], Protocol):
+    boxes: TBoxes
+    labels: TLabels
+    scores: TScores
+
+
+@runtime_checkable
+class HFDatasetInfo(Protocol):
+    @property
+    def dataset_name(self) -> str: ...
+
+
+@runtime_checkable
+class HFDataset(Protocol):
+    @property
+    def features(self) -> Mapping[str, Any]: ...
+
+    @property
+    def builder_name(self) -> str | None: ...
+
+    @property
+    def info(self) -> HFDatasetInfo: ...
+
+    @overload
+    def __getitem__(self, key: int | slice | Iterable[int]) -> dict[str, Any]: ...
+    @overload
+    def __getitem__(self, key: str) -> Sequence[int]: ...
+    def __getitem__(self, key: str | int | slice | Iterable[int]) -> dict[str, Any] | Sequence[int]: ...
+
+    def __len__(self) -> int: ...
+
+
+@runtime_checkable
+class HFFeature(Protocol):
+    @property
+    def _type(self) -> str: ...
+
+
+@runtime_checkable
+class HFClassLabel(HFFeature, Protocol):
+    @property
+    def names(self) -> list[str]: ...
+
+    @property
+    def num_classes(self) -> int: ...
+
+
+@runtime_checkable
+class HFImage(HFFeature, Protocol):
+    @property
+    def decode(self) -> bool: ...
+
+
+@runtime_checkable
+class HFArray(HFFeature, Protocol):
+    @property
+    def shape(self) -> tuple[int, ...]: ...
+    @property
+    def dtype(self) -> str: ...
+
+
+@runtime_checkable
+class HFList(HFFeature, Protocol):
+    @property
+    def feature(self) -> Any: ...
+    @property
+    def length(self) -> int: ...
+
+
+@runtime_checkable
+class HFValue(HFFeature, Protocol):
+    @property
+    def pa_type(self) -> Any: ...  # pyarrow type ... not documented
+    @property
+    def dtype(self) -> str: ...

maite_datasets/wrappers/_torch.py

@@ -1,22 +1,31 @@
 from __future__ import annotations
 
-from typing import Any, Callable, Generic, TypeAlias, TypeVar, cast, overload
+from typing import Any, Callable, Generic, Protocol, TypeAlias, TypeVar, cast, overload
 
-import numpy as np
 import torch
 from maite.protocols import DatasetMetadata, DatumMetadata
-from maite.protocols.object_detection import ObjectDetectionTarget as _ObjectDetectionTarget
+from maite.protocols.object_detection import ObjectDetectionTarget
 from torch import Tensor
 from torchvision.tv_tensors import BoundingBoxes, Image
 
-from maite_datasets._base import BaseDataset, ObjectDetectionTarget
+from maite_datasets._base import BaseDataset, ObjectDetectionTargetTuple
 from maite_datasets.protocols import Array
 
 TArray = TypeVar("TArray", bound=Array)
 TTarget = TypeVar("TTarget")
 
+
+class TorchvisionObjectDetectionTarget(Protocol):
+    @property
+    def boxes(self) -> BoundingBoxes: ...
+    @property
+    def labels(self) -> Tensor: ...
+    @property
+    def scores(self) -> Tensor: ...
+
+
 TorchvisionImageClassificationDatum: TypeAlias = tuple[Image, Tensor, DatumMetadata]
-TorchvisionObjectDetectionDatum: TypeAlias = tuple[Image, ObjectDetectionTarget, DatumMetadata]
+TorchvisionObjectDetectionDatum: TypeAlias = tuple[Image, ObjectDetectionTargetTuple, DatumMetadata]
 
 
 class TorchvisionWrapper(Generic[TArray, TTarget]):
@@ -63,31 +72,30 @@ class TorchvisionWrapper(Generic[TArray, TTarget]):
     @overload
     def __getitem__(
         self: TorchvisionWrapper[TArray, TTarget], index: int
-    ) -> tuple[Image, ObjectDetectionTarget, DatumMetadata]: ...
+    ) -> tuple[Image, TorchvisionObjectDetectionTarget, DatumMetadata]: ...
 
-    def __getitem__(self, index: int) -> tuple[Image, Tensor | ObjectDetectionTarget, DatumMetadata]:
+    def __getitem__(self, index: int) -> tuple[Image, Tensor | TorchvisionObjectDetectionTarget, DatumMetadata]:
         """Get item with torch tensor conversion."""
         image, target, metadata = self._dataset[index]
 
         # Convert image to torch tensor
-        torch_image = torch.from_numpy(image) if isinstance(image, np.ndarray) else torch.as_tensor(image)
-        torch_image = Image(torch_image)
+        torch_image = Image(torch.tensor(image))
 
         # Handle different target types
         if isinstance(target, Array):
             # Image classification case
-            torch_target = torch.as_tensor(target, dtype=torch.float32)
+            torch_target = torch.tensor(target, dtype=torch.float32)
             torch_datum = self._transform((torch_image, torch_target, metadata))
             return cast(TorchvisionImageClassificationDatum, torch_datum)
 
-        if isinstance(target, _ObjectDetectionTarget):
+        if isinstance(target, ObjectDetectionTarget):
             # Object detection case
             torch_boxes = BoundingBoxes(
-                torch.as_tensor(target.boxes), format="XYXY", canvas_size=(torch_image.shape[-2], torch_image.shape[-1])
+                torch.tensor(target.boxes), format="XYXY", canvas_size=(torch_image.shape[-2], torch_image.shape[-1])
             )  # type: ignore
-            torch_labels = torch.as_tensor(target.labels, dtype=torch.int64)
-            torch_scores = torch.as_tensor(target.scores, dtype=torch.float32)
-            torch_target = ObjectDetectionTarget(torch_boxes, torch_labels, torch_scores)
+            torch_labels = torch.tensor(target.labels, dtype=torch.int64)
+            torch_scores = torch.tensor(target.scores, dtype=torch.float32)
+            torch_target = ObjectDetectionTargetTuple(torch_boxes, torch_labels, torch_scores)
             torch_datum = self._transform((torch_image, torch_target, metadata))
             return cast(TorchvisionObjectDetectionDatum, torch_datum)
 

{maite_datasets-0.0.6.dist-info → maite_datasets-0.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: maite-datasets
-Version: 0.0.6
+Version: 0.0.8
 Summary: A collection of Image Classification and Object Detection task datasets conforming to the MAITE protocol.
 Author-email: Andrew Weng <andrew.weng@ariacoustics.com>, Ryan Wood <ryan.wood@ariacoustics.com>, Shaun Jullens <shaun.jullens@ariacoustics.com>
 License-Expression: MIT
@@ -10,11 +10,11 @@ Classifier: Framework :: Pytest
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.9
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.10
 Requires-Dist: defusedxml>=0.7.1
 Requires-Dist: maite<0.9,>=0.7
 Requires-Dist: numpy>=1.24.2
@@ -81,6 +81,8 @@ tuple(<class 'numpy.ndarray'>, <class 'numpy.ndarray'>, <class 'dict'>)
 
 Wrappers provide a way to convert datasets to allow usage of tools within specific backend frameworks.
 
+### Torchvision
+
 `TorchvisionWrapper` is a convenience class that wraps any of the datasets and provides the capability to apply
 `torchvision` transforms to the dataset.
 
@@ -129,6 +131,41 @@ type=Image, shape=torch.Size([3, 224, 224])
 tensor([16.4062, 47.4688, 28.4375, 54.0312], dtype=torch.float64)
 ```
 
+## Dataset Adapters
+
+Adapters provide a way to read in datasets from other popular formats.
+
+### Huggingface
+
+Hugging face datasets can be adapted into MAITE compliant format using the `from_huggingface` adapter.
+
+```python
+>>> from datasets import load_dataset
+>>> from maite_datasets.adapters import from_huggingface
+
+>>> cppe5 = load_dataset("cppe-5")
+>>> m_cppe5 = from_huggingface(cppe5["train"])
+>>> print(m_cppe5)
+HFObjectDetection Dataset
+-------------------------
+    Source: Dataset({
+    features: ['image_id', 'image', 'width', 'height', 'objects'],
+    num_rows: 1000
+})
+    Metadata: {'id': 'cppe-5', 'index2label': {0: 'Coverall', 1: 'Face_Shield', 2: 'Gloves', 3: 'Goggles', 4: 'Mask'}, 'description': '', 'citation': '', 'homepage': '', 'license': '', 'features': {'image_id': Value('int64'), 'image': Image(mode=None, decode=True), 'width': Value('int32'), 'height': Value('int32'), 'objects': {'id': List(Value('int64')), 'area': List(Value('int64')), 'bbox': List(List(Value('float32'), length=4)), 'category': List(ClassLabel(names=['Coverall', 'Face_Shield', 'Gloves', 'Goggles', 'Mask']))}}, 'post_processed': None, 'supervised_keys': None, 'builder_name': 'parquet', 'dataset_name': 'cppe-5', 'config_name': 'default', 'version': 0.0.0, 'splits': {'train': SplitInfo(name='train', num_bytes=240478590, num_examples=1000, shard_lengths=None, dataset_name='cppe-5'), 'test': SplitInfo(name='test', num_bytes=4172706, num_examples=29, shard_lengths=None, dataset_name='cppe-5')}, 'download_checksums': {'hf://datasets/cppe-5@66f6a5efd474e35bd7cb94bf15dea27d4c6ad3f8/data/train-00000-of-00001.parquet': {'num_bytes': 237015519, 'checksum': None}, 'hf://datasets/cppe-5@66f6a5efd474e35bd7cb94bf15dea27d4c6ad3f8/data/test-00000-of-00001.parquet': {'num_bytes': 4137134, 'checksum': None}}, 'download_size': 241152653, 'post_processing_size': None, 'dataset_size': 244651296, 'size_in_bytes': 485803949}
+
+>>> image = m_cppe5[0][0]
+>>> print(f"type={image.__class__.__name__}, shape={image.shape}")
+type=ndarray, shape=(3, 663, 943)
+
+>>> target = m_cppe5[0][1]
+>>> print(f"box={target.boxes[0]}, label={target.labels[0]}")
+box=[302.0, 109.0, 73.0, 52.0], label=4
+
+>>> print(m_cppe5[0][2])
+{'id': [114, 115, 116, 117], 'image_id': 15, 'width': 943, 'height': 663, 'area': [3796, 1596, 152768, 81002]}
+```
+
 ## Additional Information
 
 For more information on the MAITE protocol, check out their [documentation](https://mit-ll-ai-technology.github.io/maite/).

maite_datasets-0.0.8.dist-info/RECORD

@@ -0,0 +1,28 @@
+maite_datasets/__init__.py,sha256=Z_HyAe08HaHMjzZS2afFumBXYFRFj0ny5ZAIp0hcj4w,569
+maite_datasets/_base.py,sha256=6-RG3VJfpNEC7luNGCPpqyIQ5FlWeYRSm_bfo1PXdUQ,12924
+maite_datasets/_builder.py,sha256=MnCh6z5hSINlzBnK_pdbgI5zSg5d1uq4UvXt3cjn9hs,9820
+maite_datasets/_collate.py,sha256=pwUnmrbJH5olFjSwF-ZkGdfopTWUUlwmq0d5KzERcy8,4052
+maite_datasets/_fileio.py,sha256=7S-hF3xU60AdcsPsfYR7rjbeGZUlv3JjGEZhGJOxGYU,5622
+maite_datasets/_reader.py,sha256=tJqsjfXaK-mrs0Ed4BktombFMmNwCur35W7tuYCflKM,5569
+maite_datasets/_validate.py,sha256=Uokbolmv1uSv98sph44HON0HEieeK3s2mqbPMP1d5xs,6948
+maite_datasets/protocols.py,sha256=PcIVJzUc5com_pmhzh0CPb0IkrmFof6WtUQXm7_6Vko,2450
+maite_datasets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+maite_datasets/adapters/__init__.py,sha256=PDMdvRqLVS7x6ghJ_xUBD5Ebq5HrV5k3p4TqK2_6Gt8,191
+maite_datasets/adapters/_huggingface.py,sha256=dZa2guKeto-oKMM9J_TaI4DSPK18u7PvK3AsHQp3eQc,16199
+maite_datasets/image_classification/__init__.py,sha256=pcZojkdsiMoLgY4mKjoQY6WyEwiGYHxNrAGpnvn3zsY,308
+maite_datasets/image_classification/_cifar10.py,sha256=rrkJZ70NBYOSGxligXyakVxNOyGAglN6PaGQazuWNO4,8453
+maite_datasets/image_classification/_mnist.py,sha256=9isgdi-YXgs6nXoh1j8uOgh4_sIhBIky72Vyl866rTE,8192
+maite_datasets/image_classification/_ships.py,sha256=nWhte8592lpybhQCCdgT36LnuMQ0PRJWlDxT5-IPUtk,5137
+maite_datasets/object_detection/__init__.py,sha256=171KT_X6I4YGy18G240N_-ZsKvXJ6YqcBDzkhTiBj2E,587
+maite_datasets/object_detection/_antiuav.py,sha256=8b_AlX5xb5Fu02pG-dldFrHtVZLZzb6ZyGGYj-b7Lt0,8325
+maite_datasets/object_detection/_coco.py,sha256=vTPQmLSgIM7lNb2PEaCrs1fK7_82j3tXFQQ3A5e0dlM,10319
+maite_datasets/object_detection/_milco.py,sha256=aX10PbWjIFyajS5ibGA219fZd7tZTkAlvpThoTZpxtM,7988
+maite_datasets/object_detection/_seadrone.py,sha256=wKcyKQhcvNj5NrLMOD-9KdGaTaBnE6F9uif_J-PKNug,271281
+maite_datasets/object_detection/_voc.py,sha256=CDFgSpbQZgpSq51CKd9twIUHr6l_GAViVGpmFUNX0Dg,19627
+maite_datasets/object_detection/_yolo.py,sha256=GIa43Ec_542N7PRo0XngV-Npw3n4yoIND3tI7vas3u0,11877
+maite_datasets/wrappers/__init__.py,sha256=6uI0ztOB2IlMWln9JkVke4OhU2HQ8i6YCaCNq_q5qb0,225
+maite_datasets/wrappers/_torch.py,sha256=gYcAdrarcX_DiV51MD1cpOa9YAZGEmWblJEyyLS7rRs,4687
+maite_datasets-0.0.8.dist-info/METADATA,sha256=1srwva41Dtqwda2AGc0GwnjQs9f-sFMVSLaMrVQJv7Q,7845
+maite_datasets-0.0.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+maite_datasets-0.0.8.dist-info/licenses/LICENSE,sha256=6h3J3R-ajGHh_isDSftzS5_jJjB9HH4TaI0vU-VscaY,1082
+maite_datasets-0.0.8.dist-info/RECORD,,

maite_datasets-0.0.6.dist-info/RECORD

@@ -1,26 +0,0 @@
-maite_datasets/__init__.py,sha256=Z_HyAe08HaHMjzZS2afFumBXYFRFj0ny5ZAIp0hcj4w,569
-maite_datasets/_base.py,sha256=VEd4ipHPAOCbz4Zm8zdI2yQwQ_x9O4Wq01xoZ2QvNYo,12366
-maite_datasets/_builder.py,sha256=MnCh6z5hSINlzBnK_pdbgI5zSg5d1uq4UvXt3cjn9hs,9820
-maite_datasets/_collate.py,sha256=pwUnmrbJH5olFjSwF-ZkGdfopTWUUlwmq0d5KzERcy8,4052
-maite_datasets/_fileio.py,sha256=7S-hF3xU60AdcsPsfYR7rjbeGZUlv3JjGEZhGJOxGYU,5622
-maite_datasets/_reader.py,sha256=tJqsjfXaK-mrs0Ed4BktombFMmNwCur35W7tuYCflKM,5569
-maite_datasets/_validate.py,sha256=Uokbolmv1uSv98sph44HON0HEieeK3s2mqbPMP1d5xs,6948
-maite_datasets/protocols.py,sha256=YGXb-WxlneXdIQBfBy5OdbylHSVfM-RBXeGvpiWwfLU,607
-maite_datasets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-maite_datasets/image_classification/__init__.py,sha256=pcZojkdsiMoLgY4mKjoQY6WyEwiGYHxNrAGpnvn3zsY,308
-maite_datasets/image_classification/_cifar10.py,sha256=rrkJZ70NBYOSGxligXyakVxNOyGAglN6PaGQazuWNO4,8453
-maite_datasets/image_classification/_mnist.py,sha256=9isgdi-YXgs6nXoh1j8uOgh4_sIhBIky72Vyl866rTE,8192
-maite_datasets/image_classification/_ships.py,sha256=nWhte8592lpybhQCCdgT36LnuMQ0PRJWlDxT5-IPUtk,5137
-maite_datasets/object_detection/__init__.py,sha256=171KT_X6I4YGy18G240N_-ZsKvXJ6YqcBDzkhTiBj2E,587
-maite_datasets/object_detection/_antiuav.py,sha256=B20JrbouDM1o5f1ct9Zfbkks8NaVqYrxu5x-rBZvGx8,8265
-maite_datasets/object_detection/_coco.py,sha256=3abRQJ9ATcZOeqK-4pnMfr-pv7aGcRum88SRlLLXTzk,10309
-maite_datasets/object_detection/_milco.py,sha256=brxxYs5ak0vEpOSd2IW5AMMVkuadVmXCJBFPvXTmNlo,7928
-maite_datasets/object_detection/_seadrone.py,sha256=JdHL0eRZoe7pXVInOq5Xpnz3-vgeBxbO25oTYgGZ44o,271213
-maite_datasets/object_detection/_voc.py,sha256=vgRn-sa_r2-hxwpM3veRZQMcWyqJz9OGalABOccZeow,19589
-maite_datasets/object_detection/_yolo.py,sha256=Luojzhanh6AK949910jN0yTpy8zwF5_At6nThj3Zw9Q,11867
-maite_datasets/wrappers/__init__.py,sha256=6uI0ztOB2IlMWln9JkVke4OhU2HQ8i6YCaCNq_q5qb0,225
-maite_datasets/wrappers/_torch.py,sha256=dmY6nSyLyVPOzpOE4BDTyOomWdFpN0x5dmH3XUzNetc,4588
-maite_datasets-0.0.6.dist-info/METADATA,sha256=cmDnRwPTu1xWbFIpnNQ0jmhEI7XTu6CQ3cm18y0dMDk,5505
-maite_datasets-0.0.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-maite_datasets-0.0.6.dist-info/licenses/LICENSE,sha256=6h3J3R-ajGHh_isDSftzS5_jJjB9HH4TaI0vU-VscaY,1082
-maite_datasets-0.0.6.dist-info/RECORD,,