dataeval 0.76.1__py3-none-any.whl → 0.82.0__py3-none-any.whl

dataeval/utils/data/datasets/_milco.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from dataeval.utils.data.datasets._mixin import BaseDatasetNumpyMixin
+
+__all__ = []
+
+from pathlib import Path
+from typing import Any, Sequence
+
+from numpy.typing import NDArray
+
+from dataeval.utils.data.datasets._base import BaseODDataset, DataLocation
+from dataeval.utils.data.datasets._types import Transform
+
+
+class MILCO(BaseODDataset[NDArray[Any]], BaseDatasetNumpyMixin):
+    """
+    A side-scan sonar dataset focused on mine (object) detection.
+
+
+    The dataset comes from the paper
+    `Side-scan sonar imaging data of underwater vehicles for mine detection <https://doi.org/10.1016/j.dib.2024.110132>`_
+    by N.P. Santos et. al. (2024).
+
+    This class only accesses a portion of the above dataset due to size constraints.
+    The full dataset contains 1170 side-scan sonar images collected using a 900-1800 kHz Marine Sonic
+    dual frequency side-scan sonar of a Teledyne Marine Gavia Autonomous Underwater Vehicle.
+    All the images were carefully analyzed and annotated, including the image coordinates of the
+    Bounding Box (BB) of the detected objects divided into NOn-Mine-like BOttom Objects (NOMBO)
+    and MIne-Like COntacts (MILCO) classes.
+
+    This dataset is consists of 261 images (120 images from 2015, 93 images from 2017, and 48 images from 2021).
+    In these 261 images, there are 315 MILCO objects, and 175 NOMBO objects.
+    The class “0” corresponds to a MILCO object and the class “1” corresponds to a NOMBO object.
+    The raw BB coordinates provided in the downloaded text files are (x, y, w, h),
+    given as percentages of the image (x_BB = x/img_width, y_BB = y/img_height, etc.).
+    The images come in 2 sizes, 416 x 416 or 1024 x 1024.
+
+    Parameters
+    ----------
+    root : str or pathlib.Path
+        Root directory of dataset where the ``milco`` folder exists.
+    download : bool, default False
+        If True, downloads the dataset from the internet and puts it in root directory.
+        Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
+    transforms : Transform | Sequence[Transform] | None, default None
+        Transform(s) to apply to the data.
+    verbose : bool, default False
+        If True, outputs print statements.
+
+    Attributes
+    ----------
+    index2label : dict
+        Dictionary which translates from class integers to the associated class strings.
+    label2index : dict
+        Dictionary which translates from class strings to the associated class integers.
+    path : Path
+        Location of the folder containing the data.
+    metadata : dict
+        Dictionary containing Dataset metadata, such as `id` which returns the dataset class name.
+    """
+
+    _resources = [
+        DataLocation(
+            url="https://figshare.com/ndownloader/files/43169002",
+            filename="2015.zip",
+            md5=True,
+            checksum="93dfbb4fb7987734152c372496b4884c",
+        ),
+        DataLocation(
+            url="https://figshare.com/ndownloader/files/43169005",
+            filename="2017.zip",
+            md5=True,
+            checksum="9c2de230a2bbf654921416bea6fc0f42",
+        ),
+        DataLocation(
+            url="https://figshare.com/ndownloader/files/43168999",
+            filename="2021.zip",
+            md5=True,
+            checksum="b84749b21fa95a4a4c7de3741db78bc7",
+        ),
+    ]
+
+    index2label: dict[int, str] = {
+        0: "MILCO",
+        1: "NOMBO",
+    }
+
+    def __init__(
+        self,
+        root: str | Path,
+        download: bool = False,
+        transforms: Transform[NDArray[Any]] | Sequence[Transform[NDArray[Any]]] | None = None,
+        verbose: bool = False,
+    ) -> None:
+        super().__init__(
+            root,
+            download,
+            "base",
+            transforms,
+            verbose,
+        )
+
+    def _load_data(self) -> tuple[list[str], list[str], dict[str, list[Any]]]:
+        filepaths: list[str] = []
+        targets: list[str] = []
+        datum_metadata: dict[str, list[Any]] = {}
+        metadata_list: list[dict[str, Any]] = []
+
+        # Load the data
+        for resource in self._resources:
+            self._resource = resource
+            filepath, target, metadata = super()._load_data()
+            filepaths.extend(filepath)
+            targets.extend(target)
+            metadata_list.append(metadata)
+
+        # Adjust datum metadata to correct format
+        for data_dict in metadata_list:
+            for key, val in data_dict.items():
+                if key not in datum_metadata:
+                    datum_metadata[str(key)] = []
+                datum_metadata[str(key)].extend(val)
+
+        return filepaths, targets, datum_metadata
+
+    def _load_data_inner(self) -> tuple[list[str], list[str], dict[str, Any]]:
+        file_data = {"year": [], "image_id": [], "data_path": [], "label_path": []}
+        data_folder = self.path / self._resource.filename[:-4]
+        for entry in data_folder.iterdir():
+            if entry.is_file() and entry.suffix == ".jpg":
+                # Remove file extension and split by "_"
+                parts = entry.stem.split("_")
+                file_data["image_id"].append(parts[0])
+                file_data["year"].append(parts[1])
+                file_data["data_path"].append(str(entry))
+                file_data["label_path"].append(str(entry.parent / entry.stem) + ".txt")
+        data = file_data.pop("data_path")
+        annotations = file_data.pop("label_path")
+
+        return data, annotations, file_data
+
+    def _read_annotations(self, annotation: str) -> tuple[list[list[float]], list[int], dict[str, Any]]:
+        """Function for extracting the info out of the text files"""
+        labels: list[int] = []
+        boxes: list[list[float]] = []
+        with open(annotation) as f:
+            for line in f.readlines():
+                out = line.strip().split(" ")
+                labels.append(int(out[0]))
+                boxes.append([float(out[1]), float(out[2]), float(out[3]), float(out[4])])
+
+        return boxes, labels, {}

dataeval/utils/data/datasets/_mixin.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+__all__ = []
+
+from typing import Any, Generic, TypeVar
+
+import numpy as np
+import torch
+from numpy.typing import NDArray
+from PIL import Image
+
+_TArray = TypeVar("_TArray")
+
+
+class BaseDatasetMixin(Generic[_TArray]):
+    index2label: dict[int, str]
+
+    def _as_array(self, raw: list[Any]) -> _TArray: ...
+    def _one_hot_encode(self, value: int | list[int]) -> _TArray: ...
+    def _read_file(self, path: str) -> _TArray: ...
+
+
+class BaseDatasetNumpyMixin(BaseDatasetMixin[NDArray[Any]]):
+    def _as_array(self, raw: list[Any]) -> NDArray[Any]:
+        return np.asarray(raw)
+
+    def _one_hot_encode(self, value: int | list[int]) -> NDArray[Any]:
+        if isinstance(value, int):
+            encoded = np.zeros(len(self.index2label))
+            encoded[value] = 1
+        else:
+            encoded = np.zeros((len(value), len(self.index2label)))
+            encoded[np.arange(len(value)), value] = 1
+        return encoded
+
+    def _read_file(self, path: str) -> NDArray[Any]:
+        x = np.array(Image.open(path)).transpose(2, 0, 1)
+        return x
+
+
+class BaseDatasetTorchMixin(BaseDatasetMixin[torch.Tensor]):
+    def _as_array(self, raw: list[Any]) -> torch.Tensor:
+        return torch.as_tensor(raw)
+
+    def _one_hot_encode(self, value: int | list[int]) -> torch.Tensor:
+        if isinstance(value, int):
+            encoded = torch.zeros(len(self.index2label))
+            encoded[value] = 1
+        else:
+            encoded = torch.zeros((len(value), len(self.index2label)))
+            encoded[torch.arange(len(value)), value] = 1
+        return encoded
+
+    def _read_file(self, path: str) -> torch.Tensor:
+        x = torch.as_tensor(np.array(Image.open(path)).transpose(2, 0, 1))
+        return x

dataeval/utils/data/datasets/_mnist.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+__all__ = []
+
+from pathlib import Path
+from typing import Any, Literal, Sequence, TypeVar
+
+import numpy as np
+from numpy.typing import NDArray
+
+from dataeval.utils.data.datasets._base import BaseICDataset, DataLocation
+from dataeval.utils.data.datasets._mixin import BaseDatasetNumpyMixin
+from dataeval.utils.data.datasets._types import Transform
+
+MNISTClassStringMap = Literal["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"]
+TMNISTClassMap = TypeVar("TMNISTClassMap", MNISTClassStringMap, int, list[MNISTClassStringMap], list[int])
+CorruptionStringMap = Literal[
+    "identity",
+    "shot_noise",
+    "impulse_noise",
+    "glass_blur",
+    "motion_blur",
+    "shear",
+    "scale",
+    "rotate",
+    "brightness",
+    "translate",
+    "stripe",
+    "fog",
+    "spatter",
+    "dotted_line",
+    "zigzag",
+    "canny_edges",
+]
+
+
+class MNIST(BaseICDataset[NDArray[Any]], BaseDatasetNumpyMixin):
+    """`MNIST <https://en.wikipedia.org/wiki/MNIST_database>`_ Dataset and `Corruptions <https://arxiv.org/abs/1906.02337>`_.
+
+    There are 15 different styles of corruptions. This class downloads differently depending on if you
+    need just the original dataset or if you need corruptions. If you need both a corrupt version and the
+    original version then choose `corruption="identity"` as this downloads all of the corrupt datasets and
+    provides the original as `identity`. If you just need the original, then using `corruption=None` will
+    download only the original dataset to save time and space.
+
+    Parameters
+    ----------
+    root : str or pathlib.Path
+        Root directory of dataset where the ``mnist`` folder exists.
+    download : bool, default False
+        If True, downloads the dataset from the internet and puts it in root directory.
+        Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
+    image_set : "train", "test" or "base", default "train"
+        If "base", returns all of the data to allow the user to create their own splits.
+    verbose : bool, default False
+        If True, outputs print statements.
+
+    Attributes
+    ----------
+    index2label : dict
+        Dictionary which translates from class integers to the associated class strings.
+    label2index : dict
+        Dictionary which translates from class strings to the associated class integers.
+    path : Path
+        Location of the folder containing the data.
+    metadata : dict
+        Dictionary containing Dataset metadata, such as `id` which returns the dataset class name.
+    """
+
+    _resources = [
+        DataLocation(
+            url="https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz",
+            filename="mnist.npz",
+            md5=False,
+            checksum="731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1",
+        ),
+        DataLocation(
+            url="https://zenodo.org/record/3239543/files/mnist_c.zip",
+            filename="mnist_c.zip",
+            md5=True,
+            checksum="4b34b33045869ee6d424616cd3a65da3",
+        ),
+    ]
+
+    index2label: dict[int, str] = {
+        0: "zero",
+        1: "one",
+        2: "two",
+        3: "three",
+        4: "four",
+        5: "five",
+        6: "six",
+        7: "seven",
+        8: "eight",
+        9: "nine",
+    }
+
+    def __init__(
+        self,
+        root: str | Path,
+        download: bool = False,
+        image_set: Literal["train", "test", "base"] = "train",
+        corruption: CorruptionStringMap | None = None,
+        transforms: Transform[NDArray[Any]] | Sequence[Transform[NDArray[Any]]] | None = None,
+        verbose: bool = False,
+    ) -> None:
+        self.corruption = corruption
+        if self.corruption == "identity" and verbose:
+            print("Identity is not a corrupted dataset but the original MNIST dataset.")
+        self._resource_index = 0 if self.corruption is None else 1
+
+        super().__init__(
+            root,
+            download,
+            image_set,
+            transforms,
+            verbose,
+        )
+
+    def _load_data_inner(self) -> tuple[list[str], list[int], dict[str, Any]]:
+        """Function to load in the file paths for the data and labels from the correct data format"""
+        if self.corruption is None:
+            try:
+                file_path = self.path / self._resource.filename
+                self._loaded_data, labels = self._grab_data(file_path)
+            except FileNotFoundError:
+                self._loaded_data, labels = self._load_corruption()
+        else:
+            self._loaded_data, labels = self._load_corruption()
+
+        index_strings = np.arange(self._loaded_data.shape[0]).astype(str).tolist()
+        return index_strings, labels.tolist(), {}
+
+    def _load_corruption(self) -> tuple[NDArray[Any], NDArray[np.uintp]]:
+        """Function to load in the file paths for the data and labels for the different corrupt data formats"""
+        corruption = self.corruption if self.corruption is not None else "identity"
+        base_path = self.path / corruption
+        if self.image_set == "base":
+            raw_data = []
+            raw_labels = []
+            for group in ["train", "test"]:
+                file_path = base_path / f"{group}_images.npy"
+                raw_data.append(self._grab_corruption_data(file_path))
+
+                label_path = base_path / f"{group}_labels.npy"
+                raw_labels.append(self._grab_corruption_data(label_path))
+
+            data = np.concatenate(raw_data, axis=0).transpose(0, 3, 1, 2)
+            labels = np.concatenate(raw_labels).astype(np.uintp)
+        else:
+            file_path = base_path / f"{self.image_set}_images.npy"
+            data = self._grab_corruption_data(file_path)
+            data = data.astype(np.float64).transpose(0, 3, 1, 2)
+
+            label_path = base_path / f"{self.image_set}_labels.npy"
+            labels = self._grab_corruption_data(label_path)
+            labels = labels.astype(np.uintp)
+
+        return data, labels
+
+    def _grab_data(self, path: Path) -> tuple[NDArray[Any], NDArray[np.uintp]]:
+        """Function to load in the data numpy array"""
+        with np.load(path, allow_pickle=True) as data_array:
+            if self.image_set == "base":
+                data = np.concatenate([data_array["x_train"], data_array["x_test"]], axis=0)
+                labels = np.concatenate([data_array["y_train"], data_array["y_test"]], axis=0).astype(np.uintp)
+            else:
+                data, labels = data_array[f"x_{self.image_set}"], data_array[f"y_{self.image_set}"].astype(np.uintp)
+            data = np.expand_dims(data, axis=1)
+        return data, labels
+
+    def _grab_corruption_data(self, path: Path) -> NDArray[Any]:
+        """Function to load in the data numpy array for the previously chosen corrupt format"""
+        x = np.load(path, allow_pickle=False)
+        return x
+
+    def _read_file(self, path: str) -> NDArray[Any]:
+        """
+        Function to grab the correct image from the loaded data.
+        Overwrite of the base `_read_file` because data is an all or nothing load.
+        """
+        index = int(path)
+        return self._loaded_data[index]

dataeval/utils/data/datasets/_ships.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+__all__ = []
+
+from pathlib import Path
+from typing import Any, Sequence
+
+import numpy as np
+from numpy.typing import NDArray
+
+from dataeval.utils.data.datasets._base import BaseICDataset, DataLocation
+from dataeval.utils.data.datasets._mixin import BaseDatasetNumpyMixin
+from dataeval.utils.data.datasets._types import Transform
+
+
+class Ships(BaseICDataset[NDArray[Any]], BaseDatasetNumpyMixin):
+    """
+    A dataset that focuses on identifying ships from satellite images.
+
+    The dataset comes from kaggle,
+    `Ships in Satellite Imagery <https://www.kaggle.com/datasets/rhammell/ships-in-satellite-imagery>`_.
+    The images come from Planet satellite imagery when they gave
+    `open-access to a portion of their data <https://www.planet.com/pulse/open-california-rapideye-data/>`_.
+
+    There are 4000 80x80x3 (HWC) images of ships, sea, and land.
+    There are also 8 larger scene images similar to what would be operationally provided.
+
+    Parameters
+    ----------
+    root : str or pathlib.Path
+        Root directory of dataset where the ``shipdataset`` folder exists.
+    download : bool, default False
+        If True, downloads the dataset from the internet and puts it in root directory.
+        Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
+    verbose : bool, default False
+        If True, outputs print statements.
+
+    Attributes
+    ----------
+    index2label : dict
+        Dictionary which translates from class integers to the associated class strings.
+    label2index : dict
+        Dictionary which translates from class strings to the associated class integers.
+    path : Path
+        Location of the folder containing the data.
+    metadata : dict
+        Dictionary containing Dataset metadata, such as `id` which returns the dataset class name.
+    """
+
+    _resources = [
+        DataLocation(
+            url="https://zenodo.org/record/3611230/files/ships-in-satellite-imagery.zip",
+            filename="ships-in-satellite-imagery.zip",
+            md5=True,
+            checksum="b2e8a41ed029592b373bd72ee4b89f32",
+        ),
+    ]
+
+    index2label: dict[int, str] = {
+        0: "no ship",
+        1: "ship",
+    }
+
+    def __init__(
+        self,
+        root: str | Path,
+        download: bool = False,
+        transforms: Transform[NDArray[Any]] | Sequence[Transform[NDArray[Any]]] | None = None,
+        verbose: bool = False,
+    ) -> None:
+        super().__init__(
+            root,
+            download,
+            "base",
+            transforms,
+            verbose,
+        )
+        self._scenes: list[str] = self._load_scenes()
+
+    def _load_data_inner(self) -> tuple[list[str], list[int], dict[str, Any]]:
+        """Function to load in the file paths for the data and labels"""
+        file_data = {"label": [], "scene_id": [], "longitude": [], "latitude": [], "path": []}
+        data_folder = self.path / "shipsnet"
+        for entry in data_folder.iterdir():
+            # Remove file extension and split by "_"
+            parts = entry.stem.split("__")  # Removes ".png" and splits the string
+            file_data["label"].append(int(parts[0]))
+            file_data["scene_id"].append(parts[1])
+            lat_lon = parts[2].split("_")
+            file_data["longitude"].append(float(lat_lon[0]))
+            file_data["latitude"].append(float(lat_lon[1]))
+            file_data["path"].append(entry)
+        data = file_data.pop("path")
+        labels = file_data.pop("label")
+        return data, labels, file_data
+
+    def _load_scenes(self) -> list[str]:
+        """Function to load in the file paths for the scene images"""
+        data_folder = self.path / "scenes"
+        scene = [str(entry) for entry in data_folder.iterdir()]
+        return scene
+
+    def get_scene(self, index: int) -> NDArray[np.uintp]:
+        """
+        Get the desired satellite image (scene) by passing in the index of the desired file.
+
+        Args
+        ----
+        index : int
+            Value of the desired data point
+
+        Returns
+        -------
+        NDArray[np.uintp]
+            Scene image
+
+        Note
+        ----
+        The scene will be returned with the channel axis first.
+        """
+        scene = self._read_file(self._scenes[index])
+        np.moveaxis(scene, -1, 0)
+        return scene

dataeval/utils/data/datasets/_types.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+__all__ = []
+
+from dataclasses import dataclass
+from typing import Any, Generic, Protocol, TypedDict, TypeVar
+
+from torch.utils.data import Dataset
+from typing_extensions import NotRequired, Required
+
+
+class DatasetMetadata(TypedDict):
+    id: Required[str]
+    index2label: NotRequired[dict[int, str]]
+    split: NotRequired[str]
+
+
+_TDatum = TypeVar("_TDatum")
+_TArray = TypeVar("_TArray")
+
+
+class AnnotatedDataset(Dataset[_TDatum]):
+    metadata: DatasetMetadata
+
+    def __len__(self) -> int: ...
+
+
+class ImageClassificationDataset(AnnotatedDataset[tuple[_TArray, _TArray, dict[str, Any]]]): ...
+
+
+@dataclass
+class ObjectDetectionTarget(Generic[_TArray]):
+    boxes: _TArray
+    labels: _TArray
+    scores: _TArray
+
+
+class ObjectDetectionDataset(AnnotatedDataset[tuple[_TArray, ObjectDetectionTarget[_TArray], dict[str, Any]]]): ...
+
+
+@dataclass
+class SegmentationTarget(Generic[_TArray]):
+    mask: _TArray
+    labels: _TArray
+    scores: _TArray
+
+
+class SegmentationDataset(AnnotatedDataset[tuple[_TArray, SegmentationTarget[_TArray], dict[str, Any]]]): ...
+
+
+class Transform(Generic[_TArray], Protocol):
+    def __call__(self, data: _TArray, /) -> _TArray: ...