dataeval 0.86.9__py3-none-any.whl → 0.88.0__py3-none-any.whl

dataeval/utils/datasets/_fileio.py

@@ -1,142 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-import hashlib
-import tarfile
-import zipfile
-from pathlib import Path
-
-import requests
-from tqdm.auto import tqdm
-
-ARCHIVE_ENDINGS = [".zip", ".tar", ".tgz"]
-COMPRESS_ENDINGS = [".gz", ".bz2"]
-
-
-def _print(text: str, verbose: bool) -> None:
-    if verbose:
-        print(text)
-
-
-def _validate_file(fpath: Path | str, file_md5: str, md5: bool = False, chunk_size: int = 65535) -> bool:
-    hasher = hashlib.md5(usedforsecurity=False) if md5 else hashlib.sha256()
-    with open(fpath, "rb") as fpath_file:
-        while chunk := fpath_file.read(chunk_size):
-            hasher.update(chunk)
-    return hasher.hexdigest() == file_md5
-
-
-def _download_dataset(url: str, file_path: Path, timeout: int = 60, verbose: bool = False) -> None:
-    """Download a single resource from its URL to the `data_folder`."""
-    error_msg = "URL fetch failure on {}: {} -- {}"
-    try:
-        response = requests.get(url, stream=True, timeout=timeout)
-        response.raise_for_status()
-    except requests.exceptions.HTTPError as e:
-        raise RuntimeError(f"{error_msg.format(url, e.response.status_code, e.response.reason)}") from e
-    except requests.exceptions.RequestException as e:
-        raise ValueError(f"{error_msg.format(url, 'Unknown error', str(e))}") from e
-
-    total_size = int(response.headers.get("content-length", 0))
-    block_size = 8192  # 8 KB
-    progress_bar = tqdm(total=total_size, unit="iB", unit_scale=True, disable=not verbose)
-
-    with open(file_path, "wb") as f:
-        for chunk in response.iter_content(block_size):
-            f.write(chunk)
-            progress_bar.update(len(chunk))
-    progress_bar.close()
-
-
-def _extract_zip_archive(file_path: Path, extract_to: Path) -> None:
-    """Extracts the zip file to the given directory."""
-    try:
-        with zipfile.ZipFile(file_path, "r") as zip_ref:
-            zip_ref.extractall(extract_to)  # noqa: S202
-            file_path.unlink()
-    except zipfile.BadZipFile:
-        raise FileNotFoundError(f"{file_path.name} is not a valid zip file, skipping extraction.")
-
-
-def _extract_tar_archive(file_path: Path, extract_to: Path) -> None:
-    """Extracts a tar file (or compressed tar) to the specified directory."""
-    try:
-        with tarfile.open(file_path, "r:*") as tar_ref:
-            tar_ref.extractall(extract_to)  # noqa: S202
-            file_path.unlink()
-    except tarfile.TarError:
-        raise FileNotFoundError(f"{file_path.name} is not a valid tar file, skipping extraction.")
-
-
-def _extract_archive(
-    file_ext: str, file_path: Path, directory: Path, compression: bool = False, verbose: bool = False
-) -> None:
-    """
-    Single function to extract and then flatten if necessary.
-    Recursively extracts nested zip files as well.
-    Extracts and flattens all folders to the base directory.
-    """
-    if file_ext != ".zip" or compression:
-        _extract_tar_archive(file_path, directory)
-    else:
-        _extract_zip_archive(file_path, directory)
-    # Look for nested zip files in the extraction directory and extract them recursively.
-    # Does NOT extract in place - extracts everything to directory
-    for child in directory.iterdir():
-        if child.suffix == ".zip":
-            _print(f"Extracting nested zip: {child} to {directory}", verbose)
-            _extract_zip_archive(child, directory)
-
-
-def _ensure_exists(
-    url: str,
-    filename: str,
-    md5: bool,
-    checksum: str,
-    directory: Path,
-    root: Path,
-    download: bool = True,
-    verbose: bool = False,
-) -> None:
-    """
-    For each resource, download it if it doesn't exist in the dataset_dir.
-    If the resource is a zip file, extract it (including recursively extracting nested zips).
-    """
-    file_path = directory / str(filename)
-    alternate_path = root / str(filename)
-    _, file_ext = file_path.stem, file_path.suffix
-    compression = False
-    if file_ext in COMPRESS_ENDINGS:
-        file_ext = file_path.suffixes[0]
-        compression = True
-
-    check_path = alternate_path if alternate_path.exists() and not file_path.exists() else file_path
-
-    # Download file if it doesn't exist.
-    if not check_path.exists() and download:
-        _print(f"Downloading {filename} from {url}", verbose)
-        _download_dataset(url, check_path, verbose=verbose)
-
-        if not _validate_file(check_path, checksum, md5):
-            raise Exception("File checksum mismatch. Remove current file and retry download.")
-
-        # If the file is a zip, tar or tgz extract it into the designated folder.
-        if file_ext in ARCHIVE_ENDINGS:
-            _print(f"Extracting {filename}...", verbose)
-            _extract_archive(file_ext, check_path, directory, compression, verbose)
-
-    elif not check_path.exists() and not download:
-        raise FileNotFoundError(
-            "Data could not be loaded with the provided root directory, "
-            f"the file path to the file {filename} does not exist, "
-            "and the download parameter is set to False."
-        )
-    else:
-        if not _validate_file(check_path, checksum, md5):
-            raise Exception("File checksum mismatch. Remove current file and retry download.")
-        _print(f"{filename} already exists, skipping download.", verbose)
-
-        if file_ext in ARCHIVE_ENDINGS:
-            _print(f"Extracting {filename}...", verbose)
-            _extract_archive(file_ext, check_path, directory, compression, verbose)

dataeval/utils/datasets/_milco.py

@@ -1,197 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, Sequence
-
-from numpy.typing import NDArray
-
-from dataeval.utils.datasets._base import BaseODDataset, DataLocation
-from dataeval.utils.datasets._mixin import BaseDatasetNumpyMixin
-
-if TYPE_CHECKING:
-    from dataeval.typing import Transform
-
-
-class MILCO(BaseODDataset[NDArray[Any], list[str], str], BaseDatasetNumpyMixin):
-    """
-    A side-scan sonar dataset focused on mine-like object detection.
-
-    The dataset comes from the paper
-    `Side-scan sonar imaging data of underwater vehicles for mine detection <https://doi.org/10.1016/j.dib.2024.110132>`_
-    by N.P. Santos et. al. (2024).
-
-    The full dataset contains 1170 side-scan sonar images collected using a 900-1800 kHz Marine Sonic
-    dual frequency side-scan sonar of a Teledyne Marine Gavia Autonomous Underwater Vehicle.
-    All the images were carefully analyzed and annotated, including the image coordinates of the
-    Bounding Box (BB) of the detected objects divided into NOn-Mine-like BOttom Objects (NOMBO)
-    and MIne-Like COntacts (MILCO) classes.
-
-    This dataset is consists of 345 images from 2010, 120 images from 2015, 93 images from 2017, 564 images from 2018,
-    and 48 images from 2021). In these 1170 images, there are 432 MILCO objects, and 235 NOMBO objects.
-    The class “0” corresponds to a MILCO object and the class “1” corresponds to a NOMBO object.
-    The raw BB coordinates provided in the downloaded text files are (x, y, w, h),
-    given as percentages of the image (x_BB = x/img_width, y_BB = y/img_height, etc.).
-    The images come in 2 sizes, 416 x 416 or 1024 x 1024.
-
-    Parameters
-    ----------
-    root : str or pathlib.Path
-        Root directory where the data should be downloaded to or the ``milco`` folder of the already downloaded data.
-    image_set: "train", "operational", or "base", default "train"
-        If "train", then the images from 2015, 2017 and 2021 are selected,
-        resulting in 315 MILCO objects and 177 NOMBO objects.
-        If "operational", then the images from 2010 and 2018 are selected,
-        resulting in 117 MILCO objects and 58 NOMBO objects.
-        If "base", then the full dataset is selected.
-    transforms : Transform, Sequence[Transform] or None, default None
-        Transform(s) to apply to the data.
-    download : bool, default False
-        If True, downloads the dataset from the internet and puts it in root directory.
-        Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
-    verbose : bool, default False
-        If True, outputs print statements.
-
-    Attributes
-    ----------
-    path : pathlib.Path
-        Location of the folder containing the data.
-    image_set : "train", "operational" or "base"
-        The selected image set from the dataset.
-    index2label : dict[int, str]
-        Dictionary which translates from class integers to the associated class strings.
-    label2index : dict[str, int]
-        Dictionary which translates from class strings to the associated class integers.
-    metadata : DatasetMetadata
-        Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
-    transforms : Sequence[Transform]
-        The transforms to be applied to the data.
-    size : int
-        The size of the dataset.
-
-    Note
-    ----
-    Data License: `CC BY 4.0 <https://creativecommons.org/licenses/by/4.0/>`_
-    """
-
-    _resources = [
-        DataLocation(
-            url="https://figshare.com/ndownloader/files/43169002",
-            filename="2015.zip",
-            md5=True,
-            checksum="93dfbb4fb7987734152c372496b4884c",
-        ),
-        DataLocation(
-            url="https://figshare.com/ndownloader/files/43169005",
-            filename="2017.zip",
-            md5=True,
-            checksum="9c2de230a2bbf654921416bea6fc0f42",
-        ),
-        DataLocation(
-            url="https://figshare.com/ndownloader/files/43168999",
-            filename="2021.zip",
-            md5=True,
-            checksum="b84749b21fa95a4a4c7de3741db78bc7",
-        ),
-        DataLocation(
-            url="https://figshare.com/ndownloader/files/43169008",
-            filename="2010.zip",
-            md5=True,
-            checksum="43347a0cc383c0d3dbe0d24ae56f328d",
-        ),
-        DataLocation(
-            url="https://figshare.com/ndownloader/files/43169011",
-            filename="2018.zip",
-            md5=True,
-            checksum="25d091044a10c78674fedad655023e3b",
-        ),
-    ]
-
-    index2label: dict[int, str] = {
-        0: "MILCO",
-        1: "NOMBO",
-    }
-
-    def __init__(
-        self,
-        root: str | Path,
-        image_set: Literal["train", "operational", "base"] = "train",
-        transforms: Transform[NDArray[Any]] | Sequence[Transform[NDArray[Any]]] | None = None,
-        download: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        super().__init__(
-            root,
-            image_set,
-            transforms,
-            download,
-            verbose,
-        )
-        self._bboxes_per_size = True
-
-    def _load_data(self) -> tuple[list[str], list[str], dict[str, list[Any]]]:
-        filepaths: list[str] = []
-        targets: list[str] = []
-        datum_metadata: dict[str, list[Any]] = {}
-        metadata_list: list[dict[str, Any]] = []
-        image_sets: dict[str, list[int]] = {
-            "base": list(range(len(self._resources))),
-            "train": list(range(3)),
-            "operational": list(range(3, len(self._resources))),
-        }
-
-        # Load the data
-        resource_indices = image_sets[self.image_set]
-        for idx in resource_indices:
-            self._resource = self._resources[idx]
-            filepath, target, metadata = super()._load_data()
-            filepaths.extend(filepath)
-            targets.extend(target)
-            metadata_list.append(metadata)
-
-        # Adjust datum metadata to correct format
-        for data_dict in metadata_list:
-            for key, val in data_dict.items():
-                if key not in datum_metadata:
-                    datum_metadata[str(key)] = []
-                datum_metadata[str(key)].extend(val)
-
-        return filepaths, targets, datum_metadata
-
-    def _load_data_inner(self) -> tuple[list[str], list[str], dict[str, Any]]:
-        file_data = {"year": [], "image_id": [], "data_path": [], "label_path": []}
-        data_folder = sorted((self.path / self._resource.filename[:-4]).glob("*.jpg"))
-        if not data_folder:
-            raise FileNotFoundError
-
-        for entry in data_folder:
-            # Remove file extension and split by "_"
-            parts = entry.stem.split("_")
-            file_data["image_id"].append(parts[0])
-            file_data["year"].append(parts[1])
-            file_data["data_path"].append(str(entry))
-            file_data["label_path"].append(str(entry.parent / entry.stem) + ".txt")
-        data = file_data.pop("data_path")
-        annotations = file_data.pop("label_path")
-
-        return data, annotations, file_data
-
-    def _read_annotations(self, annotation: str) -> tuple[list[list[float]], list[int], dict[str, Any]]:
-        """Function for extracting the info out of the text files"""
-        labels: list[int] = []
-        boxes: list[list[float]] = []
-        with open(annotation) as f:
-            for line in f.readlines():
-                out = line.strip().split()
-                labels.append(int(out[0]))
-
-                xcenter, ycenter, width, height = [float(out[1]), float(out[2]), float(out[3]), float(out[4])]
-
-                x0 = xcenter - width / 2
-                x1 = x0 + width
-                y0 = ycenter - height / 2
-                y1 = y0 + height
-                boxes.append([x0, y0, x1, y1])
-
-        return boxes, labels, {}

dataeval/utils/datasets/_mixin.py

@@ -1,54 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-from typing import Any, Generic, TypeVar
-
-import numpy as np
-import torch
-from numpy.typing import NDArray
-from PIL import Image
-
-_TArray = TypeVar("_TArray")
-
-
-class BaseDatasetMixin(Generic[_TArray]):
-    index2label: dict[int, str]
-
-    def _as_array(self, raw: list[Any]) -> _TArray: ...
-    def _one_hot_encode(self, value: int | list[int]) -> _TArray: ...
-    def _read_file(self, path: str) -> _TArray: ...
-
-
-class BaseDatasetNumpyMixin(BaseDatasetMixin[NDArray[Any]]):
-    def _as_array(self, raw: list[Any]) -> NDArray[Any]:
-        return np.asarray(raw)
-
-    def _one_hot_encode(self, value: int | list[int]) -> NDArray[Any]:
-        if isinstance(value, int):
-            encoded = np.zeros(len(self.index2label))
-            encoded[value] = 1
-        else:
-            encoded = np.zeros((len(value), len(self.index2label)))
-            encoded[np.arange(len(value)), value] = 1
-        return encoded
-
-    def _read_file(self, path: str) -> NDArray[Any]:
-        return np.array(Image.open(path)).transpose(2, 0, 1)
-
-
-class BaseDatasetTorchMixin(BaseDatasetMixin[torch.Tensor]):
-    def _as_array(self, raw: list[Any]) -> torch.Tensor:
-        return torch.as_tensor(raw)
-
-    def _one_hot_encode(self, value: int | list[int]) -> torch.Tensor:
-        if isinstance(value, int):
-            encoded = torch.zeros(len(self.index2label))
-            encoded[value] = 1
-        else:
-            encoded = torch.zeros((len(value), len(self.index2label)))
-            encoded[torch.arange(len(value)), value] = 1
-        return encoded
-
-    def _read_file(self, path: str) -> torch.Tensor:
-        return torch.as_tensor(np.array(Image.open(path)).transpose(2, 0, 1))

dataeval/utils/datasets/_mnist.py

@@ -1,202 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, Sequence, TypeVar
-
-import numpy as np
-from numpy.typing import NDArray
-
-from dataeval.utils.datasets._base import BaseICDataset, DataLocation
-from dataeval.utils.datasets._mixin import BaseDatasetNumpyMixin
-
-if TYPE_CHECKING:
-    from dataeval.typing import Transform
-
-MNISTClassStringMap = Literal["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"]
-TMNISTClassMap = TypeVar("TMNISTClassMap", MNISTClassStringMap, int, list[MNISTClassStringMap], list[int])
-CorruptionStringMap = Literal[
-    "identity",
-    "shot_noise",
-    "impulse_noise",
-    "glass_blur",
-    "motion_blur",
-    "shear",
-    "scale",
-    "rotate",
-    "brightness",
-    "translate",
-    "stripe",
-    "fog",
-    "spatter",
-    "dotted_line",
-    "zigzag",
-    "canny_edges",
-]
-
-
-class MNIST(BaseICDataset[NDArray[Any]], BaseDatasetNumpyMixin):
-    """`MNIST <https://en.wikipedia.org/wiki/MNIST_database>`_ Dataset and `Corruptions <https://arxiv.org/abs/1906.02337>`_.
-
-    There are 15 different styles of corruptions. This class downloads differently depending on if you
-    need just the original dataset or if you need corruptions. If you need both a corrupt version and the
-    original version then choose `corruption="identity"` as this downloads all of the corrupt datasets and
-    provides the original as `identity`. If you just need the original, then using `corruption=None` will
-    download only the original dataset to save time and space.
-
-    Parameters
-    ----------
-    root : str or pathlib.Path
-        Root directory where the data should be downloaded to or the ``minst`` folder of the already downloaded data.
-    image_set : "train", "test" or "base", default "train"
-        If "base", returns all of the data to allow the user to create their own splits.
-    corruption : "identity", "shot_noise", "impulse_noise", "glass_blur", "motion_blur", \
-        "shear", "scale", "rotate", "brightness", "translate", "stripe", "fog", "spatter", \
-        "dotted_line", "zigzag", "canny_edges" or None, default None
-        Corruption to apply to the data.
-    transforms : Transform, Sequence[Transform] or None, default None
-        Transform(s) to apply to the data.
-    download : bool, default False
-        If True, downloads the dataset from the internet and puts it in root directory.
-        Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
-    verbose : bool, default False
-        If True, outputs print statements.
-
-    Attributes
-    ----------
-    path : pathlib.Path
-        Location of the folder containing the data.
-    image_set : "train", "test" or "base"
-        The selected image set from the dataset.
-    index2label : dict[int, str]
-        Dictionary which translates from class integers to the associated class strings.
-    label2index : dict[str, int]
-        Dictionary which translates from class strings to the associated class integers.
-    metadata : DatasetMetadata
-        Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
-    corruption : str or None
-        Corruption applied to the data.
-    transforms : Sequence[Transform]
-        The transforms to be applied to the data.
-    size : int
-        The size of the dataset.
-    
-    Note
-    ----
-    Data License: `CC BY 4.0 <https://creativecommons.org/licenses/by/4.0/>`_ for corruption dataset
-    """
-
-    _resources = [
-        DataLocation(
-            url="https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz",
-            filename="mnist.npz",
-            md5=False,
-            checksum="731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1",
-        ),
-        DataLocation(
-            url="https://zenodo.org/record/3239543/files/mnist_c.zip",
-            filename="mnist_c.zip",
-            md5=True,
-            checksum="4b34b33045869ee6d424616cd3a65da3",
-        ),
-    ]
-
-    index2label: dict[int, str] = {
-        0: "zero",
-        1: "one",
-        2: "two",
-        3: "three",
-        4: "four",
-        5: "five",
-        6: "six",
-        7: "seven",
-        8: "eight",
-        9: "nine",
-    }
-
-    def __init__(
-        self,
-        root: str | Path,
-        image_set: Literal["train", "test", "base"] = "train",
-        corruption: CorruptionStringMap | None = None,
-        transforms: Transform[NDArray[Any]] | Sequence[Transform[NDArray[Any]]] | None = None,
-        download: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        self.corruption = corruption
-        if self.corruption == "identity" and verbose:
-            print("Identity is not a corrupted dataset but the original MNIST dataset.")
-        self._resource_index = 0 if self.corruption is None else 1
-
-        super().__init__(
-            root,
-            image_set,
-            transforms,
-            download,
-            verbose,
-        )
-
-    def _load_data_inner(self) -> tuple[list[str], list[int], dict[str, Any]]:
-        """Function to load in the file paths for the data and labels from the correct data format"""
-        if self.corruption is None:
-            try:
-                file_path = self.path / self._resource.filename
-                self._loaded_data, labels = self._grab_data(file_path)
-            except FileNotFoundError:
-                self._loaded_data, labels = self._load_corruption()
-        else:
-            self._loaded_data, labels = self._load_corruption()
-
-        index_strings = np.arange(self._loaded_data.shape[0]).astype(str).tolist()
-        return index_strings, labels.tolist(), {}
-
-    def _load_corruption(self) -> tuple[NDArray[Any], NDArray[np.uintp]]:
-        """Function to load in the file paths for the data and labels for the different corrupt data formats"""
-        corruption = self.corruption if self.corruption is not None else "identity"
-        base_path = self.path / "mnist_c" / corruption
-        if self.image_set == "base":
-            raw_data = []
-            raw_labels = []
-            for group in ["train", "test"]:
-                file_path = base_path / f"{group}_images.npy"
-                raw_data.append(self._grab_corruption_data(file_path))
-
-                label_path = base_path / f"{group}_labels.npy"
-                raw_labels.append(self._grab_corruption_data(label_path))
-
-            data = np.concatenate(raw_data, axis=0).transpose(0, 3, 1, 2)
-            labels = np.concatenate(raw_labels).astype(np.uintp)
-        else:
-            file_path = base_path / f"{self.image_set}_images.npy"
-            data = self._grab_corruption_data(file_path)
-            data = data.astype(np.float64).transpose(0, 3, 1, 2)
-
-            label_path = base_path / f"{self.image_set}_labels.npy"
-            labels = self._grab_corruption_data(label_path)
-            labels = labels.astype(np.uintp)
-
-        return data, labels
-
-    def _grab_data(self, path: Path) -> tuple[NDArray[Any], NDArray[np.uintp]]:
-        """Function to load in the data numpy array"""
-        with np.load(path, allow_pickle=True) as data_array:
-            if self.image_set == "base":
-                data = np.concatenate([data_array["x_train"], data_array["x_test"]], axis=0)
-                labels = np.concatenate([data_array["y_train"], data_array["y_test"]], axis=0).astype(np.uintp)
-            else:
-                data, labels = data_array[f"x_{self.image_set}"], data_array[f"y_{self.image_set}"].astype(np.uintp)
-            data = np.expand_dims(data, axis=1)
-        return data, labels
-
-    def _grab_corruption_data(self, path: Path) -> NDArray[Any]:
-        """Function to load in the data numpy array for the previously chosen corrupt format"""
-        return np.load(path, allow_pickle=False)
-
-    def _read_file(self, path: str) -> NDArray[Any]:
-        """
-        Function to grab the correct image from the loaded data.
-        Overwrite of the base `_read_file` because data is an all or nothing load.
-        """
-        index = int(path)
-        return self._loaded_data[index]