dataeval 0.86.9__py3-none-any.whl → 0.87.0__py3-none-any.whl

dataeval/utils/datasets/_cifar10.py

@@ -1,201 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, Sequence, TypeVar
-
-import numpy as np
-from numpy.typing import NDArray
-
-from dataeval.utils.datasets._base import BaseICDataset, DataLocation
-from dataeval.utils.datasets._mixin import BaseDatasetNumpyMixin
-
-if TYPE_CHECKING:
-    from dataeval.typing import Transform
-
-CIFARClassStringMap = Literal["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"]
-TCIFARClassMap = TypeVar("TCIFARClassMap", CIFARClassStringMap, int, list[CIFARClassStringMap], list[int])
-
-
-class CIFAR10(BaseICDataset[NDArray[Any]], BaseDatasetNumpyMixin):
-    """
-    `CIFAR10 <https://www.cs.toronto.edu/~kriz/cifar.html>`_ Dataset as NumPy arrays.
-
-    Parameters
-    ----------
-    root : str or pathlib.Path
-        Root directory where the data should be downloaded to or the ``cifar10`` folder of the already downloaded data.
-    image_set : "train", "test" or "base", default "train"
-        If "base", returns all of the data to allow the user to create their own splits.
-    transforms : Transform, Sequence[Transform] or None, default None
-        Transform(s) to apply to the data.
-    download : bool, default False
-        If True, downloads the dataset from the internet and puts it in root directory.
-        Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
-    verbose : bool, default False
-        If True, outputs print statements.
-
-    Attributes
-    ----------
-    path : pathlib.Path
-        Location of the folder containing the data.
-    image_set : "train", "test" or "base"
-        The selected image set from the dataset.
-    transforms : Sequence[Transform]
-        The transforms to be applied to the data.
-    size : int
-        The size of the dataset.
-    index2label : dict[int, str]
-        Dictionary which translates from class integers to the associated class strings.
-    label2index : dict[str, int]
-        Dictionary which translates from class strings to the associated class integers.
-    metadata : DatasetMetadata
-        Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
-    """
-
-    _resources = [
-        DataLocation(
-            url="https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz",
-            filename="cifar-10-binary.tar.gz",
-            md5=True,
-            checksum="c32a1d4ab5d03f1284b67883e8d87530",
-        ),
-    ]
-
-    index2label: dict[int, str] = {
-        0: "airplane",
-        1: "automobile",
-        2: "bird",
-        3: "cat",
-        4: "deer",
-        5: "dog",
-        6: "frog",
-        7: "horse",
-        8: "ship",
-        9: "truck",
-    }
-
-    def __init__(
-        self,
-        root: str | Path,
-        image_set: Literal["train", "test", "base"] = "train",
-        transforms: Transform[NDArray[Any]] | Sequence[Transform[NDArray[Any]]] | None = None,
-        download: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        super().__init__(
-            root,
-            image_set,
-            transforms,
-            download,
-            verbose,
-        )
-
-    def _load_bin_data(self, data_folder: list[Path]) -> tuple[list[str], list[int], dict[str, Any]]:
-        batch_nums = np.zeros(60000, dtype=np.uint8)
-        all_labels = np.zeros(60000, dtype=np.uint8)
-        all_images = np.zeros((60000, 3, 32, 32), dtype=np.uint8)
-        # Process each batch file, skipping .meta and .html files
-        for batch_file in data_folder:
-            # Get batch parameters
-            batch_type = "test" if "test" in batch_file.stem else "train"
-            batch_num = 5 if batch_type == "test" else int(batch_file.stem.split("_")[-1]) - 1
-
-            # Load data
-            batch_images, batch_labels = self._unpack_batch_files(batch_file)
-
-            # Stack data
-            num_images = batch_images.shape[0]
-            batch_start = batch_num * num_images
-            all_images[batch_start : batch_start + num_images] = batch_images
-            all_labels[batch_start : batch_start + num_images] = batch_labels
-            batch_nums[batch_start : batch_start + num_images] = batch_num
-
-        # Save data
-        self._loaded_data = all_images
-        np.savez(self.path / "cifar10", images=self._loaded_data, labels=all_labels, batches=batch_nums)
-
-        # Select data
-        image_list = np.arange(all_labels.shape[0]).astype(str)
-        if self.image_set == "train":
-            return (
-                image_list[np.nonzero(batch_nums != 5)[0]].tolist(),
-                all_labels[batch_nums != 5].tolist(),
-                {"batch_num": batch_nums[batch_nums != 5].tolist()},
-            )
-        if self.image_set == "test":
-            return (
-                image_list[np.nonzero(batch_nums == 5)[0]].tolist(),
-                all_labels[batch_nums == 5].tolist(),
-                {"batch_num": batch_nums[batch_nums == 5].tolist()},
-            )
-        return image_list.tolist(), all_labels.tolist(), {"batch_num": batch_nums.tolist()}
-
-    def _load_data_inner(self) -> tuple[list[str], list[int], dict[str, Any]]:
-        """Function to load in the file paths for the data and labels and retrieve metadata"""
-        data_file = self.path / "cifar10.npz"
-        if not data_file.exists():
-            data_folder = sorted((self.path / "cifar-10-batches-bin").glob("*.bin"))
-            if not data_folder:
-                raise FileNotFoundError
-            return self._load_bin_data(data_folder)
-
-        # Load data
-        data = np.load(data_file)
-        self._loaded_data = data["images"]
-        all_labels = data["labels"]
-        batch_nums = data["batches"]
-
-        # Select data
-        image_list = np.arange(all_labels.shape[0]).astype(str)
-        if self.image_set == "train":
-            return (
-                image_list[np.nonzero(batch_nums != 5)[0]].tolist(),
-                all_labels[batch_nums != 5].tolist(),
-                {"batch_num": batch_nums[batch_nums != 5].tolist()},
-            )
-        if self.image_set == "test":
-            return (
-                image_list[np.nonzero(batch_nums == 5)[0]].tolist(),
-                all_labels[batch_nums == 5].tolist(),
-                {"batch_num": batch_nums[batch_nums == 5].tolist()},
-            )
-        return image_list.tolist(), all_labels.tolist(), {"batch_num": batch_nums.tolist()}
-
-    def _unpack_batch_files(self, file_path: Path) -> tuple[NDArray[np.uint8], NDArray[np.uint8]]:
-        # Load pickle data with latin1 encoding
-        with file_path.open("rb") as f:
-            buffer = np.frombuffer(f.read(), dtype=np.uint8)
-            # Each entry is 1 byte for label + 3072 bytes for image (3*32*32)
-            entry_size = 1 + 3072
-            num_entries = buffer.size // entry_size
-            # Extract labels (first byte of each entry)
-            labels = buffer[::entry_size]
-
-            # Extract image data and reshape to (N, 3, 32, 32)
-            images = np.zeros((num_entries, 3, 32, 32), dtype=np.uint8)
-            for i in range(num_entries):
-                # Skip the label byte and get image data for this entry
-                start_idx = i * entry_size + 1  # +1 to skip label
-                img_flat = buffer[start_idx : start_idx + 3072]
-
-                # The CIFAR format stores channels in blocks (all R, then all G, then all B)
-                # Each channel block is 1024 bytes (32x32)
-                red_channel = img_flat[0:1024].reshape(32, 32)
-                green_channel = img_flat[1024:2048].reshape(32, 32)
-                blue_channel = img_flat[2048:3072].reshape(32, 32)
-
-                # Stack the channels in the proper C×H×W format
-                images[i, 0] = red_channel  # Red channel
-                images[i, 1] = green_channel  # Green channel
-                images[i, 2] = blue_channel  # Blue channel
-        return images, labels
-
-    def _read_file(self, path: str) -> NDArray[Any]:
-        """
-        Function to grab the correct image from the loaded data.
-        Overwrite of the base `_read_file` because data is an all or nothing load.
-        """
-        index = int(path)
-        return self._loaded_data[index]

dataeval/utils/datasets/_fileio.py

@@ -1,142 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-import hashlib
-import tarfile
-import zipfile
-from pathlib import Path
-
-import requests
-from tqdm.auto import tqdm
-
-ARCHIVE_ENDINGS = [".zip", ".tar", ".tgz"]
-COMPRESS_ENDINGS = [".gz", ".bz2"]
-
-
-def _print(text: str, verbose: bool) -> None:
-    if verbose:
-        print(text)
-
-
-def _validate_file(fpath: Path | str, file_md5: str, md5: bool = False, chunk_size: int = 65535) -> bool:
-    hasher = hashlib.md5(usedforsecurity=False) if md5 else hashlib.sha256()
-    with open(fpath, "rb") as fpath_file:
-        while chunk := fpath_file.read(chunk_size):
-            hasher.update(chunk)
-    return hasher.hexdigest() == file_md5
-
-
-def _download_dataset(url: str, file_path: Path, timeout: int = 60, verbose: bool = False) -> None:
-    """Download a single resource from its URL to the `data_folder`."""
-    error_msg = "URL fetch failure on {}: {} -- {}"
-    try:
-        response = requests.get(url, stream=True, timeout=timeout)
-        response.raise_for_status()
-    except requests.exceptions.HTTPError as e:
-        raise RuntimeError(f"{error_msg.format(url, e.response.status_code, e.response.reason)}") from e
-    except requests.exceptions.RequestException as e:
-        raise ValueError(f"{error_msg.format(url, 'Unknown error', str(e))}") from e
-
-    total_size = int(response.headers.get("content-length", 0))
-    block_size = 8192  # 8 KB
-    progress_bar = tqdm(total=total_size, unit="iB", unit_scale=True, disable=not verbose)
-
-    with open(file_path, "wb") as f:
-        for chunk in response.iter_content(block_size):
-            f.write(chunk)
-            progress_bar.update(len(chunk))
-    progress_bar.close()
-
-
-def _extract_zip_archive(file_path: Path, extract_to: Path) -> None:
-    """Extracts the zip file to the given directory."""
-    try:
-        with zipfile.ZipFile(file_path, "r") as zip_ref:
-            zip_ref.extractall(extract_to)  # noqa: S202
-            file_path.unlink()
-    except zipfile.BadZipFile:
-        raise FileNotFoundError(f"{file_path.name} is not a valid zip file, skipping extraction.")
-
-
-def _extract_tar_archive(file_path: Path, extract_to: Path) -> None:
-    """Extracts a tar file (or compressed tar) to the specified directory."""
-    try:
-        with tarfile.open(file_path, "r:*") as tar_ref:
-            tar_ref.extractall(extract_to)  # noqa: S202
-            file_path.unlink()
-    except tarfile.TarError:
-        raise FileNotFoundError(f"{file_path.name} is not a valid tar file, skipping extraction.")
-
-
-def _extract_archive(
-    file_ext: str, file_path: Path, directory: Path, compression: bool = False, verbose: bool = False
-) -> None:
-    """
-    Single function to extract and then flatten if necessary.
-    Recursively extracts nested zip files as well.
-    Extracts and flattens all folders to the base directory.
-    """
-    if file_ext != ".zip" or compression:
-        _extract_tar_archive(file_path, directory)
-    else:
-        _extract_zip_archive(file_path, directory)
-    # Look for nested zip files in the extraction directory and extract them recursively.
-    # Does NOT extract in place - extracts everything to directory
-    for child in directory.iterdir():
-        if child.suffix == ".zip":
-            _print(f"Extracting nested zip: {child} to {directory}", verbose)
-            _extract_zip_archive(child, directory)
-
-
-def _ensure_exists(
-    url: str,
-    filename: str,
-    md5: bool,
-    checksum: str,
-    directory: Path,
-    root: Path,
-    download: bool = True,
-    verbose: bool = False,
-) -> None:
-    """
-    For each resource, download it if it doesn't exist in the dataset_dir.
-    If the resource is a zip file, extract it (including recursively extracting nested zips).
-    """
-    file_path = directory / str(filename)
-    alternate_path = root / str(filename)
-    _, file_ext = file_path.stem, file_path.suffix
-    compression = False
-    if file_ext in COMPRESS_ENDINGS:
-        file_ext = file_path.suffixes[0]
-        compression = True
-
-    check_path = alternate_path if alternate_path.exists() and not file_path.exists() else file_path
-
-    # Download file if it doesn't exist.
-    if not check_path.exists() and download:
-        _print(f"Downloading {filename} from {url}", verbose)
-        _download_dataset(url, check_path, verbose=verbose)
-
-        if not _validate_file(check_path, checksum, md5):
-            raise Exception("File checksum mismatch. Remove current file and retry download.")
-
-        # If the file is a zip, tar or tgz extract it into the designated folder.
-        if file_ext in ARCHIVE_ENDINGS:
-            _print(f"Extracting {filename}...", verbose)
-            _extract_archive(file_ext, check_path, directory, compression, verbose)
-
-    elif not check_path.exists() and not download:
-        raise FileNotFoundError(
-            "Data could not be loaded with the provided root directory, "
-            f"the file path to the file {filename} does not exist, "
-            "and the download parameter is set to False."
-        )
-    else:
-        if not _validate_file(check_path, checksum, md5):
-            raise Exception("File checksum mismatch. Remove current file and retry download.")
-        _print(f"{filename} already exists, skipping download.", verbose)
-
-        if file_ext in ARCHIVE_ENDINGS:
-            _print(f"Extracting {filename}...", verbose)
-            _extract_archive(file_ext, check_path, directory, compression, verbose)

dataeval/utils/datasets/_milco.py

@@ -1,197 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, Sequence
-
-from numpy.typing import NDArray
-
-from dataeval.utils.datasets._base import BaseODDataset, DataLocation
-from dataeval.utils.datasets._mixin import BaseDatasetNumpyMixin
-
-if TYPE_CHECKING:
-    from dataeval.typing import Transform
-
-
-class MILCO(BaseODDataset[NDArray[Any], list[str], str], BaseDatasetNumpyMixin):
-    """
-    A side-scan sonar dataset focused on mine-like object detection.
-
-    The dataset comes from the paper
-    `Side-scan sonar imaging data of underwater vehicles for mine detection <https://doi.org/10.1016/j.dib.2024.110132>`_
-    by N.P. Santos et. al. (2024).
-
-    The full dataset contains 1170 side-scan sonar images collected using a 900-1800 kHz Marine Sonic
-    dual frequency side-scan sonar of a Teledyne Marine Gavia Autonomous Underwater Vehicle.
-    All the images were carefully analyzed and annotated, including the image coordinates of the
-    Bounding Box (BB) of the detected objects divided into NOn-Mine-like BOttom Objects (NOMBO)
-    and MIne-Like COntacts (MILCO) classes.
-
-    This dataset is consists of 345 images from 2010, 120 images from 2015, 93 images from 2017, 564 images from 2018,
-    and 48 images from 2021). In these 1170 images, there are 432 MILCO objects, and 235 NOMBO objects.
-    The class “0” corresponds to a MILCO object and the class “1” corresponds to a NOMBO object.
-    The raw BB coordinates provided in the downloaded text files are (x, y, w, h),
-    given as percentages of the image (x_BB = x/img_width, y_BB = y/img_height, etc.).
-    The images come in 2 sizes, 416 x 416 or 1024 x 1024.
-
-    Parameters
-    ----------
-    root : str or pathlib.Path
-        Root directory where the data should be downloaded to or the ``milco`` folder of the already downloaded data.
-    image_set: "train", "operational", or "base", default "train"
-        If "train", then the images from 2015, 2017 and 2021 are selected,
-        resulting in 315 MILCO objects and 177 NOMBO objects.
-        If "operational", then the images from 2010 and 2018 are selected,
-        resulting in 117 MILCO objects and 58 NOMBO objects.
-        If "base", then the full dataset is selected.
-    transforms : Transform, Sequence[Transform] or None, default None
-        Transform(s) to apply to the data.
-    download : bool, default False
-        If True, downloads the dataset from the internet and puts it in root directory.
-        Class checks to see if data is already downloaded to ensure it does not create a duplicate download.
-    verbose : bool, default False
-        If True, outputs print statements.
-
-    Attributes
-    ----------
-    path : pathlib.Path
-        Location of the folder containing the data.
-    image_set : "train", "operational" or "base"
-        The selected image set from the dataset.
-    index2label : dict[int, str]
-        Dictionary which translates from class integers to the associated class strings.
-    label2index : dict[str, int]
-        Dictionary which translates from class strings to the associated class integers.
-    metadata : DatasetMetadata
-        Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
-    transforms : Sequence[Transform]
-        The transforms to be applied to the data.
-    size : int
-        The size of the dataset.
-
-    Note
-    ----
-    Data License: `CC BY 4.0 <https://creativecommons.org/licenses/by/4.0/>`_
-    """
-
-    _resources = [
-        DataLocation(
-            url="https://figshare.com/ndownloader/files/43169002",
-            filename="2015.zip",
-            md5=True,
-            checksum="93dfbb4fb7987734152c372496b4884c",
-        ),
-        DataLocation(
-            url="https://figshare.com/ndownloader/files/43169005",
-            filename="2017.zip",
-            md5=True,
-            checksum="9c2de230a2bbf654921416bea6fc0f42",
-        ),
-        DataLocation(
-            url="https://figshare.com/ndownloader/files/43168999",
-            filename="2021.zip",
-            md5=True,
-            checksum="b84749b21fa95a4a4c7de3741db78bc7",
-        ),
-        DataLocation(
-            url="https://figshare.com/ndownloader/files/43169008",
-            filename="2010.zip",
-            md5=True,
-            checksum="43347a0cc383c0d3dbe0d24ae56f328d",
-        ),
-        DataLocation(
-            url="https://figshare.com/ndownloader/files/43169011",
-            filename="2018.zip",
-            md5=True,
-            checksum="25d091044a10c78674fedad655023e3b",
-        ),
-    ]
-
-    index2label: dict[int, str] = {
-        0: "MILCO",
-        1: "NOMBO",
-    }
-
-    def __init__(
-        self,
-        root: str | Path,
-        image_set: Literal["train", "operational", "base"] = "train",
-        transforms: Transform[NDArray[Any]] | Sequence[Transform[NDArray[Any]]] | None = None,
-        download: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        super().__init__(
-            root,
-            image_set,
-            transforms,
-            download,
-            verbose,
-        )
-        self._bboxes_per_size = True
-
-    def _load_data(self) -> tuple[list[str], list[str], dict[str, list[Any]]]:
-        filepaths: list[str] = []
-        targets: list[str] = []
-        datum_metadata: dict[str, list[Any]] = {}
-        metadata_list: list[dict[str, Any]] = []
-        image_sets: dict[str, list[int]] = {
-            "base": list(range(len(self._resources))),
-            "train": list(range(3)),
-            "operational": list(range(3, len(self._resources))),
-        }
-
-        # Load the data
-        resource_indices = image_sets[self.image_set]
-        for idx in resource_indices:
-            self._resource = self._resources[idx]
-            filepath, target, metadata = super()._load_data()
-            filepaths.extend(filepath)
-            targets.extend(target)
-            metadata_list.append(metadata)
-
-        # Adjust datum metadata to correct format
-        for data_dict in metadata_list:
-            for key, val in data_dict.items():
-                if key not in datum_metadata:
-                    datum_metadata[str(key)] = []
-                datum_metadata[str(key)].extend(val)
-
-        return filepaths, targets, datum_metadata
-
-    def _load_data_inner(self) -> tuple[list[str], list[str], dict[str, Any]]:
-        file_data = {"year": [], "image_id": [], "data_path": [], "label_path": []}
-        data_folder = sorted((self.path / self._resource.filename[:-4]).glob("*.jpg"))
-        if not data_folder:
-            raise FileNotFoundError
-
-        for entry in data_folder:
-            # Remove file extension and split by "_"
-            parts = entry.stem.split("_")
-            file_data["image_id"].append(parts[0])
-            file_data["year"].append(parts[1])
-            file_data["data_path"].append(str(entry))
-            file_data["label_path"].append(str(entry.parent / entry.stem) + ".txt")
-        data = file_data.pop("data_path")
-        annotations = file_data.pop("label_path")
-
-        return data, annotations, file_data
-
-    def _read_annotations(self, annotation: str) -> tuple[list[list[float]], list[int], dict[str, Any]]:
-        """Function for extracting the info out of the text files"""
-        labels: list[int] = []
-        boxes: list[list[float]] = []
-        with open(annotation) as f:
-            for line in f.readlines():
-                out = line.strip().split()
-                labels.append(int(out[0]))
-
-                xcenter, ycenter, width, height = [float(out[1]), float(out[2]), float(out[3]), float(out[4])]
-
-                x0 = xcenter - width / 2
-                x1 = x0 + width
-                y0 = ycenter - height / 2
-                y1 = y0 + height
-                boxes.append([x0, y0, x1, y1])
-
-        return boxes, labels, {}

dataeval/utils/datasets/_mixin.py

@@ -1,54 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-from typing import Any, Generic, TypeVar
-
-import numpy as np
-import torch
-from numpy.typing import NDArray
-from PIL import Image
-
-_TArray = TypeVar("_TArray")
-
-
-class BaseDatasetMixin(Generic[_TArray]):
-    index2label: dict[int, str]
-
-    def _as_array(self, raw: list[Any]) -> _TArray: ...
-    def _one_hot_encode(self, value: int | list[int]) -> _TArray: ...
-    def _read_file(self, path: str) -> _TArray: ...
-
-
-class BaseDatasetNumpyMixin(BaseDatasetMixin[NDArray[Any]]):
-    def _as_array(self, raw: list[Any]) -> NDArray[Any]:
-        return np.asarray(raw)
-
-    def _one_hot_encode(self, value: int | list[int]) -> NDArray[Any]:
-        if isinstance(value, int):
-            encoded = np.zeros(len(self.index2label))
-            encoded[value] = 1
-        else:
-            encoded = np.zeros((len(value), len(self.index2label)))
-            encoded[np.arange(len(value)), value] = 1
-        return encoded
-
-    def _read_file(self, path: str) -> NDArray[Any]:
-        return np.array(Image.open(path)).transpose(2, 0, 1)
-
-
-class BaseDatasetTorchMixin(BaseDatasetMixin[torch.Tensor]):
-    def _as_array(self, raw: list[Any]) -> torch.Tensor:
-        return torch.as_tensor(raw)
-
-    def _one_hot_encode(self, value: int | list[int]) -> torch.Tensor:
-        if isinstance(value, int):
-            encoded = torch.zeros(len(self.index2label))
-            encoded[value] = 1
-        else:
-            encoded = torch.zeros((len(value), len(self.index2label)))
-            encoded[torch.arange(len(value)), value] = 1
-        return encoded
-
-    def _read_file(self, path: str) -> torch.Tensor:
-        return torch.as_tensor(np.array(Image.open(path)).transpose(2, 0, 1))