dataeval 0.69.2__py3-none-any.whl → 0.69.4__py3-none-any.whl

dataeval/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.69.2"
+__version__ = "0.69.4"
 
 from importlib.util import find_spec
 

dataeval/_internal/datasets.py

@@ -0,0 +1,300 @@
+from __future__ import annotations
+
+import hashlib
+import os
+import zipfile
+from pathlib import Path
+from typing import Literal
+from urllib.error import HTTPError, URLError
+from urllib.request import urlretrieve
+
+import numpy as np
+from numpy.typing import NDArray
+from torch.utils.data import Dataset
+from torchvision.datasets import CIFAR10, VOCDetection  # noqa: F401
+
+
+def _validate_file(fpath, file_md5, chunk_size=65535):
+    hasher = hashlib.md5()
+    with open(fpath, "rb") as fpath_file:
+        while chunk := fpath_file.read(chunk_size):
+            hasher.update(chunk)
+    return hasher.hexdigest() == file_md5
+
+
+def _get_file(
+    root: str | Path,
+    fname: str,
+    origin: str,
+    file_md5: str | None = None,
+):
+    fname = os.fspath(fname) if isinstance(fname, os.PathLike) else fname
+    fpath = os.path.join(root, fname)
+
+    download = False
+    if os.path.exists(fpath):
+        if file_md5 is not None and not _validate_file(fpath, file_md5):
+            download = True
+        else:
+            print("Files already downloaded and verified")
+    else:
+        download = True
+
+    if download:
+        try:
+            error_msg = "URL fetch failure on {}: {} -- {}"
+            try:
+                urlretrieve(origin, fpath)
+            except HTTPError as e:
+                raise Exception(error_msg.format(origin, e.code, e.msg)) from e
+            except URLError as e:
+                raise Exception(error_msg.format(origin, e.errno, e.reason)) from e
+        except (Exception, KeyboardInterrupt):
+            if os.path.exists(fpath):
+                os.remove(fpath)
+            raise
+
+        if os.path.exists(fpath) and file_md5 is not None and not _validate_file(fpath, file_md5):
+            raise ValueError(
+                "Incomplete or corrupted file detected. "
+                f"The md5 file hash does not match the provided value "
+                f"of {file_md5}.",
+            )
+    return fpath
+
+
+def download_dataset(url: str, root: str | Path, fname: str, md5: str) -> str:
+    """Code to download mnist and corruptions, originates from tensorflow_datasets (tfds):
+    https://github.com/tensorflow/datasets/blob/master/tensorflow_datasets/image_classification/mnist_corrupted.py
+    """
+    name, _ = os.path.splitext(fname)
+    folder = os.path.join(root, name)
+    os.makedirs(folder, exist_ok=True)
+
+    path = _get_file(
+        root,
+        fname,
+        origin=url + fname,
+        file_md5=md5,
+    )
+    extract_archive(path, remove_finished=True)
+    return path
+
+
+def extract_archive(
+    from_path: str | Path,
+    to_path: str | Path | None = None,
+    remove_finished: bool = False,
+):
+    """Extract an archive.
+
+    The archive type and a possible compression is automatically detected from the file name.
+    """
+    from_path = Path(from_path)
+    if not from_path.is_absolute():
+        from_path = from_path.resolve()
+
+    if to_path is None:
+        to_path = os.path.dirname(from_path)
+
+    # Extracting zip
+    with zipfile.ZipFile(from_path, "r", compression=zipfile.ZIP_STORED) as zzip:
+        zzip.extractall(to_path)
+
+    if remove_finished:
+        os.remove(from_path)
+
+
+class MNIST(Dataset):
+    """MNIST Dataset and Corruptions.
+
+    Args:
+        root : str | ``pathlib.Path``
+            Root directory of dataset where the ``mnist_c/`` folder exists.
+        train : bool, default True
+            If True, creates dataset from ``train_images.npy`` and ``train_labels.npy``.
+        download : bool, default False
+            If True, downloads the dataset from the internet and puts it in root
+            directory. If dataset is already downloaded, it is not downloaded again.
+        size : int, default -1
+            Limit the dataset size, must be a value greater than 0.
+        unit_interval : bool, default False
+            Shift the data values to the unit interval [0-1].
+        dtype : type | None, default None
+            Change the numpy dtype - data is loaded as np.uint8
+        channels : Literal['channels_first' | 'channels_last'] | None, default None
+            Location of channel axis if desired, default has no channels (N, 28, 28)
+        flatten : bool, default False
+            Flatten data into single dimension (N, 784) - cannot use both channels and flatten,
+            channels takes priority over flatten.
+        normalize : tuple[mean, std] | None, default None
+            Normalize images acorrding to provided mean and standard deviation
+        corruption : Literal['identity' | 'shot_noise' | 'impulse_noise' | 'glass_blur' |
+            'motion_blur' | 'shear' | 'scale' | 'rotate' | 'brightness' | 'translate' | 'stripe' |
+            'fog' | 'spatter' | 'dotted_line' | 'zigzag' | 'canny_edges'] | None, default None
+            The desired corruption style or None.
+    """
+
+    mirror = "https://zenodo.org/record/3239543/files/"
+
+    resources = ("mnist_c.zip", "4b34b33045869ee6d424616cd3a65da3")
+
+    classes = [
+        "0 - zero",
+        "1 - one",
+        "2 - two",
+        "3 - three",
+        "4 - four",
+        "5 - five",
+        "6 - six",
+        "7 - seven",
+        "8 - eight",
+        "9 - nine",
+    ]
+
+    @property
+    def train_labels(self):
+        return self.targets
+
+    @property
+    def test_labels(self):
+        return self.targets
+
+    @property
+    def train_data(self):
+        return self.data
+
+    @property
+    def test_data(self):
+        return self.data
+
+    def __init__(
+        self,
+        root: str | Path,
+        train: bool = True,
+        download: bool = False,
+        size: int = -1,
+        unit_interval: bool = False,
+        dtype: type | None = None,
+        channels: Literal["channels_first", "channels_last"] | None = None,
+        flatten: bool = False,
+        normalize: tuple[float, float] | None = None,
+        corruption: Literal[
+            "identity",
+            "shot_noise",
+            "impulse_noise",
+            "glass_blur",
+            "motion_blur",
+            "shear",
+            "scale",
+            "rotate",
+            "brightness",
+            "translate",
+            "stripe",
+            "fog",
+            "spatter",
+            "dotted_line",
+            "zigzag",
+            "canny_edges",
+        ]
+        | None = None,
+    ) -> None:
+        if isinstance(root, str):
+            root = os.path.expanduser(root)
+        self.root = root  # location of stored dataset
+        self.train = train  # training set or test set
+        self.size = size
+        self.unit_interval = unit_interval
+        self.dtype = dtype
+        self.channels = channels
+        self.flatten = flatten
+        self.normalize = normalize
+
+        if corruption is None:
+            corruption = "identity"
+        elif corruption == "identity":
+            print("Identity is not a corrupted dataset but the original MNIST dataset")
+        self.corruption = corruption
+
+        if os.path.exists(self.mnist_folder):
+            print("Files already downloaded and verified")
+        elif download:
+            download_dataset(self.mirror, self.root, self.resources[0], self.resources[1])
+        else:
+            raise RuntimeError("Dataset not found. You can use download=True to download it")
+
+        self.data, self.targets = self._load_data()
+
+    def _load_data(self):
+        image_file = f"{'train' if self.train else 'test'}_images.npy"
+        data = self._read_image_file(os.path.join(self.mnist_folder, image_file))
+
+        label_file = f"{'train' if self.train else 'test'}_labels.npy"
+        targets = self._read_label_file(os.path.join(self.mnist_folder, label_file))
+
+        if self.size >= 1 and self.size >= len(self.classes):
+            final_data = []
+            final_targets = []
+            for label in range(len(self.classes)):
+                indices = np.where(targets == label)[0]
+                selected_indices = indices[: int(self.size / len(self.classes))]
+                final_data.append(data[selected_indices])
+                final_targets.append(targets[selected_indices])
+            data = np.concatenate(final_data)
+            targets = np.concatenate(final_targets)
+            shuffled_indices = np.random.permutation(data.shape[0])
+            data = data[shuffled_indices]
+            targets = targets[shuffled_indices]
+        elif self.size >= 1:
+            data = data[: self.size]
+            targets = targets[: self.size]
+
+        if self.unit_interval:
+            data = data / 255
+
+        if self.normalize:
+            data = (data - self.normalize[0]) / self.normalize[1]
+
+        if self.dtype:
+            data = data.astype(self.dtype)
+
+        if self.channels == "channels_first":
+            data = np.moveaxis(data, -1, 1)
+        elif self.channels is None:
+            data = data[:, :, :, 0]
+
+        if self.flatten and self.channels is None:
+            data = data.reshape(data.shape[0], -1)
+
+        return data, targets
+
+    def __getitem__(self, index: int) -> tuple[NDArray, int]:
+        """
+        Args:
+            index (int): Index
+
+        Returns:
+            tuple: (image, target) where target is index of the target class.
+        """
+        img, target = self.data[index], int(self.targets[index])
+
+        return img, target
+
+    def __len__(self) -> int:
+        return len(self.data)
+
+    @property
+    def mnist_folder(self) -> str:
+        return os.path.join(self.root, "mnist_c", self.corruption)
+
+    @property
+    def class_to_idx(self) -> dict[str, int]:
+        return {_class: i for i, _class in enumerate(self.classes)}
+
+    def _read_label_file(self, path: str) -> NDArray:
+        x = np.load(path, allow_pickle=False)
+        return x
+
+    def _read_image_file(self, path: str) -> NDArray:
+        x = np.load(path, allow_pickle=False)
+        return x

dataeval/_internal/metrics/diversity.py

@@ -130,7 +130,10 @@ def diversity_simpson(
         p_i = cnts / cnts.sum()
         # inverse Simpson index normalized by (number of bins)
         s_0 = 1 / np.sum(p_i**2) / num_bins[col]
-        ev_index[col] = (s_0 * num_bins[col] - 1) / (num_bins[col] - 1)
+        if num_bins[col] == 1:
+            ev_index[col] = 0
+        else:
+            ev_index[col] = (s_0 * num_bins[col] - 1) / (num_bins[col] - 1)
     return ev_index
 
 

dataeval/_internal/metrics/parity.py

@@ -348,6 +348,7 @@ def parity(
     chi_scores = np.zeros(len(factors))
     p_values = np.zeros(len(factors))
     n_cls = len(np.unique(labels))
+    not_enough_data = {}
     for i, (current_factor_name, factor_values) in enumerate(factors.items()):
         unique_factor_values = np.unique(factor_values)
         contingency_matrix = np.zeros((len(unique_factor_values), n_cls))
@@ -361,13 +362,12 @@ def parity(
                 with_both = np.bitwise_and((labels == label), factor_values == factor_value)
                 contingency_matrix[fi, label] = np.sum(with_both)
                 if 0 < contingency_matrix[fi, label] < 5:
-                    warnings.warn(
-                        f"Factor {current_factor_name} value {factor_value} co-occurs "
-                        f"only {contingency_matrix[fi, label]} times with label {label}. "
-                        "This can cause inaccurate chi_square calculation. Recommend"
-                        "ensuring each label occurs either 0 times or at least 5 times. "
-                        "Alternatively, digitize any continuous-valued factors "
-                        "into fewer bins."
+                    if current_factor_name not in not_enough_data:
+                        not_enough_data[current_factor_name] = {}
+                    if factor_value not in not_enough_data[current_factor_name]:
+                        not_enough_data[current_factor_name][factor_value] = []
+                    not_enough_data[current_factor_name][factor_value].append(
+                        (label, int(contingency_matrix[fi, label]))
                     )
 
         # This deletes rows containing only zeros,
@@ -381,4 +381,23 @@ def parity(
         chi_scores[i] = chi2
         p_values[i] = p
 
+    if not_enough_data:
+        factor_msg = []
+        for factor, fact_dict in not_enough_data.items():
+            stacked_msg = []
+            for key, value in fact_dict.items():
+                msg = []
+                for item in value:
+                    msg.append(f"label {item[0]}: {item[1]} occurrences")
+                flat_msg = "\n\t\t".join(msg)
+                stacked_msg.append(f"value {key} - {flat_msg}\n\t")
+            factor_msg.append(factor + " - " + "".join(stacked_msg))
+
+        message = "\n".join(factor_msg)
+
+        warnings.warn(
+            f"The following factors did not meet the recommended 5 occurrences for each value-label combination. \nRecommend rerunning parity after adjusting the following factor-value-label combinations: \n{message}",  # noqa: E501
+            UserWarning,
+        )
+
     return ParityOutput(chi_scores, p_values)

{dataeval-0.69.2.dist-info → dataeval-0.69.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataeval
-Version: 0.69.2
+Version: 0.69.4
 Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
 Home-page: https://dataeval.ai/
 License: MIT
@@ -33,6 +33,7 @@ Requires-Dist: tensorflow (>=2.14.1,<2.16) ; extra == "tensorflow" or extra == "
 Requires-Dist: tensorflow-io-gcs-filesystem (>=0.35.0,<0.37) ; extra == "tensorflow" or extra == "all"
 Requires-Dist: tensorflow_probability (>=0.22.1,<0.24) ; extra == "tensorflow" or extra == "all"
 Requires-Dist: torch (>=2.0.1,!=2.2.0) ; extra == "torch" or extra == "all"
+Requires-Dist: torchvision (>=0.16.0) ; extra == "torch" or extra == "all"
 Requires-Dist: xxhash (>=3.3)
 Project-URL: Documentation, https://dataeval.readthedocs.io/
 Project-URL: Repository, https://github.com/aria-ml/dataeval/

{dataeval-0.69.2.dist-info → dataeval-0.69.4.dist-info}/RECORD

@@ -1,4 +1,5 @@
-dataeval/__init__.py,sha256=NUQixSNyEc0GiI7YgbfY9IL0OEkIN9kdbrOGAB041Ig,590
+dataeval/__init__.py,sha256=KOZnb9SovSSuD2UrqV-NS_b5vpfWdQlsweB55fned58,590
+dataeval/_internal/datasets.py,sha256=MwN6xgZW1cA5yIxXZ05qBBz4aO3bjKzIEbZZfa1HkQo,9790
 dataeval/_internal/detectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataeval/_internal/detectors/clusterer.py,sha256=hJwELUeAdZZ3OVLIfwalw2P7Zz13q2ZqrV6gx90s44E,20695
 dataeval/_internal/detectors/drift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -25,8 +26,8 @@ dataeval/_internal/metrics/balance.py,sha256=eAHvgjiGCH893XSQLqh9j9wgvAECoNPVT8k
 dataeval/_internal/metrics/ber.py,sha256=Onsi47AbT9rMvng-Pbu8LIrYRfLpI13En1FxkFoMKQs,4668
 dataeval/_internal/metrics/coverage.py,sha256=EZVES1rbZW2j_CtQv1VFfSO-UmWcrt5nmqxDErtrG14,3473
 dataeval/_internal/metrics/divergence.py,sha256=nmMUfr9FGnH798eb6xzEiMj4C42rQVthh5HeexiY6EE,4119
-dataeval/_internal/metrics/diversity.py,sha256=nGjYQ-NLjb8mPt1PAYnvkWH4D58kjM39IPs2FULfis4,7503
-dataeval/_internal/metrics/parity.py,sha256=suv1Pf7gPj0_NxsS0_M6ewfUndsFJyEhbt5NPp6ktMI,15457
+dataeval/_internal/metrics/diversity.py,sha256=_oT0FHsgfLOoe_TLD2Aax4r4jmH6WnOPVIkcl_YjaoY,7582
+dataeval/_internal/metrics/parity.py,sha256=VszQNbHWjct2bCqrIXUZC_qFi4ZIq2Lm-vs-DiarBFo,16244
 dataeval/_internal/metrics/stats.py,sha256=ILKteVMGjrp1s2CECPL_hbLsijIKR2d6II2-8w9oxW8,18105
 dataeval/_internal/metrics/uap.py,sha256=w-wvXXnX16kUq-weaZD2SrJi22LJ8EjOFbOhPxeGejI,2043
 dataeval/_internal/metrics/utils.py,sha256=mSYa-3cHGcsQwPr7zbdpzrnK_8jIXCiAcu2HCcvrtaY,13007
@@ -67,7 +68,7 @@ dataeval/torch/models/__init__.py,sha256=YnDnePYpRIKHyYn3F5qR1OObMSb-g0FGvI8X-uT
 dataeval/torch/trainer/__init__.py,sha256=Te-qElt8h-Zv8NN0r-VJOEdCPHTQ2yO3rd2MhRiZGZs,93
 dataeval/utils/__init__.py,sha256=ExQ1xj62MjcM9uIu1-g1P2fW0EPJpcIofnvxjQ908c4,172
 dataeval/workflows/__init__.py,sha256=gkU2B6yUiefexcYrBwqfZKNl8BvX8abUjfeNvVBXF4E,186
-dataeval-0.69.2.dist-info/LICENSE.txt,sha256=Kpzcfobf1HlqafF-EX6dQLw9TlJiaJzfgvLQFukyXYw,1060
-dataeval-0.69.2.dist-info/METADATA,sha256=_9rVrbIh4EPYStZtOUYnB-Xo3cZ5JMUAf06TqDKvrZs,4217
-dataeval-0.69.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-dataeval-0.69.2.dist-info/RECORD,,
+dataeval-0.69.4.dist-info/LICENSE.txt,sha256=Kpzcfobf1HlqafF-EX6dQLw9TlJiaJzfgvLQFukyXYw,1060
+dataeval-0.69.4.dist-info/METADATA,sha256=R_YlthIsAkOizGWkgXiOCEsD_6F5wJm8qjU4hjhL_c8,4292
+dataeval-0.69.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+dataeval-0.69.4.dist-info/RECORD,,