dataeval 0.73.1__py3-none-any.whl → 0.74.1__py3-none-any.whl

dataeval/utils/torch/trainer.py

@@ -1,15 +1,15 @@
 from __future__ import annotations
 
-__all__ = ["AETrainer"]
-
-from typing import Any
+from typing import Any, Callable
 
 import torch
 import torch.nn as nn
+from numpy.typing import NDArray
 from torch.optim import Adam
-from torch.utils.data import DataLoader, Dataset
+from torch.utils.data import DataLoader, Dataset, TensorDataset
+from tqdm import tqdm
 
-torch.manual_seed(0)
+__all__ = ["AETrainer", "trainer"]
 
 
 def get_images_from_batch(batch: Any) -> Any:
@@ -176,3 +176,82 @@ class AETrainer:
             encodings = torch.vstack((encodings, embeddings)) if len(encodings) else embeddings
 
         return encodings
+
+
+def trainer(
+    model: torch.nn.Module,
+    x_train: NDArray[Any],
+    y_train: NDArray[Any] | None,
+    loss_fn: Callable[..., torch.Tensor | torch.nn.Module] | None,
+    optimizer: torch.optim.Optimizer | None,
+    preprocess_fn: Callable[[torch.Tensor], torch.Tensor] | None,
+    epochs: int,
+    batch_size: int,
+    device: torch.device,
+    verbose: bool,
+) -> None:
+    """
+    Train Pytorch model.
+
+    Parameters
+    ----------
+    model
+        Model to train.
+    loss_fn
+        Loss function used for training.
+    x_train
+        Training data.
+    y_train
+        Training labels.
+    optimizer
+        Optimizer used for training.
+    preprocess_fn
+        Preprocessing function applied to each training batch.
+    epochs
+        Number of training epochs.
+    reg_loss_fn
+        Allows an additional regularisation term to be defined as reg_loss_fn(model)
+    batch_size
+        Batch size used for training.
+    buffer_size
+        Maximum number of elements that will be buffered when prefetching.
+    verbose
+        Whether to print training progress.
+    """
+    if optimizer is None:
+        optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
+
+    if y_train is None:
+        dataset = TensorDataset(torch.from_numpy(x_train).to(torch.float32))
+
+    else:
+        dataset = TensorDataset(
+            torch.from_numpy(x_train).to(torch.float32), torch.from_numpy(y_train).to(torch.float32)
+        )
+
+    loader = DataLoader(dataset=dataset)
+
+    model = model.to(device)
+
+    # iterate over epochs
+    loss = torch.nan
+    disable_tqdm = not verbose
+    for epoch in (pbar := tqdm(range(epochs), disable=disable_tqdm)):
+        epoch_loss = loss
+        for step, data in enumerate(loader):
+            if step % 250 == 0:
+                pbar.set_description(f"Epoch: {epoch} ({epoch_loss:.3f}), loss: {loss:.3f}")
+
+            x, y = [d.to(device) for d in data] if len(data) > 1 else (data[0].to(device), None)
+
+            if isinstance(preprocess_fn, Callable):
+                x = preprocess_fn(x)
+
+            y_hat = model(x)
+            y = x if y is None else y
+
+            loss = loss_fn(y, y_hat)  # type: ignore
+
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()

dataeval/utils/torch/utils.py

@@ -3,8 +3,12 @@ from __future__ import annotations
 __all__ = ["read_dataset"]
 
 from collections import defaultdict
-from typing import Any
+from functools import partial
+from typing import Any, Callable
 
+import numpy as np
+import torch
+from numpy.typing import NDArray
 from torch.utils.data import Dataset
 
 
@@ -61,3 +65,105 @@ def read_dataset(dataset: Dataset[Any]) -> list[list[Any]]:
             ddict[i].append(d)
 
     return list(ddict.values())
+
+
+def get_device(device: str | torch.device | None = None) -> torch.device:
+    """
+    Instantiates a PyTorch device object.
+
+    Parameters
+    ----------
+    device : str | torch.device | None, default None
+        Either ``None``, a str ('gpu' or 'cpu') indicating the device to choose, or an
+        already instantiated device object. If ``None``, the GPU is selected if it is
+        detected, otherwise the CPU is used as a fallback.
+
+    Returns
+    -------
+    The instantiated device object.
+    """
+    if isinstance(device, torch.device):  # Already a torch device
+        return device
+    else:  # Instantiate device
+        if device is None or device.lower() in ["gpu", "cuda"]:
+            torch_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        else:
+            torch_device = torch.device("cpu")
+    return torch_device
+
+
+def predict_batch(
+    x: NDArray[Any] | torch.Tensor,
+    model: Callable | torch.nn.Module | torch.nn.Sequential,
+    device: torch.device | None = None,
+    batch_size: int = int(1e10),
+    preprocess_fn: Callable | None = None,
+    dtype: type[np.generic] | torch.dtype = np.float32,
+) -> NDArray[Any] | torch.Tensor | tuple[Any, ...]:
+    """
+    Make batch predictions on a model.
+
+    Parameters
+    ----------
+    x : np.ndarray | torch.Tensor
+        Batch of instances.
+    model : Callable | nn.Module | nn.Sequential
+        PyTorch model.
+    device : torch.device | None, default None
+        Device type used. The default None tries to use the GPU and falls back on CPU.
+        Can be specified by passing either torch.device('cuda') or torch.device('cpu').
+    batch_size : int, default 1e10
+        Batch size used during prediction.
+    preprocess_fn : Callable | None, default None
+        Optional preprocessing function for each batch.
+    dtype : np.dtype | torch.dtype, default np.float32
+        Model output type, either a :term:`NumPy` or torch dtype, e.g. np.float32 or torch.float32.
+
+    Returns
+    -------
+    NDArray | torch.Tensor | tuple
+        Numpy array, torch tensor or tuples of those with model outputs.
+    """
+    device = get_device(device)
+    if isinstance(x, np.ndarray):
+        x = torch.from_numpy(x).to(device)
+    n = len(x)
+    n_minibatch = int(np.ceil(n / batch_size))
+    return_np = not isinstance(dtype, torch.dtype)
+    preds = []
+    with torch.no_grad():
+        for i in range(n_minibatch):
+            istart, istop = i * batch_size, min((i + 1) * batch_size, n)
+            x_batch = x[istart:istop]
+            if isinstance(preprocess_fn, Callable):
+                x_batch = preprocess_fn(x_batch)
+
+            preds_tmp = model(x_batch.to(torch.float32).to(device))
+            if isinstance(preds_tmp, (list, tuple)):
+                if len(preds) == 0:  # init tuple with lists to store predictions
+                    preds = tuple([] for _ in range(len(preds_tmp)))
+                for j, p in enumerate(preds_tmp):
+                    if isinstance(p, torch.Tensor):
+                        p = p.cpu()
+                    preds[j].append(p if not return_np or isinstance(p, np.ndarray) else p.numpy())
+            elif isinstance(preds_tmp, (np.ndarray, torch.Tensor)):
+                if isinstance(preds_tmp, torch.Tensor):
+                    preds_tmp = preds_tmp.cpu()
+                if isinstance(preds, tuple):
+                    preds = list(preds)
+                preds.append(
+                    preds_tmp
+                    if not return_np or isinstance(preds_tmp, np.ndarray)  # type: ignore
+                    else preds_tmp.numpy()
+                )
+            else:
+                raise TypeError(
+                    f"Model output type {type(preds_tmp)} not supported. The model \
+                    output type needs to be one of list, tuple, NDArray or \
+                    torch.Tensor."
+                )
+    concat = partial(np.concatenate, axis=0) if return_np else partial(torch.cat, dim=0)
+    out: tuple | np.ndarray | torch.Tensor = (
+        tuple(concat(p) for p in preds) if isinstance(preds, tuple) else concat(preds)  # type: ignore
+    )
+    return out

dataeval/workflows/sufficiency.py

@@ -16,11 +16,11 @@ from scipy.optimize import basinhopping
 from torch.utils.data import Dataset
 
 from dataeval.interop import as_numpy
-from dataeval.output import OutputMetadata, set_metadata
+from dataeval.output import Output, set_metadata
 
 
 @dataclass(frozen=True)
-class SufficiencyOutput(OutputMetadata):
+class SufficiencyOutput(Output):
     """
     Output class for :class:`Sufficiency` workflow
 
@@ -47,7 +47,7 @@ class SufficiencyOutput(OutputMetadata):
             if c != c_v:
                 raise ValueError(f"{m} does not contain the expected number ({c}) of data points.")
 
-    @set_metadata()
+    @set_metadata
     def project(
         self,
         projection: int | Iterable[int],
@@ -484,7 +484,7 @@ class Sufficiency(Generic[T]):
     def eval_kwargs(self, value: Mapping[str, Any] | None) -> None:
         self._eval_kwargs = {} if value is None else value
 
-    @set_metadata(["runs", "substeps"])
+    @set_metadata(state=["runs", "substeps"])
     def evaluate(self, eval_at: int | Iterable[int] | None = None, niter: int = 1000) -> SufficiencyOutput:
         """
         Creates data indices, trains models, and returns plotting data

{dataeval-0.73.1.dist-info → dataeval-0.74.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataeval
-Version: 0.73.1
+Version: 0.74.1
 Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
 Home-page: https://dataeval.ai/
 License: MIT
@@ -21,18 +21,12 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Scientific/Engineering
 Provides-Extra: all
-Provides-Extra: tensorflow
 Provides-Extra: torch
-Requires-Dist: hdbscan (>=0.8.36)
-Requires-Dist: markupsafe (<3.0.2) ; extra == "tensorflow" or extra == "all"
-Requires-Dist: matplotlib ; extra == "torch" or extra == "all"
-Requires-Dist: numpy (>1.24.3)
+Requires-Dist: matplotlib ; extra == "all"
+Requires-Dist: numpy (>=1.24.3)
 Requires-Dist: pillow (>=10.3.0)
 Requires-Dist: scikit-learn (>=1.5.0)
 Requires-Dist: scipy (>=1.10)
-Requires-Dist: tensorflow (>=2.16,<2.18) ; extra == "tensorflow" or extra == "all"
-Requires-Dist: tensorflow_probability (>=0.24,<0.25) ; extra == "tensorflow" or extra == "all"
-Requires-Dist: tf-keras (>=2.16,<2.18) ; extra == "tensorflow" or extra == "all"
 Requires-Dist: torch (>=2.2.0) ; extra == "torch" or extra == "all"
 Requires-Dist: torchvision (>=0.17.0) ; extra == "torch" or extra == "all"
 Requires-Dist: tqdm

dataeval-0.74.1.dist-info/RECORD

@@ -0,0 +1,65 @@
+dataeval/__init__.py,sha256=HNOjwnFIQCD7vwBBo0xMexlnNG3xRZ3s3VUMsA4Qozw,392
+dataeval/detectors/__init__.py,sha256=Y-0bbyWyuMvZU80bCx6WPt3IV_r2hu9ymzpA8uzMqoI,206
+dataeval/detectors/drift/__init__.py,sha256=BSXm21y7cAawHep-ZldCJ5HOvzYjPzYGKGrmoEs3i0E,737
+dataeval/detectors/drift/base.py,sha256=QDGHMu1WADD-38MEIOwjQMEQM3DE7B0yFHO3hsMbV-E,14481
+dataeval/detectors/drift/cvm.py,sha256=kc59w2_wtxFGNnLcaJRvX5v_38gPXiebSGNiFVdunEQ,4142
+dataeval/detectors/drift/ks.py,sha256=gcpe1WIQeNeZdLYkdMZCFLXUp1bHMQUxwJE6-RLVOXs,4229
+dataeval/detectors/drift/mmd.py,sha256=C0FX5v9ZJzmKNYEcYUaC7sDtMpJ2dZpwikNDu-AEWiI,7584
+dataeval/detectors/drift/torch.py,sha256=igEQ2DV9JmcpTdUKCOHBi5LxtoNeCAslJS2Ldulg1hw,7585
+dataeval/detectors/drift/uncertainty.py,sha256=Xz2yzJjtJfw1vLag234jwRvaa_HK36nMajGx8bQaNRs,5322
+dataeval/detectors/drift/updates.py,sha256=UJ0z5hlunRi7twnkLABfdJG3tT2EqX4y9IGx8_USYvo,1780
+dataeval/detectors/linters/__init__.py,sha256=BvpaB1RUpkEhhXk3Mqi5NYoOcJKZRFSBOJCmQOIfYRU,483
+dataeval/detectors/linters/clusterer.py,sha256=hK-ak02GaxwWuufesZMKDsvoE5fMdXO7UWsLiK8hfY0,21008
+dataeval/detectors/linters/duplicates.py,sha256=2bmPTFqoefeiAQV9y4CGlHV_mJNrysJSEFLXLd2DO4I,5661
+dataeval/detectors/linters/merged_stats.py,sha256=X-bDTwjyR8RuVmzxLaHZmQ5nI3oOWvsqVlitdSncapk,1355
+dataeval/detectors/linters/outliers.py,sha256=X48bzTfTr1LqC6WKVKBRfvpjcQRgmb93cNLT7Oipe3M,10113
+dataeval/detectors/ood/__init__.py,sha256=-D4Fq-ysFylNNMqjHG1ALbB9qBCm_UinkCAgsK9HGg0,408
+dataeval/detectors/ood/ae_torch.py,sha256=pO9w5221bXR9lEBkE7oakXeE7PXUUR--xcTpmHvOCSk,2142
+dataeval/detectors/ood/base.py,sha256=UzcDbXl8Gv43VFzjrOegTnKSIoEYmfDP7fAySeWyWPw,6955
+dataeval/detectors/ood/base_torch.py,sha256=yFbSfQsBMwZeVf8mrixmkZYBGChhV5oAHtkgzWnMzsA,3405
+dataeval/detectors/ood/metadata_ks_compare.py,sha256=LNDNWGEDKTW8_-djgmK53sn9EZzzXq1Sgwc47k0QI-Y,5380
+dataeval/detectors/ood/metadata_least_likely.py,sha256=nxMCXUOjOfWHDTGT2SLE7OYBCydRq8zHLd8t17k7hMM,5193
+dataeval/detectors/ood/metadata_ood_mi.py,sha256=KLay2BmgHrStBV92VpIs_B1yEfQKllsMTgzOQEng01I,4065
+dataeval/interop.py,sha256=SB5Nca12rluZeXrpmmlfY7LFJbN5opYM7jmAb2c29hM,1748
+dataeval/metrics/__init__.py,sha256=fPBNLd-T6mCErZBBJrxWmXIL0jCk7fNUYIcNEBkMa80,238
+dataeval/metrics/bias/__init__.py,sha256=dYiPHenS8J7pgRMMW2jNkTBmTbPoYTxT04fZu9PFats,747
+dataeval/metrics/bias/balance.py,sha256=_TZEe17AT-qOvPp-QFrQfTqNwh8uVVCYjC4Sv6JBx9o,9118
+dataeval/metrics/bias/coverage.py,sha256=o65_IgrWSlGnYeYZFABjwKaxq09uqyy5esHJM67PJ-k,4528
+dataeval/metrics/bias/diversity.py,sha256=WL1NbZiRrv0SIq97FY3womZNCSl_EBMVlBWQZAUtjk8,7701
+dataeval/metrics/bias/metadata_preprocessing.py,sha256=ekUFiirkmaHDiH7nJjkNpiUQD7OolAPhHorjLxpXv_Y,12248
+dataeval/metrics/bias/metadata_utils.py,sha256=HmTjlRRTdM9566oKUDDdVMJ8luss4DYykFOiS2FQzhM,6558
+dataeval/metrics/bias/parity.py,sha256=hnA7qQH4Uy3tl_krluZ9BPD5zYjjagUxZt2fEiIa2yE,12745
+dataeval/metrics/estimators/__init__.py,sha256=O6ocxJq8XDkfJWwXeJnnnzbOyRnFPKF4kTIVTTZYOA8,380
+dataeval/metrics/estimators/ber.py,sha256=fs3_e9pgu7I50QIALWtF2aidkBZhTCKVE2pA7PyB5Go,5019
+dataeval/metrics/estimators/divergence.py,sha256=r_SKSurf1TdI5E1ivENqDnz8cQ3_sxVGKAqmF9cqcT4,4275
+dataeval/metrics/estimators/uap.py,sha256=Aw5ReoWNK73Tq96r__qN_-cvHrELauqtDX3Af_QxX4s,2157
+dataeval/metrics/stats/__init__.py,sha256=igLRaAt1nX6yRwC4xI0zNPBADi3u7EsSxWP3OZ8AqcU,1086
+dataeval/metrics/stats/base.py,sha256=_C05KUAuDrfX3N-19o25V3vmXr0-45A5fc57cXyV8qs,12161
+dataeval/metrics/stats/boxratiostats.py,sha256=bZunY-b8Y2IQqHlTusQN77ujLOHftogEQIARDpdVv6A,6463
+dataeval/metrics/stats/datasetstats.py,sha256=rZUDiciHwEpnXmkI8-uJNiYwUuTL9ssZMKMx73hVX-Y,6219
+dataeval/metrics/stats/dimensionstats.py,sha256=xITgQF_oomb6Ty_dJcbT3ARGGNp4QRcYSgnkjB4f-YE,4054
+dataeval/metrics/stats/hashstats.py,sha256=vxw_K74EJM9CZy-EV617vdrysFO8nEspVWqIYsIHC-c,4958
+dataeval/metrics/stats/labelstats.py,sha256=K0hJTphMe7htSjyss8GPtKDiHepTuU60_hX0xRA-uAg,4096
+dataeval/metrics/stats/pixelstats.py,sha256=2zr9i3GLNx1i_SCtbfdtZNxXBEc_9wCe4qDpmXLVbKY,4576
+dataeval/metrics/stats/visualstats.py,sha256=vLIC4sMo796axWl-4e4RzT33ll-_6ki54Dirn3V-EL8,4948
+dataeval/output.py,sha256=SmzH9W9yewdL9SBKVBkUUvOo45oA5lHphE2DYvJJMu0,3573
+dataeval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dataeval/utils/__init__.py,sha256=z7HxSijjycey-rGdQkgVOdpvT0oO2pKAuT4uYyxYGMs,555
+dataeval/utils/gmm.py,sha256=YuLsJKsVWgH_wHr1u_hSRH5Yeexdj8exht8h99L7bLo,561
+dataeval/utils/image.py,sha256=KgC_1nW__nGN5q6bVZNvG4U_qIBdjcPATz9qe8f2XuA,1928
+dataeval/utils/metadata.py,sha256=0A--iru0zEmi044mKz5P35q69KrI30yoiRSlvs7TSdQ,9418
+dataeval/utils/shared.py,sha256=xvF3VLfyheVwJtdtDrneOobkKf7t-JTmf_w91FWXmqo,3616
+dataeval/utils/split_dataset.py,sha256=Ot1ZJhbIhVfcShYXF9MkWXak5odBXyuBdRh-noXh-MI,19555
+dataeval/utils/torch/__init__.py,sha256=lpkqfgyARUxgrV94cZESQv8PIP2p-UnwItZ_wIr0XzQ,675
+dataeval/utils/torch/blocks.py,sha256=HVhBTMMD5NA4qheMUgyol1KWiKZDIuc8k5j4RcMKmhk,1466
+dataeval/utils/torch/datasets.py,sha256=10elNgLuH_FDX_CHE3y2Z215JN4-PQovQm5brcIJOeM,15021
+dataeval/utils/torch/gmm.py,sha256=VbLlUQohwToApT493_tjQBWy2UM5R-3ppS9Dp-eP7BA,3240
+dataeval/utils/torch/models.py,sha256=sdGeo7a8vshCTGA4lYyVxxb_aDWUlxdtIVxrddS-_ls,8542
+dataeval/utils/torch/trainer.py,sha256=8BEXr6xtk-CHJTcNxOBnWgkFWfJUAiBy28cEdBhLMRU,7883
+dataeval/utils/torch/utils.py,sha256=nWRcT6z6DbFVrL1RyxCOX3DPoCrv9G0B-VI_9LdGCQQ,5784
+dataeval/workflows/__init__.py,sha256=ef1MiVL5IuhlDXXbwsiAfafhnr7tD3TXF9GRusy9_O8,290
+dataeval/workflows/sufficiency.py,sha256=v9AV3BZT0NW-zD2VNIL_5aWspvoscrxRIUKcUdpy7HI,18540
+dataeval-0.74.1.dist-info/LICENSE.txt,sha256=Kpzcfobf1HlqafF-EX6dQLw9TlJiaJzfgvLQFukyXYw,1060
+dataeval-0.74.1.dist-info/METADATA,sha256=nd7os3kaLfp-A5HWH0QYVxe-gQdj5q3dIn9d0fPf-Lk,4298
+dataeval-0.74.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+dataeval-0.74.1.dist-info/RECORD,,

dataeval/detectors/ood/aegmm.py

@@ -1,66 +0,0 @@
-"""
-Source code derived from Alibi-Detect 0.11.4
-https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
-
-Original code Copyright (c) 2023 Seldon Technologies Ltd
-Licensed under Apache Software License (Apache 2.0)
-"""
-
-from __future__ import annotations
-
-__all__ = ["OOD_AEGMM"]
-
-from typing import TYPE_CHECKING, Callable
-
-from numpy.typing import ArrayLike
-
-from dataeval.detectors.ood.base import OODGMMBase, OODScoreOutput
-from dataeval.interop import to_numpy
-from dataeval.utils.lazy import lazyload
-from dataeval.utils.tensorflow._internal.gmm import gmm_energy
-from dataeval.utils.tensorflow._internal.loss import LossGMM
-from dataeval.utils.tensorflow._internal.utils import predict_batch
-
-if TYPE_CHECKING:
-    import tensorflow as tf
-    import tf_keras as keras
-
-    import dataeval.utils.tensorflow._internal.models as tf_models
-else:
-    tf = lazyload("tensorflow")
-    keras = lazyload("tf_keras")
-    tf_models = lazyload("dataeval.utils.tensorflow._internal.models")
-
-
-class OOD_AEGMM(OODGMMBase):
-    """
-    AE with Gaussian Mixture Model based outlier detector.
-
-    Parameters
-    ----------
-    model : AEGMM
-       An AEGMM model.
-    """
-
-    def __init__(self, model: tf_models.AEGMM) -> None:
-        super().__init__(model)
-
-    def fit(
-        self,
-        x_ref: ArrayLike,
-        threshold_perc: float = 100.0,
-        loss_fn: Callable[..., tf.Tensor] | None = None,
-        optimizer: keras.optimizers.Optimizer | None = None,
-        epochs: int = 20,
-        batch_size: int = 64,
-        verbose: bool = True,
-    ) -> None:
-        if loss_fn is None:
-            loss_fn = LossGMM()
-        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
-
-    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
-        self._validate(X := to_numpy(X))
-        _, z, _ = predict_batch(X, self.model, batch_size=batch_size)
-        energy, _ = gmm_energy(z, self.gmm_params, return_mean=False)
-        return OODScoreOutput(energy.numpy())  # type: ignore