dataeval 0.84.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

dataeval/utils/torch/_internal.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 __all__ = []
 
-from functools import partial
 from typing import Any, Callable
 
 import numpy as np
@@ -12,16 +11,16 @@ from torch.utils.data import DataLoader, TensorDataset
 from tqdm import tqdm
 
 from dataeval.config import DeviceLike, get_device
+from dataeval.typing import Array
 
 
 def predict_batch(
-    x: NDArray[Any] | torch.Tensor,
-    model: Callable | torch.nn.Module | torch.nn.Sequential,
+    x: Array,
+    model: torch.nn.Module,
     device: DeviceLike | None = None,
     batch_size: int = int(1e10),
     preprocess_fn: Callable[[torch.Tensor], torch.Tensor] | None = None,
-    dtype: type[np.generic] | torch.dtype = np.float32,
-) -> NDArray[Any] | torch.Tensor | tuple[Any, ...]:
+) -> torch.Tensor:
     """
     Make batch predictions on a model.
 
@@ -29,7 +28,7 @@ def predict_batch(
     ----------
     x : np.ndarray | torch.Tensor
         Batch of instances.
-    model : Callable | nn.Module | nn.Sequential
+    model : nn.Module
         PyTorch model.
     device : DeviceLike or None, default None
         The hardware device to use if specified, otherwise uses the DataEval
@@ -38,21 +37,18 @@ def predict_batch(
         Batch size used during prediction.
     preprocess_fn : Callable | None, default None
         Optional preprocessing function for each batch.
-    dtype : np.dtype | torch.dtype, default np.float32
-        Model output type, either a :term:`NumPy` or torch dtype, e.g. np.float32 or torch.float32.
 
     Returns
     -------
-    NDArray | torch.Tensor | tuple
-        Numpy array, torch tensor or tuples of those with model outputs.
+    torch.Tensor
+        PyTorch tensor with model outputs.
     """
     device = get_device(device)
-    if isinstance(x, np.ndarray):
-        x = torch.tensor(x, device=device)
+    if isinstance(model, torch.nn.Module):
+        model = model.to(device).eval()
+    x = torch.tensor(x, device=device)
     n = len(x)
     n_minibatch = int(np.ceil(n / batch_size))
-    return_np = not isinstance(dtype, torch.dtype)
-    preds_tuple = None
     preds_array = []
     with torch.no_grad():
         for i in range(n_minibatch):
@@ -60,28 +56,9 @@ def predict_batch(
             x_batch = x[istart:istop]
             if isinstance(preprocess_fn, Callable):
                 x_batch = preprocess_fn(x_batch)
+            preds_array.append(model(x_batch.to(dtype=torch.float32)).cpu())
 
-            preds_tmp = model(x_batch.to(dtype=torch.float32))
-            if isinstance(preds_tmp, (list, tuple)):
-                if preds_tuple is None:  # init tuple with lists to store predictions
-                    preds_tuple = tuple([] for _ in range(len(preds_tmp)))
-                for j, p in enumerate(preds_tmp):
-                    p = p.cpu() if isinstance(p, torch.Tensor) else p
-                    preds_tuple[j].append(p if not return_np or isinstance(p, np.ndarray) else p.numpy())
-            elif isinstance(preds_tmp, (np.ndarray, torch.Tensor)):
-                preds_tmp = preds_tmp.cpu() if isinstance(preds_tmp, torch.Tensor) else preds_tmp
-                preds_array.append(
-                    preds_tmp if not return_np or isinstance(preds_tmp, np.ndarray) else preds_tmp.numpy()
-                )
-            else:
-                raise TypeError(
-                    f"Model output type {type(preds_tmp)} not supported. The model \
-                    output type needs to be one of list, tuple, NDArray or \
-                    torch.Tensor."
-                )
-    concat = partial(np.concatenate, axis=0) if return_np else partial(torch.cat, dim=0)
-    out = tuple(concat(p) for p in preds_tuple) if preds_tuple is not None else concat(preds_array)
-    return out
+    return torch.cat(preds_array, dim=0)
 
 
 def trainer(

{dataeval-0.84.0.dist-info → dataeval-1.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataeval
-Version: 0.84.0
+Version: 1.0.0
 Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
 Home-page: https://dataeval.ai/
 License: MIT
@@ -82,8 +82,7 @@ using MAITE-compliant datasets and models.
 
 **Python versions:** 3.9 - 3.12
 
-**Supported packages**: *NumPy*, *Pandas*, *Sci-kit learn*, *MAITE*, *NRTK*,
-*Gradient*
+**Supported packages**: *NumPy*, *Pandas*, *Sci-kit learn*, *MAITE*, *NRTK*
 
 Choose your preferred method of installation below or follow our
 [installation guide](https://dataeval.readthedocs.io/en/v0.74.2/installation.html).

dataeval-1.0.0.dist-info/RECORD

@@ -0,0 +1,107 @@
+dataeval/__init__.py,sha256=xd1GfD7QmzBG-WN7K6BMJSzV9_UZlX5OiKICdQ5xGfU,1635
+dataeval/_log.py,sha256=Mn5bRWO0cgtAYd5VGYSFiPgu57ta3zoktrtHAZ1m3dU,357
+dataeval/config.py,sha256=lD1YDH8HosFeRU5rQEYRBcmXMZy-csWaMlJTRZGd9iU,3582
+dataeval/data/__init__.py,sha256=qNnRRiVP_sLthkkHpUrMgI_r8dQK-cC-xoGrrjQeRKc,544
+dataeval/data/_embeddings.py,sha256=6Medqj_JCQt1iwZwWGSs1OeX-bHB8bg5BJqADY1N2s8,12883
+dataeval/data/_images.py,sha256=WF9XJRka8ohUdyI2IKBMAy3JoJhOm1iC-8tbYl8woRM,2642
+dataeval/data/_metadata.py,sha256=hNgsCEN8EyfDDX7zLKcQnsaDl-9xvvs5tUzqMjVLvI4,14457
+dataeval/data/_selection.py,sha256=V61_pTFj0hSzmltA6CV5t51Znqw2dIQZ71Iu46bLm44,4486
+dataeval/data/_split.py,sha256=6Jtm_i__CcPtNE3eSeBdPxc7gn7Cp-GM7g9wJWFlVus,16761
+dataeval/data/_targets.py,sha256=ws5d9wRiDkIuOV7GSAKNxzgSm6AWTgb0BFroQK5nAmM,3057
+dataeval/data/selections/__init__.py,sha256=2m8ZB53wXzqLcqmc6p5atO6graB6ZyiRSNJFxf11X_g,613
+dataeval/data/selections/_classbalance.py,sha256=7v8ApoL3X8eCZ6fGDNTehE_bZ1loaP3TlhsJLaICVWg,1458
+dataeval/data/selections/_classfilter.py,sha256=rEeq959p_SLl_etS7pcM8ZxK4yzEYlYZAQ3FlcLV0R8,4330
+dataeval/data/selections/_indices.py,sha256=RFsR9z10aM3N0gJSfKrukFpi-LkiQGXoOwXhmOQ5cpg,630
+dataeval/data/selections/_limit.py,sha256=JG4GmEiNKt3sk4PbOUbBnGGzNlyz72H-kQrt8COMm4Y,512
+dataeval/data/selections/_prioritize.py,sha256=yw51ZQk6FPvyC38M4_pS_Se2Dq0LDFcdDhfbsELzTZc,11306
+dataeval/data/selections/_reverse.py,sha256=b67kNC43A5KpQOic5gifjo9HpJ7FMh4LFCrfovPiJ-M,368
+dataeval/data/selections/_shuffle.py,sha256=gVz_2T4rlucq8Ytqz5jvmmZdTrZDaIv43jJbq97tLjQ,1173
+dataeval/detectors/__init__.py,sha256=3Sg-XWlwr75zEEH3hZKA4nWMtGvaRlnfzTWvZG_Ak6U,189
+dataeval/detectors/drift/__init__.py,sha256=gD8aY5PotS-S2ot7iB_z_zzSOjIbQLw5znFBNj0jtHE,646
+dataeval/detectors/drift/_base.py,sha256=amGqzUAe8fU5qwM5lq1p8PCuhjGh9MHkdW1zeBF1LEE,7574
+dataeval/detectors/drift/_cvm.py,sha256=cS33zWJmFY1fft1XcANcP2jSD5ou7TxvIU2AldhTynM,3004
+dataeval/detectors/drift/_ks.py,sha256=uMc5-NA-lSV1IODrY8uJe87ll3uRJT_oXLJFXy95M1w,3186
+dataeval/detectors/drift/_mmd.py,sha256=wHUy_vUafCikrZ_WX8qQXpxFwzw07-5zVutloR6hl1k,11589
+dataeval/detectors/drift/_uncertainty.py,sha256=BHlykJ-r7TGLJxdPfoazXnoAJ1qVDzbk5HjAMdsnHz8,5847
+dataeval/detectors/drift/updates.py,sha256=L1PnrPlIE1x6ujCc5mCwjcAZwadVTn-Zjb6MnTDvzJQ,2251
+dataeval/detectors/linters/__init__.py,sha256=xn2zPwUcmsuf-Jd9uw6AVI11C9z1b1Y9fYtuFnXenZ0,404
+dataeval/detectors/linters/duplicates.py,sha256=X5WSEvI_BHkLoXjkaHK6wTnSkx4IjpO_exMRjSlhc70,4963
+dataeval/detectors/linters/outliers.py,sha256=D8A-Fov5iUrlU9xMX5Ht33FqUY8Lk5ulC6BlHbUoLwU,9048
+dataeval/detectors/ood/__init__.py,sha256=juCYBDs7CQEAtMhnEpPqF6uTrOIH9kTBSuQ_GRw6a8o,283
+dataeval/detectors/ood/ae.py,sha256=fTrUfFxv6xUqzKpwMC8rW3JrizA16M_bgzqLuBKMrS0,2944
+dataeval/detectors/ood/base.py,sha256=9b-Ljznf0lB1SXF4F_Aj3eJ4Y3ijGEDPMjucUsWOGJM,3051
+dataeval/detectors/ood/mixin.py,sha256=0_o-1HPvgf3-Lf1MSOIfjj5UB8LTLEBGYtJJfyCCzwc,5431
+dataeval/detectors/ood/vae.py,sha256=Fcq0-WbLhzYCgYOAJPBklHm7yuXmFJuEpBkhgwM5kiA,2291
+dataeval/metadata/__init__.py,sha256=XDDmJbOZBNM6pL0r6Nbu6oMRoyAh22IDkPYGndNlkZU,316
+dataeval/metadata/_distance.py,sha256=T1Umju_QwBiLmn1iUbxZagzBS2VnHaDIdp6j-NpaZuk,4076
+dataeval/metadata/_ood.py,sha256=lnKtKModArnUrAhH_XswEtUAhUkh1U_oNsLt1UmNP44,12748
+dataeval/metadata/_utils.py,sha256=r8qBJT83RblobD5W5zyTVi6vYi51Dwkqswizdbzss-M,1169
+dataeval/metrics/__init__.py,sha256=8VC8q3HuJN3o_WN51Ae2_wXznl3RMXIvA5GYVcy7vr8,225
+dataeval/metrics/bias/__init__.py,sha256=329S1_3WnWqeU4-qVcbe0fMy4lDrj9uKslWHIQf93yg,839
+dataeval/metrics/bias/_balance.py,sha256=l1hTVkVwD85bP20MTthA-I5BkvbytylQkJu3Q6iTuPA,6152
+dataeval/metrics/bias/_completeness.py,sha256=BysXU2Jpw33n5dl3acJFEqF3mFGiJLsfG4n5Q2fkTaY,4608
+dataeval/metrics/bias/_coverage.py,sha256=PeUoOiaghUEdn6Ov8z2-am7-fnBVIPcFbJK7Ty5JObA,3647
+dataeval/metrics/bias/_diversity.py,sha256=B_qWVDMZfh818U0qVm8yidquB0H0XvW8N75OWVWXy2g,5814
+dataeval/metrics/bias/_parity.py,sha256=ea1D-eJh6cJxQ11XD6VbDXBKecE0jJJwptGD7LQJmBw,11529
+dataeval/metrics/estimators/__init__.py,sha256=Pnds8uIyAovt2fKqZjiHCIP_kVoBWlVllekYuK5UmmU,568
+dataeval/metrics/estimators/_ber.py,sha256=C30E5LiGGTAfo31zWFYDptDg0R7CTJGJ-a60YgzSkYY,5382
+dataeval/metrics/estimators/_clusterer.py,sha256=1HrpihGTJ63IkNSOy4Ibw633Gllkm1RxKmoKT5MOgt0,1434
+dataeval/metrics/estimators/_divergence.py,sha256=QDWl1lyAYoO9D3Ho7qOHSk6ud8Gi2MGuXEsYwO1HxvA,4043
+dataeval/metrics/estimators/_uap.py,sha256=BULEBbJ9BQ1IcTeZf0x7iI60QHAWCccBOM97FIu9VXA,1928
+dataeval/metrics/stats/__init__.py,sha256=6tA_9nbbM5ObJ6cds8Y1VBtTQiTOxrpGQSFLu_lWGGA,1098
+dataeval/metrics/stats/_base.py,sha256=YIfOVGd7E19B4dpAnzDYRQkaikvRRyJIpznJNfVtPdw,10750
+dataeval/metrics/stats/_boxratiostats.py,sha256=8Kd2FTZ5PLNYZfdAjU_R385gb0Z16JY0L9H_d5ZhgQs,6341
+dataeval/metrics/stats/_dimensionstats.py,sha256=73mFP-Myxne0peFliwvTntc0kk4cpq0krzMvSLDSIMM,2702
+dataeval/metrics/stats/_hashstats.py,sha256=gp9X_pnTT3mPH9YNrWLdn2LQPK_epJ3dQRoyOCwmKlg,4758
+dataeval/metrics/stats/_imagestats.py,sha256=gUPNgN5Zwzdr7WnSwbve1NXNsyxd5dy3cSnlR_7guCg,3007
+dataeval/metrics/stats/_labelstats.py,sha256=lz8I6eSd8tFkmQqy5cOG8hn9yxs0mP-Ic9ratFHiuoU,2813
+dataeval/metrics/stats/_pixelstats.py,sha256=SfergRbjNJE4h0xqe-0c8RnKtZmEkZ9MwExdipLSGvg,3247
+dataeval/metrics/stats/_visualstats.py,sha256=cq4AbF2B50Ihbzb86FphcnKQ1TSwNnP3PsnbpiPQZWw,3698
+dataeval/outputs/__init__.py,sha256=ciK-RdXgtn_s7MSCUW1UXvrXltMbltqbpfe9_V7xGrI,1701
+dataeval/outputs/_base.py,sha256=aZFbgybnZSQ3ws7QYRLTbDFqUfBFRVtIwX2LZfeGFUA,5703
+dataeval/outputs/_bias.py,sha256=7L-d3DUWY6Vud7iX_VoQT0HG0KaV1U35gvmRApqzyB0,12401
+dataeval/outputs/_drift.py,sha256=gOiu2C-ERTWiRqlP0auMYxPBGdm9HecWPqWfg7I4tZg,2015
+dataeval/outputs/_estimators.py,sha256=a2oAIxxEDZ9WLGfMWH8KD-BVUS_SnULRPR-iI9hFPoQ,3047
+dataeval/outputs/_linters.py,sha256=YOdjrfm8ypdRrqYOaPM9nc6wVJI3-ita3Haj7LHDNaw,6416
+dataeval/outputs/_metadata.py,sha256=ffZgpX8KWURPHXpOWjbvJ2KRqWQkS2nWuIjKUzoHhMI,1710
+dataeval/outputs/_ood.py,sha256=suLKVXULGtXH0rq9eXHI1d3d2jhGmItJtz4QiQd47A4,1718
+dataeval/outputs/_stats.py,sha256=c73Yc3Kkrl-MN6BGKe1V0Yr6Ix2Yp_DZZfFSp8fZMZ0,13180
+dataeval/outputs/_utils.py,sha256=HHlGC7sk416m_3Bgn075Qdblz_aPup_UOafJpB0RuXY,893
+dataeval/outputs/_workflows.py,sha256=MkRD6ubI4NCBXb9v3kjXy64cUGs3G-JKkBdOpRD9XVE,10750
+dataeval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dataeval/typing.py,sha256=GDMuef-oFFukNtsiKFmsExHdNvYR_j-tQcsCwZ9reow,7198
+dataeval/utils/__init__.py,sha256=hRvyUK7b3d6JBEV5u47rFcOHEcmDYqAvZQw_T5pDAWw,264
+dataeval/utils/_array.py,sha256=KqAdXEMjcXYvdWdYEEoEbigwQJ4S9VYxQS3sRFeY5XY,5929
+dataeval/utils/_bin.py,sha256=nylthmsC3vzLHLhlUMACvZs--h7xvAh9Pt75InaQJW8,7322
+dataeval/utils/_clusterer.py,sha256=fw5x-2QN0TIbiodDKHZxRgxKHINedpPcOklzce0Rbjg,5436
+dataeval/utils/_fast_mst.py,sha256=4_7ykVihCL5jWtxcGnrecIsDQo65kUml9SZ1JxgBZYY,7172
+dataeval/utils/_image.py,sha256=capzF_X5H0jy0PmTP3Hf52GFgLqrnfU6gS4tiwck9jo,1939
+dataeval/utils/_method.py,sha256=9B9JQbgqWJBRhQJb7glajUtWaQzUTIUuvrZ9_bisxsM,394
+dataeval/utils/_mst.py,sha256=f0vXytTUjlOS6AyL7c6PkXmaHuuGUK-vMLpq-5xMgxk,2183
+dataeval/utils/_plot.py,sha256=mTRQNbJsA42QMiOwZbJaH8sNYgP996QFDEGVVE9HSgY,7076
+dataeval/utils/data/__init__.py,sha256=xGzrjrOxOP2DP1tU84AWMKPnSxFvSjM81CTlDg4rNM8,331
+dataeval/utils/data/_dataset.py,sha256=MHY582yRm4FxQkkLWUhKZBb7ZyvWypM6ldUG89vd3uE,7936
+dataeval/utils/data/collate.py,sha256=5egEEKhNNCGeNLChO1p6dZ4Wg6x51VEaMNHz7hEZUxI,3936
+dataeval/utils/data/metadata.py,sha256=1XeGYj_e97-nJ_IrWEHPhWICmouYU5qbXWbp7uhZrIE,14171
+dataeval/utils/datasets/__init__.py,sha256=Jfe7XI_9U5S4wuI_2QCoeuWNOxz4j0nAQvxc5wG5mWY,486
+dataeval/utils/datasets/_base.py,sha256=TpmgPzF3EShCLAF5S4Zf9lFN78q17bTZF6AUE1qKdlk,8857
+dataeval/utils/datasets/_cifar10.py,sha256=oSX5JEzbBM4zGC9kC7-hVTOglms3rYaUuYiA00_DUJ4,5439
+dataeval/utils/datasets/_fileio.py,sha256=SixIk5nIlIwJdX9zjNXS10vHA3hL8aaYbqHsDg1xSpY,6447
+dataeval/utils/datasets/_milco.py,sha256=BF2XvyzuOop1mg5pFZcRfYmZcezlbpZWHyd_TtEHFF4,7573
+dataeval/utils/datasets/_mixin.py,sha256=FJgZP_cpJkgAHA3j3ai_j3Wt7aFSEjIMVmt9NpvVXzg,1757
+dataeval/utils/datasets/_mnist.py,sha256=4WOkQTORYMs6KEeyyJgChTnH03797y4ezgaZtYqplh4,8102
+dataeval/utils/datasets/_ships.py,sha256=RMdX2KlnXJYOTzBb6euA5TAqxs-S8b56pAGiyQhNMuo,4870
+dataeval/utils/datasets/_types.py,sha256=iSKyHXRlGuomXs0FHK6md8lXLQrQQ4fxgVOwr4o81bo,1089
+dataeval/utils/datasets/_voc.py,sha256=kif6ms_romK6VElP4pf2SK4cJ5dEHDOkxSaSaeP3c5k,15565
+dataeval/utils/torch/__init__.py,sha256=dn5mjCrFp0b1aL_UEURhONU0Ag0cmXoTOBSGagpkTiA,325
+dataeval/utils/torch/_blocks.py,sha256=HVhBTMMD5NA4qheMUgyol1KWiKZDIuc8k5j4RcMKmhk,1466
+dataeval/utils/torch/_gmm.py,sha256=XM68GNEP97EjaB1U49-ZXRb81d0CEFnPS910alrcB3g,3740
+dataeval/utils/torch/_internal.py,sha256=vHy-DzPhmvE8h3wmWc3aciBJ8nDGzQ1z1jTZgGjmDyM,4154
+dataeval/utils/torch/models.py,sha256=hmroEs6C6jQ5tAoZa71RFeIvXLxfXrTJSFH_jG2LGQU,9749
+dataeval/utils/torch/trainer.py,sha256=iUotX4OdirH8-ZtjdpU8gbJavkYW9YY9qpA2mAlFy1Y,5520
+dataeval/workflows/__init__.py,sha256=ou8y0KO-d6W5lgmcyLjKlf-J_ckP3vilW7wHkgiDlZ4,255
+dataeval/workflows/sufficiency.py,sha256=mjKmfRrAjShLUFIARv5o8yT5fnFvDsS5Qu6ujIPUgQg,8497
+dataeval-1.0.0.dist-info/LICENSE.txt,sha256=uAooygKWvX6NbU9Ran9oG2msttoG8aeTeHSTe5JeCnY,1061
+dataeval-1.0.0.dist-info/METADATA,sha256=ma_TquWQQl0QETiK4-wH1jfAe2my33Cl37GswNe0ZM8,5307
+dataeval-1.0.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+dataeval-1.0.0.dist-info/RECORD,,

dataeval/detectors/drift/_torch.py

@@ -1,222 +0,0 @@
-"""
-Source code derived from Alibi-Detect 0.11.4
-https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
-
-Original code Copyright (c) 2023 Seldon Technologies Ltd
-Licensed under Apache Software License (Apache 2.0)
-"""
-
-from __future__ import annotations
-
-__all__ = []
-
-from typing import Any, Callable
-
-import numpy as np
-import torch
-import torch.nn as nn
-from numpy.typing import NDArray
-
-from dataeval.config import DeviceLike, get_device
-from dataeval.utils.torch._internal import predict_batch
-
-
-def mmd2_from_kernel_matrix(
-    kernel_mat: torch.Tensor, m: int, permute: bool = False, zero_diag: bool = True
-) -> torch.Tensor:
-    """
-    Compute maximum mean discrepancy (MMD^2) between 2 samples x and y from the
-    full kernel matrix between the samples.
-
-    Parameters
-    ----------
-    kernel_mat : torch.Tensor
-        Kernel matrix between samples x and y.
-    m : int
-        Number of instances in y.
-    permute : bool, default False
-        Whether to permute the row indices. Used for permutation tests.
-    zero_diag : bool, default True
-        Whether to zero out the diagonal of the kernel matrix.
-
-    Returns
-    -------
-    torch.Tensor
-        MMD^2 between the samples from the kernel matrix.
-    """
-    n = kernel_mat.shape[0] - m
-    if zero_diag:
-        kernel_mat = kernel_mat - torch.diag(kernel_mat.diag())
-    if permute:
-        idx = torch.randperm(kernel_mat.shape[0])
-        kernel_mat = kernel_mat[idx][:, idx]
-    k_xx, k_yy, k_xy = kernel_mat[:-m, :-m], kernel_mat[-m:, -m:], kernel_mat[-m:, :-m]
-    c_xx, c_yy = 1 / (n * (n - 1)), 1 / (m * (m - 1))
-    mmd2 = c_xx * k_xx.sum() + c_yy * k_yy.sum() - 2.0 * k_xy.mean()
-    return mmd2
-
-
-def preprocess_drift(
-    x: NDArray[Any],
-    model: nn.Module,
-    device: DeviceLike | None = None,
-    preprocess_batch_fn: Callable | None = None,
-    batch_size: int = int(1e10),
-    dtype: type[np.generic] | torch.dtype = np.float32,
-) -> NDArray[Any] | torch.Tensor | tuple[Any, ...]:
-    """
-    Prediction function used for preprocessing step of drift detector.
-
-    Parameters
-    ----------
-    x : NDArray
-        Batch of instances.
-    model : nn.Module
-        Model used for preprocessing.
-    device : DeviceLike or None, default None
-        The hardware device to use if specified, otherwise uses the DataEval
-        default or torch default.
-    preprocess_batch_fn : Callable or None, default None
-        Optional batch preprocessing function. For example to convert a list of objects
-        to a batch which can be processed by the PyTorch model.
-    batch_size : int, default 1e10
-        Batch size used during prediction.
-    dtype : np.dtype or torch.dtype, default np.float32
-        Model output type, either a :term:`NumPy` or torch dtype, e.g. np.float32 or torch.float32.
-
-    Returns
-    -------
-    NDArray | torch.Tensor | tuple
-        Numpy array, torch tensor or tuples of those with model outputs.
-    """
-    return predict_batch(
-        x,
-        model,
-        device=get_device(device),
-        batch_size=batch_size,
-        preprocess_fn=preprocess_batch_fn,
-        dtype=dtype,
-    )
-
-
-@torch.jit.script
-def _squared_pairwise_distance(
-    x: torch.Tensor, y: torch.Tensor, a_min: float = 1e-30
-) -> torch.Tensor:  # pragma: no cover - torch.jit.script code is compiled and copied
-    """
-    PyTorch pairwise squared Euclidean distance between samples x and y.
-
-    Parameters
-    ----------
-    x : torch.Tensor
-        Batch of instances of shape [Nx, features].
-    y : torch.Tensor
-        Batch of instances of shape [Ny, features].
-    a_min : float
-        Lower bound to clip distance values.
-
-    Returns
-    -------
-    torch.Tensor
-        Pairwise squared Euclidean distance [Nx, Ny].
-    """
-    x2 = x.pow(2).sum(dim=-1, keepdim=True)
-    y2 = y.pow(2).sum(dim=-1, keepdim=True)
-    dist = torch.addmm(y2.transpose(-2, -1), x, y.transpose(-2, -1), alpha=-2).add_(x2)
-    return dist.clamp_min_(a_min)
-
-
-def sigma_median(x: torch.Tensor, y: torch.Tensor, dist: torch.Tensor) -> torch.Tensor:
-    """
-    Bandwidth estimation using the median heuristic `Gretton2012`
-
-    Parameters
-    ----------
-    x : torch.Tensor
-        Tensor of instances with dimension [Nx, features].
-    y : torch.Tensor
-        Tensor of instances with dimension [Ny, features].
-    dist : torch.Tensor
-        Tensor with dimensions [Nx, Ny], containing the pairwise distances
-        between `x` and `y`.
-
-    Returns
-    -------
-    torch.Tensor
-        The computed bandwidth, `sigma`.
-    """
-    n = min(x.shape[0], y.shape[0])
-    n = n if (x[:n] == y[:n]).all() and x.shape == y.shape else 0
-    n_median = n + (np.prod(dist.shape) - n) // 2 - 1
-    sigma = (0.5 * dist.flatten().sort().values[int(n_median)].unsqueeze(dim=-1)) ** 0.5
-    return sigma
-
-
-class GaussianRBF(nn.Module):
-    """
-    Gaussian RBF kernel: k(x,y) = exp(-(1/(2*sigma^2)||x-y||^2).
-
-    A forward pass takes a batch of instances x [Nx, features] and
-    y [Ny, features] and returns the kernel matrix [Nx, Ny].
-
-    Parameters
-    ----------
-    sigma : torch.Tensor | None, default None
-        Bandwidth used for the kernel. Needn't be specified if being inferred or
-        trained. Can pass multiple values to eval kernel with and then average.
-    init_sigma_fn : Callable | None, default None
-        Function used to compute the bandwidth ``sigma``. Used when ``sigma`` is to be
-        inferred. The function's signature should take in the tensors ``x``, ``y`` and
-        ``dist`` and return ``sigma``. If ``None``, it is set to ``sigma_median``.
-    trainable : bool, default False
-        Whether or not to track gradients w.r.t. `sigma` to allow it to be trained.
-    """
-
-    def __init__(
-        self,
-        sigma: torch.Tensor | None = None,
-        init_sigma_fn: Callable | None = None,
-        trainable: bool = False,
-    ) -> None:
-        super().__init__()
-        init_sigma_fn = sigma_median if init_sigma_fn is None else init_sigma_fn
-        self.config: dict[str, Any] = {
-            "sigma": sigma,
-            "trainable": trainable,
-            "init_sigma_fn": init_sigma_fn,
-        }
-        if sigma is None:
-            self.log_sigma: nn.Parameter = nn.Parameter(torch.empty(1), requires_grad=trainable)
-            self.init_required: bool = True
-        else:
-            sigma = sigma.reshape(-1)  # [Ns,]
-            self.log_sigma: nn.Parameter = nn.Parameter(sigma.log(), requires_grad=trainable)
-            self.init_required: bool = False
-        self.init_sigma_fn = init_sigma_fn
-        self.trainable = trainable
-
-    @property
-    def sigma(self) -> torch.Tensor:
-        return self.log_sigma.exp()
-
-    def forward(
-        self,
-        x: np.ndarray[Any, Any] | torch.Tensor,
-        y: np.ndarray[Any, Any] | torch.Tensor,
-        infer_sigma: bool = False,
-    ) -> torch.Tensor:
-        x, y = torch.as_tensor(x), torch.as_tensor(y)
-        dist = _squared_pairwise_distance(x.flatten(1), y.flatten(1))  # [Nx, Ny]
-
-        if infer_sigma or self.init_required:
-            if self.trainable and infer_sigma:
-                raise ValueError("Gradients cannot be computed w.r.t. an inferred sigma value")
-            sigma = self.init_sigma_fn(x, y, dist)
-            with torch.no_grad():
-                self.log_sigma.copy_(sigma.log().clone())
-            self.init_required: bool = False
-
-        gamma = 1.0 / (2.0 * self.sigma**2)  # [Ns,]
-        # TODO: do matrix multiplication after all?
-        kernel_mat = torch.exp(-torch.cat([(g * dist)[None, :, :] for g in gamma], dim=0))  # [Ns, Nx, Ny]
-        return kernel_mat.mean(dim=0)  # [Nx, Ny]

dataeval/utils/data/_embeddings.py

@@ -1,186 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-import math
-from typing import Any, Iterator, Sequence, cast
-
-import torch
-from torch.utils.data import DataLoader, Subset
-from tqdm import tqdm
-
-from dataeval.config import DeviceLike, get_device
-from dataeval.typing import Array, Dataset, Transform
-from dataeval.utils.torch.models import SupportsEncode
-
-
-class Embeddings:
-    """
-    Collection of image embeddings from a dataset.
-
-    Embeddings are accessed by index or slice and are only loaded on-demand.
-
-    Parameters
-    ----------
-    dataset : ImageClassificationDataset or ObjectDetectionDataset
-        Dataset to access original images from.
-    batch_size : int
-        Batch size to use when encoding images.
-    transforms : Transform or Sequence[Transform] or None, default None
-        Transforms to apply to images before encoding.
-    model : torch.nn.Module or None, default None
-        Model to use for encoding images.
-    device : DeviceLike or None, default None
-        The hardware device to use if specified, otherwise uses the DataEval
-        default or torch default.
-    cache : bool, default False
-        Whether to cache the embeddings in memory.
-    verbose : bool, default False
-        Whether to print progress bar when encoding images.
-    """
-
-    device: torch.device
-    batch_size: int
-    verbose: bool
-
-    def __init__(
-        self,
-        dataset: Dataset[tuple[Array, Any, Any]],
-        batch_size: int,
-        transforms: Transform[torch.Tensor] | Sequence[Transform[torch.Tensor]] | None = None,
-        model: torch.nn.Module | None = None,
-        device: DeviceLike | None = None,
-        cache: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        self.device = get_device(device)
-        self.cache = cache
-        self.batch_size = batch_size if batch_size > 0 else 1
-        self.verbose = verbose
-
-        self._dataset = dataset
-        self._length = len(dataset)
-        model = torch.nn.Flatten() if model is None else model
-        self._transforms = [transforms] if isinstance(transforms, Transform) else transforms
-        self._model = model.to(self.device).eval()
-        self._encoder = model.encode if isinstance(model, SupportsEncode) else model
-        self._collate_fn = lambda datum: [torch.as_tensor(i) for i, _, _ in datum]
-        self._cached_idx = set()
-        self._embeddings: torch.Tensor = torch.empty(())
-        self._shallow: bool = False
-
-    def to_tensor(self, indices: Sequence[int] | None = None) -> torch.Tensor:
-        """
-        Converts dataset to embeddings.
-
-        Parameters
-        ----------
-        indices : Sequence[int] or None, default None
-            The indices to convert to embeddings
-
-        Returns
-        -------
-        torch.Tensor
-
-        Warning
-        -------
-        Processing large quantities of data can be resource intensive.
-        """
-        if indices is not None:
-            return torch.vstack(list(self._batch(indices))).to(self.device)
-        else:
-            return self[:]
-
-    @classmethod
-    def from_array(cls, array: Array, device: DeviceLike | None = None) -> Embeddings:
-        """
-        Instantiates a shallow Embeddings object using an array.
-
-        Parameters
-        ----------
-        array : Array
-            The array to convert to embeddings.
-        device : DeviceLike or None, default None
-            The hardware device to use if specified, otherwise uses the DataEval
-            default or torch default.
-
-        Returns
-        -------
-        Embeddings
-
-        Example
-        -------
-        >>> import numpy as np
-        >>> from dataeval.utils.data._embeddings import Embeddings
-        >>> array = np.random.randn(100, 3, 224, 224)
-        >>> embeddings = Embeddings.from_array(array)
-        >>> print(embeddings.to_tensor().shape)
-        torch.Size([100, 3, 224, 224])
-        """
-        embeddings = Embeddings([], 0, None, None, device, True, False)
-        embeddings._length = len(array)
-        embeddings._cached_idx = set(range(len(array)))
-        embeddings._embeddings = torch.as_tensor(array).to(get_device(device))
-        embeddings._shallow = True
-        return embeddings
-
-    def _encode(self, images: list[torch.Tensor]) -> torch.Tensor:
-        if self._transforms:
-            images = [transform(image) for transform in self._transforms for image in images]
-        return self._encoder(torch.stack(images).to(self.device))
-
-    @torch.no_grad()  # Reduce overhead cost by not tracking tensor gradients
-    def _batch(self, indices: Sequence[int]) -> Iterator[torch.Tensor]:
-        dataset = cast(torch.utils.data.Dataset[tuple[Array, Any, Any]], self._dataset)
-        total_batches = math.ceil(len(indices) / self.batch_size)
-
-        # If not caching, process all indices normally
-        if not self.cache:
-            for images in tqdm(
-                DataLoader(Subset(dataset, indices), self.batch_size, collate_fn=self._collate_fn),
-                total=total_batches,
-                desc="Batch embedding",
-                disable=not self.verbose,
-            ):
-                yield self._encode(images)
-            return
-
-        # If caching, process each batch of indices at a time, preserving original order
-        for i in tqdm(range(0, len(indices), self.batch_size), desc="Batch embedding", disable=not self.verbose):
-            batch = indices[i : i + self.batch_size]
-            uncached = [idx for idx in batch if idx not in self._cached_idx]
-
-            if uncached:
-                # Process uncached indices as as single batch
-                for images in DataLoader(Subset(dataset, uncached), len(uncached), collate_fn=self._collate_fn):
-                    embeddings = self._encode(images)
-
-                    if not self._embeddings.shape:
-                        full_shape = (len(self._dataset), *embeddings.shape[1:])
-                        self._embeddings = torch.empty(full_shape, dtype=embeddings.dtype, device=self.device)
-
-                    self._embeddings[uncached] = embeddings
-                    self._cached_idx.update(uncached)
-
-            yield self._embeddings[batch]
-
-    def __getitem__(self, key: int | slice, /) -> torch.Tensor:
-        if not isinstance(key, slice) and not hasattr(key, "__int__"):
-            raise TypeError("Invalid argument type.")
-
-        if self._shallow:
-            if not self._embeddings.shape:
-                raise ValueError("Embeddings not initialized.")
-            return self._embeddings[key]
-
-        indices = list(range(len(self._dataset))[key]) if isinstance(key, slice) else [int(key)]
-        result = torch.vstack(list(self._batch(indices))).to(self.device)
-        return result.squeeze(0) if len(indices) == 1 else result
-
-    def __iter__(self) -> Iterator[torch.Tensor]:
-        # process in batches while yielding individual embeddings
-        for batch in self._batch(range(self._length)):
-            yield from batch
-
-    def __len__(self) -> int:
-        return self._length

dataeval/utils/data/datasets/__init__.py

@@ -1,17 +0,0 @@
-"""Provides access to common Computer Vision datasets."""
-
-from dataeval.utils.data.datasets._cifar10 import CIFAR10
-from dataeval.utils.data.datasets._milco import MILCO
-from dataeval.utils.data.datasets._mnist import MNIST
-from dataeval.utils.data.datasets._ships import Ships
-from dataeval.utils.data.datasets._voc import VOCDetection, VOCDetectionTorch, VOCSegmentation
-
-__all__ = [
-    "MNIST",
-    "Ships",
-    "CIFAR10",
-    "MILCO",
-    "VOCDetection",
-    "VOCDetectionTorch",
-    "VOCSegmentation",
-]

dataeval/utils/data/selections/__init__.py

@@ -1,17 +0,0 @@
-"""Provides selection classes for selecting subsets of Computer Vision datasets."""
-
-__all__ = [
-    "ClassFilter",
-    "Indices",
-    "Limit",
-    "Prioritize",
-    "Reverse",
-    "Shuffle",
-]
-
-from dataeval.utils.data.selections._classfilter import ClassFilter
-from dataeval.utils.data.selections._indices import Indices
-from dataeval.utils.data.selections._limit import Limit
-from dataeval.utils.data.selections._prioritize import Prioritize
-from dataeval.utils.data.selections._reverse import Reverse
-from dataeval.utils.data.selections._shuffle import Shuffle