dataeval 0.76.1__py3-none-any.whl → 0.82.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataeval/__init__.py +3 -3
- dataeval/config.py +77 -0
- dataeval/detectors/__init__.py +1 -1
- dataeval/detectors/drift/__init__.py +6 -6
- dataeval/detectors/drift/{base.py → _base.py} +40 -85
- dataeval/detectors/drift/{cvm.py → _cvm.py} +21 -28
- dataeval/detectors/drift/{ks.py → _ks.py} +20 -26
- dataeval/detectors/drift/{mmd.py → _mmd.py} +31 -43
- dataeval/detectors/drift/{torch.py → _torch.py} +2 -1
- dataeval/detectors/drift/{uncertainty.py → _uncertainty.py} +24 -7
- dataeval/detectors/drift/updates.py +20 -3
- dataeval/detectors/linters/__init__.py +3 -5
- dataeval/detectors/linters/duplicates.py +13 -36
- dataeval/detectors/linters/outliers.py +23 -148
- dataeval/detectors/ood/__init__.py +1 -1
- dataeval/detectors/ood/ae.py +30 -9
- dataeval/detectors/ood/base.py +5 -4
- dataeval/detectors/ood/mixin.py +21 -7
- dataeval/detectors/ood/vae.py +73 -0
- dataeval/metadata/__init__.py +6 -0
- dataeval/metadata/_distance.py +167 -0
- dataeval/metadata/_ood.py +217 -0
- dataeval/metadata/_utils.py +44 -0
- dataeval/metrics/__init__.py +1 -1
- dataeval/metrics/bias/__init__.py +6 -4
- dataeval/metrics/bias/{balance.py → _balance.py} +15 -101
- dataeval/metrics/bias/_coverage.py +98 -0
- dataeval/metrics/bias/{diversity.py → _diversity.py} +18 -111
- dataeval/metrics/bias/{parity.py → _parity.py} +39 -77
- dataeval/metrics/estimators/__init__.py +15 -4
- dataeval/metrics/estimators/{ber.py → _ber.py} +42 -29
- dataeval/metrics/estimators/_clusterer.py +44 -0
- dataeval/metrics/estimators/{divergence.py → _divergence.py} +18 -30
- dataeval/metrics/estimators/{uap.py → _uap.py} +4 -18
- dataeval/metrics/stats/__init__.py +16 -13
- dataeval/metrics/stats/{base.py → _base.py} +82 -133
- dataeval/metrics/stats/{boxratiostats.py → _boxratiostats.py} +15 -18
- dataeval/metrics/stats/_dimensionstats.py +75 -0
- dataeval/metrics/stats/{hashstats.py → _hashstats.py} +21 -37
- dataeval/metrics/stats/_imagestats.py +94 -0
- dataeval/metrics/stats/_labelstats.py +131 -0
- dataeval/metrics/stats/{pixelstats.py → _pixelstats.py} +19 -50
- dataeval/metrics/stats/{visualstats.py → _visualstats.py} +23 -54
- dataeval/outputs/__init__.py +53 -0
- dataeval/{output.py → outputs/_base.py} +55 -25
- dataeval/outputs/_bias.py +381 -0
- dataeval/outputs/_drift.py +83 -0
- dataeval/outputs/_estimators.py +114 -0
- dataeval/outputs/_linters.py +184 -0
- dataeval/{detectors/ood/output.py → outputs/_ood.py} +22 -22
- dataeval/outputs/_stats.py +387 -0
- dataeval/outputs/_utils.py +44 -0
- dataeval/outputs/_workflows.py +364 -0
- dataeval/typing.py +234 -0
- dataeval/utils/__init__.py +2 -2
- dataeval/utils/_array.py +169 -0
- dataeval/utils/_bin.py +199 -0
- dataeval/utils/_clusterer.py +144 -0
- dataeval/utils/_fast_mst.py +189 -0
- dataeval/utils/{image.py → _image.py} +6 -4
- dataeval/utils/_method.py +14 -0
- dataeval/utils/{shared.py → _mst.py} +3 -65
- dataeval/utils/{plot.py → _plot.py} +6 -6
- dataeval/utils/data/__init__.py +26 -0
- dataeval/utils/data/_dataset.py +217 -0
- dataeval/utils/data/_embeddings.py +104 -0
- dataeval/utils/data/_images.py +68 -0
- dataeval/utils/data/_metadata.py +360 -0
- dataeval/utils/data/_selection.py +126 -0
- dataeval/utils/{dataset/split.py → data/_split.py} +12 -38
- dataeval/utils/data/_targets.py +85 -0
- dataeval/utils/data/collate.py +103 -0
- dataeval/utils/data/datasets/__init__.py +17 -0
- dataeval/utils/data/datasets/_base.py +254 -0
- dataeval/utils/data/datasets/_cifar10.py +134 -0
- dataeval/utils/data/datasets/_fileio.py +168 -0
- dataeval/utils/data/datasets/_milco.py +153 -0
- dataeval/utils/data/datasets/_mixin.py +56 -0
- dataeval/utils/data/datasets/_mnist.py +183 -0
- dataeval/utils/data/datasets/_ships.py +123 -0
- dataeval/utils/data/datasets/_types.py +52 -0
- dataeval/utils/data/datasets/_voc.py +352 -0
- dataeval/utils/data/selections/__init__.py +15 -0
- dataeval/utils/data/selections/_classfilter.py +57 -0
- dataeval/utils/data/selections/_indices.py +26 -0
- dataeval/utils/data/selections/_limit.py +26 -0
- dataeval/utils/data/selections/_reverse.py +18 -0
- dataeval/utils/data/selections/_shuffle.py +29 -0
- dataeval/utils/metadata.py +51 -376
- dataeval/utils/torch/{gmm.py → _gmm.py} +4 -2
- dataeval/utils/torch/{internal.py → _internal.py} +21 -51
- dataeval/utils/torch/models.py +43 -2
- dataeval/workflows/__init__.py +2 -1
- dataeval/workflows/sufficiency.py +11 -346
- {dataeval-0.76.1.dist-info → dataeval-0.82.0.dist-info}/METADATA +5 -2
- dataeval-0.82.0.dist-info/RECORD +104 -0
- dataeval/detectors/linters/clusterer.py +0 -512
- dataeval/detectors/linters/merged_stats.py +0 -49
- dataeval/detectors/ood/metadata_ks_compare.py +0 -129
- dataeval/detectors/ood/metadata_least_likely.py +0 -119
- dataeval/interop.py +0 -69
- dataeval/metrics/bias/coverage.py +0 -194
- dataeval/metrics/stats/datasetstats.py +0 -202
- dataeval/metrics/stats/dimensionstats.py +0 -115
- dataeval/metrics/stats/labelstats.py +0 -210
- dataeval/utils/dataset/__init__.py +0 -7
- dataeval/utils/dataset/datasets.py +0 -412
- dataeval/utils/dataset/read.py +0 -63
- dataeval-0.76.1.dist-info/RECORD +0 -67
- /dataeval/{log.py → _log.py} +0 -0
- /dataeval/utils/torch/{blocks.py → _blocks.py} +0 -0
- {dataeval-0.76.1.dist-info → dataeval-0.82.0.dist-info}/LICENSE.txt +0 -0
- {dataeval-0.76.1.dist-info → dataeval-0.82.0.dist-info}/WHEEL +0 -0
@@ -11,30 +11,7 @@ from numpy.typing import NDArray
|
|
11
11
|
from torch.utils.data import DataLoader, TensorDataset
|
12
12
|
from tqdm import tqdm
|
13
13
|
|
14
|
-
|
15
|
-
def get_device(device: str | torch.device | None = None) -> torch.device:
|
16
|
-
"""
|
17
|
-
Instantiates a PyTorch device object.
|
18
|
-
|
19
|
-
Parameters
|
20
|
-
----------
|
21
|
-
device : str | torch.device | None, default None
|
22
|
-
Either ``None``, a str ('gpu' or 'cpu') indicating the device to choose, or an
|
23
|
-
already instantiated device object. If ``None``, the GPU is selected if it is
|
24
|
-
detected, otherwise the CPU is used as a fallback.
|
25
|
-
|
26
|
-
Returns
|
27
|
-
-------
|
28
|
-
The instantiated device object.
|
29
|
-
"""
|
30
|
-
if isinstance(device, torch.device): # Already a torch device
|
31
|
-
return device
|
32
|
-
else: # Instantiate device
|
33
|
-
if device is None or device.lower() in ["gpu", "cuda"]:
|
34
|
-
torch_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
35
|
-
else:
|
36
|
-
torch_device = torch.device("cpu")
|
37
|
-
return torch_device
|
14
|
+
from dataeval.config import get_device
|
38
15
|
|
39
16
|
|
40
17
|
def predict_batch(
|
@@ -42,7 +19,7 @@ def predict_batch(
|
|
42
19
|
model: Callable | torch.nn.Module | torch.nn.Sequential,
|
43
20
|
device: torch.device | None = None,
|
44
21
|
batch_size: int = int(1e10),
|
45
|
-
preprocess_fn: Callable | None = None,
|
22
|
+
preprocess_fn: Callable[[torch.Tensor], torch.Tensor] | None = None,
|
46
23
|
dtype: type[np.generic] | torch.dtype = np.float32,
|
47
24
|
) -> NDArray[Any] | torch.Tensor | tuple[Any, ...]:
|
48
25
|
"""
|
@@ -71,11 +48,12 @@ def predict_batch(
|
|
71
48
|
"""
|
72
49
|
device = get_device(device)
|
73
50
|
if isinstance(x, np.ndarray):
|
74
|
-
x = torch.
|
51
|
+
x = torch.tensor(x, device=device)
|
75
52
|
n = len(x)
|
76
53
|
n_minibatch = int(np.ceil(n / batch_size))
|
77
54
|
return_np = not isinstance(dtype, torch.dtype)
|
78
|
-
|
55
|
+
preds_tuple = None
|
56
|
+
preds_array = []
|
79
57
|
with torch.no_grad():
|
80
58
|
for i in range(n_minibatch):
|
81
59
|
istart, istop = i * batch_size, min((i + 1) * batch_size, n)
|
@@ -83,23 +61,17 @@ def predict_batch(
|
|
83
61
|
if isinstance(preprocess_fn, Callable):
|
84
62
|
x_batch = preprocess_fn(x_batch)
|
85
63
|
|
86
|
-
preds_tmp = model(x_batch.to(torch.float32)
|
64
|
+
preds_tmp = model(x_batch.to(dtype=torch.float32))
|
87
65
|
if isinstance(preds_tmp, (list, tuple)):
|
88
|
-
if
|
89
|
-
|
66
|
+
if preds_tuple is None: # init tuple with lists to store predictions
|
67
|
+
preds_tuple = tuple([] for _ in range(len(preds_tmp)))
|
90
68
|
for j, p in enumerate(preds_tmp):
|
91
|
-
if isinstance(p, torch.Tensor)
|
92
|
-
|
93
|
-
preds[j].append(p if not return_np or isinstance(p, np.ndarray) else p.numpy())
|
69
|
+
p = p.cpu() if isinstance(p, torch.Tensor) else p
|
70
|
+
preds_tuple[j].append(p if not return_np or isinstance(p, np.ndarray) else p.numpy())
|
94
71
|
elif isinstance(preds_tmp, (np.ndarray, torch.Tensor)):
|
95
|
-
if isinstance(preds_tmp, torch.Tensor)
|
96
|
-
|
97
|
-
|
98
|
-
preds = list(preds)
|
99
|
-
preds.append(
|
100
|
-
preds_tmp
|
101
|
-
if not return_np or isinstance(preds_tmp, np.ndarray) # type: ignore
|
102
|
-
else preds_tmp.numpy()
|
72
|
+
preds_tmp = preds_tmp.cpu() if isinstance(preds_tmp, torch.Tensor) else preds_tmp
|
73
|
+
preds_array.append(
|
74
|
+
preds_tmp if not return_np or isinstance(preds_tmp, np.ndarray) else preds_tmp.numpy()
|
103
75
|
)
|
104
76
|
else:
|
105
77
|
raise TypeError(
|
@@ -108,9 +80,7 @@ def predict_batch(
|
|
108
80
|
torch.Tensor."
|
109
81
|
)
|
110
82
|
concat = partial(np.concatenate, axis=0) if return_np else partial(torch.cat, dim=0)
|
111
|
-
out
|
112
|
-
tuple(concat(p) for p in preds) if isinstance(preds, tuple) else concat(preds) # type: ignore
|
113
|
-
)
|
83
|
+
out = tuple(concat(p) for p in preds_tuple) if preds_tuple is not None else concat(preds_array)
|
114
84
|
return out
|
115
85
|
|
116
86
|
|
@@ -154,18 +124,18 @@ def trainer(
|
|
154
124
|
verbose
|
155
125
|
Whether to print training progress.
|
156
126
|
"""
|
127
|
+
if loss_fn is None:
|
128
|
+
loss_fn = torch.nn.MSELoss()
|
129
|
+
|
157
130
|
if optimizer is None:
|
158
131
|
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
|
159
132
|
|
160
133
|
if y_train is None:
|
161
|
-
dataset = TensorDataset(torch.
|
162
|
-
|
134
|
+
dataset = TensorDataset(torch.tensor(x_train, dtype=torch.float32))
|
163
135
|
else:
|
164
|
-
dataset = TensorDataset(
|
165
|
-
torch.from_numpy(x_train).to(torch.float32), torch.from_numpy(y_train).to(torch.float32)
|
166
|
-
)
|
136
|
+
dataset = TensorDataset(torch.tensor(x_train, dtype=torch.float32), torch.tensor(y_train, dtype=torch.float32))
|
167
137
|
|
168
|
-
loader = DataLoader(dataset=dataset)
|
138
|
+
loader = DataLoader(dataset=dataset, batch_size=batch_size)
|
169
139
|
|
170
140
|
model = model.to(device)
|
171
141
|
|
@@ -186,7 +156,7 @@ def trainer(
|
|
186
156
|
y_hat = model(x)
|
187
157
|
y = x if y is None else y
|
188
158
|
|
189
|
-
loss = loss_fn(y, y_hat) # type: ignore
|
159
|
+
loss = loss_fn(y, *y_hat) if isinstance(y_hat, tuple) else loss_fn(y, y_hat) # type: ignore
|
190
160
|
|
191
161
|
optimizer.zero_grad()
|
192
162
|
loss.backward()
|
dataeval/utils/torch/models.py
CHANGED
@@ -2,13 +2,19 @@
|
|
2
2
|
|
3
3
|
from __future__ import annotations
|
4
4
|
|
5
|
-
__all__ = ["Autoencoder", "Encoder", "Decoder"]
|
5
|
+
__all__ = ["Autoencoder", "Encoder", "Decoder", "ResNet18"]
|
6
6
|
|
7
7
|
import math
|
8
|
-
from typing import Any
|
8
|
+
from typing import Any, Protocol, runtime_checkable
|
9
9
|
|
10
10
|
import torch
|
11
11
|
import torch.nn as nn
|
12
|
+
from torchvision.models import ResNet18_Weights, resnet18
|
13
|
+
|
14
|
+
|
15
|
+
@runtime_checkable
|
16
|
+
class SupportsEncode(Protocol):
|
17
|
+
def encode(self, x: Any) -> Any: ...
|
12
18
|
|
13
19
|
|
14
20
|
class Autoencoder(nn.Module):
|
@@ -330,3 +336,38 @@ class Decoder_AE(nn.Module):
|
|
330
336
|
x = self.decoder(x)
|
331
337
|
x = x.reshape((-1, *self.input_shape))
|
332
338
|
return x
|
339
|
+
|
340
|
+
|
341
|
+
class ResNet18(nn.Module):
|
342
|
+
"""
|
343
|
+
A wrapper class for the torchvision.models.resnet18 model
|
344
|
+
|
345
|
+
|
346
|
+
Note
|
347
|
+
----
|
348
|
+
This class is provided for the use of DataEval documentation and excludes many features
|
349
|
+
of the torchvision implementation.
|
350
|
+
|
351
|
+
Warning
|
352
|
+
-------
|
353
|
+
This class has been thoroughly tested for the purposes
|
354
|
+
of DataEval's documentation but not for operational use.
|
355
|
+
Please use with caution if deploying this class or subclasses.
|
356
|
+
"""
|
357
|
+
|
358
|
+
def __init__(self, embedding_size: int = 128):
|
359
|
+
super().__init__()
|
360
|
+
self.model: nn.Module = resnet18(weights=ResNet18_Weights.DEFAULT, progress=False)
|
361
|
+
self.model.fc = nn.Linear(self.model.fc.in_features, embedding_size)
|
362
|
+
|
363
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
364
|
+
return self.model(x)
|
365
|
+
|
366
|
+
@staticmethod
|
367
|
+
def transforms() -> Any:
|
368
|
+
"""(Returns) the default ResNet18 IMAGENET1K_V1 transforms"""
|
369
|
+
|
370
|
+
return ResNet18_Weights.DEFAULT.transforms()
|
371
|
+
|
372
|
+
def __str__(self) -> str:
|
373
|
+
return str(self.model)
|
dataeval/workflows/__init__.py
CHANGED
@@ -4,4 +4,5 @@ Workflows perform a sequence of actions to analyze the dataset and make predicti
|
|
4
4
|
|
5
5
|
__all__ = ["Sufficiency", "SufficiencyOutput"]
|
6
6
|
|
7
|
-
from dataeval.
|
7
|
+
from dataeval.outputs._workflows import SufficiencyOutput
|
8
|
+
from dataeval.workflows.sufficiency import Sufficiency
|
@@ -2,260 +2,16 @@ from __future__ import annotations
|
|
2
2
|
|
3
3
|
__all__ = []
|
4
4
|
|
5
|
-
import
|
6
|
-
import warnings
|
7
|
-
from dataclasses import dataclass
|
8
|
-
from typing import Any, Callable, Generic, Iterable, Mapping, Sequence, TypeVar, cast
|
5
|
+
from typing import Any, Callable, Generic, Iterable, Mapping, Sequence, Sized, TypeVar
|
9
6
|
|
10
7
|
import numpy as np
|
11
8
|
import torch
|
12
9
|
import torch.nn as nn
|
13
|
-
from numpy.typing import ArrayLike, NDArray
|
14
|
-
from scipy.optimize import basinhopping
|
15
10
|
from torch.utils.data import Dataset
|
16
11
|
|
17
|
-
from dataeval.
|
18
|
-
from dataeval.
|
19
|
-
|
20
|
-
with contextlib.suppress(ImportError):
|
21
|
-
from matplotlib.figure import Figure
|
22
|
-
|
23
|
-
|
24
|
-
@dataclass(frozen=True)
|
25
|
-
class SufficiencyOutput(Output):
|
26
|
-
"""
|
27
|
-
Output class for :class:`Sufficiency` workflow.
|
28
|
-
|
29
|
-
Attributes
|
30
|
-
----------
|
31
|
-
steps : NDArray
|
32
|
-
Array of sample sizes
|
33
|
-
params : Dict[str, NDArray]
|
34
|
-
Inverse power curve coefficients for the line of best fit for each measure
|
35
|
-
measures : Dict[str, NDArray]
|
36
|
-
Average of values observed for each sample size step for each measure
|
37
|
-
"""
|
38
|
-
|
39
|
-
steps: NDArray[np.uint32]
|
40
|
-
params: dict[str, NDArray[np.float64]]
|
41
|
-
measures: dict[str, NDArray[np.float64]]
|
42
|
-
|
43
|
-
def __post_init__(self) -> None:
|
44
|
-
c = len(self.steps)
|
45
|
-
if set(self.params) != set(self.measures):
|
46
|
-
raise ValueError("params and measures have a key mismatch")
|
47
|
-
for m, v in self.measures.items():
|
48
|
-
c_v = v.shape[1] if v.ndim > 1 else len(v)
|
49
|
-
if c != c_v:
|
50
|
-
raise ValueError(f"{m} does not contain the expected number ({c}) of data points.")
|
51
|
-
|
52
|
-
@set_metadata
|
53
|
-
def project(
|
54
|
-
self,
|
55
|
-
projection: int | Iterable[int],
|
56
|
-
) -> SufficiencyOutput:
|
57
|
-
"""Projects the measures for each value of X
|
58
|
-
|
59
|
-
Parameters
|
60
|
-
----------
|
61
|
-
projection : int | Iterable[int]
|
62
|
-
Step or steps to project
|
63
|
-
|
64
|
-
Returns
|
65
|
-
-------
|
66
|
-
SufficiencyOutput
|
67
|
-
Dataclass containing the projected measures per projection
|
68
|
-
|
69
|
-
Raises
|
70
|
-
------
|
71
|
-
ValueError
|
72
|
-
If the length of data points in the measures do not match
|
73
|
-
If `projection` is not numerical
|
74
|
-
"""
|
75
|
-
projection = np.asarray(list(projection) if isinstance(projection, Iterable) else [projection])
|
76
|
-
|
77
|
-
if not np.issubdtype(projection.dtype, np.number):
|
78
|
-
raise ValueError("'projection' must consist of numerical values")
|
79
|
-
|
80
|
-
output = {}
|
81
|
-
for name, measures in self.measures.items():
|
82
|
-
if measures.ndim > 1:
|
83
|
-
result = []
|
84
|
-
for i in range(len(measures)):
|
85
|
-
projected = project_steps(self.params[name][i], projection)
|
86
|
-
result.append(projected)
|
87
|
-
output[name] = np.array(result)
|
88
|
-
else:
|
89
|
-
output[name] = project_steps(self.params[name], projection)
|
90
|
-
return SufficiencyOutput(projection, self.params, output)
|
91
|
-
|
92
|
-
def plot(self, class_names: Sequence[str] | None = None) -> list[Figure]:
|
93
|
-
"""Plotting function for data :term:`sufficience<Sufficiency>` tasks
|
94
|
-
|
95
|
-
Parameters
|
96
|
-
----------
|
97
|
-
class_names : Sequence[str] | None, default None
|
98
|
-
List of class names
|
99
|
-
|
100
|
-
Returns
|
101
|
-
-------
|
102
|
-
list[Figure]
|
103
|
-
List of Figures for each measure
|
104
|
-
|
105
|
-
Raises
|
106
|
-
------
|
107
|
-
ValueError
|
108
|
-
If the length of data points in the measures do not match
|
109
|
-
"""
|
110
|
-
# Extrapolation parameters
|
111
|
-
last_X = self.steps[-1]
|
112
|
-
geomshape = (0.01 * last_X, last_X * 4, len(self.steps))
|
113
|
-
extrapolated = np.geomspace(*geomshape).astype(np.int64)
|
114
|
-
|
115
|
-
# Stores all plots
|
116
|
-
plots = []
|
117
|
-
|
118
|
-
# Create a plot for each measure on one figure
|
119
|
-
for name, measures in self.measures.items():
|
120
|
-
if measures.ndim > 1:
|
121
|
-
if class_names is not None and len(measures) != len(class_names):
|
122
|
-
raise IndexError("Class name count does not align with measures")
|
123
|
-
for i, measure in enumerate(measures):
|
124
|
-
class_name = str(i) if class_names is None else class_names[i]
|
125
|
-
fig = plot_measure(
|
126
|
-
f"{name}_{class_name}",
|
127
|
-
self.steps,
|
128
|
-
measure,
|
129
|
-
self.params[name][i],
|
130
|
-
extrapolated,
|
131
|
-
)
|
132
|
-
plots.append(fig)
|
133
|
-
|
134
|
-
else:
|
135
|
-
fig = plot_measure(name, self.steps, measures, self.params[name], extrapolated)
|
136
|
-
plots.append(fig)
|
137
|
-
|
138
|
-
return plots
|
139
|
-
|
140
|
-
def inv_project(self, targets: Mapping[str, ArrayLike]) -> dict[str, NDArray[np.float64]]:
|
141
|
-
"""
|
142
|
-
Calculate training samples needed to achieve target model metric values.
|
143
|
-
|
144
|
-
Parameters
|
145
|
-
----------
|
146
|
-
targets : Mapping[str, ArrayLike]
|
147
|
-
Mapping of target metric scores (from 0.0 to 1.0) that we want
|
148
|
-
to achieve, where the key is the name of the metric.
|
149
|
-
|
150
|
-
Returns
|
151
|
-
-------
|
152
|
-
dict[str, NDArray]
|
153
|
-
List of the number of training samples needed to achieve each
|
154
|
-
corresponding entry in targets
|
155
|
-
"""
|
156
|
-
|
157
|
-
projection = {}
|
158
|
-
|
159
|
-
for name, target in targets.items():
|
160
|
-
tarray = as_numpy(target)
|
161
|
-
if name not in self.measures:
|
162
|
-
continue
|
163
|
-
|
164
|
-
measure = self.measures[name]
|
165
|
-
if measure.ndim > 1:
|
166
|
-
projection[name] = np.zeros((len(measure), len(tarray)))
|
167
|
-
for i in range(len(measure)):
|
168
|
-
projection[name][i] = inv_project_steps(
|
169
|
-
self.params[name][i], tarray[i] if tarray.ndim == measure.ndim else tarray
|
170
|
-
)
|
171
|
-
else:
|
172
|
-
projection[name] = inv_project_steps(self.params[name], tarray)
|
173
|
-
|
174
|
-
return projection
|
175
|
-
|
176
|
-
|
177
|
-
def f_out(n_i: NDArray[Any], x: NDArray[Any]) -> NDArray[Any]:
|
178
|
-
"""
|
179
|
-
Calculates the line of best fit based on its free parameters
|
180
|
-
|
181
|
-
Parameters
|
182
|
-
----------
|
183
|
-
n_i : NDArray
|
184
|
-
Array of sample sizes
|
185
|
-
x : NDArray
|
186
|
-
Array of inverse power curve coefficients
|
187
|
-
|
188
|
-
Returns
|
189
|
-
-------
|
190
|
-
NDArray
|
191
|
-
Data points for the line of best fit
|
192
|
-
"""
|
193
|
-
return x[0] * n_i ** (-x[1]) + x[2]
|
194
|
-
|
195
|
-
|
196
|
-
def f_inv_out(y_i: NDArray[Any], x: NDArray[Any]) -> NDArray[np.uint64]:
|
197
|
-
"""
|
198
|
-
Inverse function for f_out()
|
199
|
-
|
200
|
-
Parameters
|
201
|
-
----------
|
202
|
-
y_i : NDArray
|
203
|
-
Data points for the line of best fit
|
204
|
-
x : NDArray
|
205
|
-
Array of inverse power curve coefficients
|
206
|
-
|
207
|
-
Returns
|
208
|
-
-------
|
209
|
-
NDArray
|
210
|
-
Array of sample sizes
|
211
|
-
"""
|
212
|
-
n_i = ((y_i - x[2]) / x[0]) ** (-1 / x[1])
|
213
|
-
return np.asarray(n_i, dtype=np.uint64)
|
214
|
-
|
215
|
-
|
216
|
-
def calc_params(p_i: NDArray[Any], n_i: NDArray[Any], niter: int) -> NDArray[Any]:
|
217
|
-
"""
|
218
|
-
Retrieves the inverse power curve coefficients for the line of best fit.
|
219
|
-
Global minimization is done via basin hopping. More info on this algorithm
|
220
|
-
can be found here: https://arxiv.org/abs/cond-mat/9803344 .
|
221
|
-
|
222
|
-
Parameters
|
223
|
-
----------
|
224
|
-
p_i : NDArray
|
225
|
-
Array of corresponding losses
|
226
|
-
n_i : NDArray
|
227
|
-
Array of sample sizes
|
228
|
-
niter : int
|
229
|
-
Number of iterations to perform in the basin-hopping
|
230
|
-
numerical process to curve-fit p_i
|
231
|
-
|
232
|
-
Returns
|
233
|
-
-------
|
234
|
-
NDArray
|
235
|
-
Array of parameters to recreate line of best fit
|
236
|
-
"""
|
237
|
-
|
238
|
-
def is_valid(f_new, x_new, f_old, x_old):
|
239
|
-
return f_new != np.nan
|
240
|
-
|
241
|
-
def f(x):
|
242
|
-
try:
|
243
|
-
return np.sum(np.square(p_i - f_out(n_i, x)))
|
244
|
-
except RuntimeWarning:
|
245
|
-
return np.nan
|
246
|
-
|
247
|
-
with warnings.catch_warnings():
|
248
|
-
warnings.filterwarnings("error", category=RuntimeWarning)
|
249
|
-
res = basinhopping(
|
250
|
-
f,
|
251
|
-
np.array([0.5, 0.5, 0.1]),
|
252
|
-
niter=niter,
|
253
|
-
stepsize=1.0,
|
254
|
-
minimizer_kwargs={"method": "Powell"},
|
255
|
-
accept_test=is_valid,
|
256
|
-
niter_success=200,
|
257
|
-
)
|
258
|
-
return res.x
|
12
|
+
from dataeval.outputs import SufficiencyOutput
|
13
|
+
from dataeval.outputs._base import set_metadata
|
14
|
+
from dataeval.typing import ArrayLike
|
259
15
|
|
260
16
|
|
261
17
|
def reset_parameters(model: nn.Module) -> nn.Module:
|
@@ -277,102 +33,14 @@ def reset_parameters(model: nn.Module) -> nn.Module:
|
|
277
33
|
|
278
34
|
|
279
35
|
def validate_dataset_len(dataset: Dataset[Any]) -> int:
|
280
|
-
if not
|
36
|
+
if not isinstance(dataset, Sized):
|
281
37
|
raise TypeError("Must provide a dataset with a length attribute")
|
282
|
-
length: int = dataset
|
38
|
+
length: int = len(dataset)
|
283
39
|
if length <= 0:
|
284
40
|
raise ValueError("Dataset length must be greater than 0")
|
285
41
|
return length
|
286
42
|
|
287
43
|
|
288
|
-
def project_steps(params: NDArray[Any], projection: NDArray[Any]) -> NDArray[Any]:
|
289
|
-
"""Projects the measures for each value of X
|
290
|
-
|
291
|
-
Parameters
|
292
|
-
----------
|
293
|
-
params : NDArray
|
294
|
-
Inverse power curve coefficients used to calculate projection
|
295
|
-
projection : NDArray
|
296
|
-
Steps to extrapolate
|
297
|
-
|
298
|
-
Returns
|
299
|
-
-------
|
300
|
-
NDArray
|
301
|
-
Extrapolated measure values at each projection step
|
302
|
-
|
303
|
-
"""
|
304
|
-
return 1 - f_out(projection, params)
|
305
|
-
|
306
|
-
|
307
|
-
def inv_project_steps(params: NDArray[Any], targets: NDArray[Any]) -> NDArray[np.uint64]:
|
308
|
-
"""Inverse function for project_steps()
|
309
|
-
|
310
|
-
Parameters
|
311
|
-
----------
|
312
|
-
params : NDArray
|
313
|
-
Inverse power curve coefficients used to calculate projection
|
314
|
-
targets : NDArray
|
315
|
-
Desired measure values
|
316
|
-
|
317
|
-
Returns
|
318
|
-
-------
|
319
|
-
NDArray
|
320
|
-
Array of sample sizes, or 0 if overflow
|
321
|
-
"""
|
322
|
-
steps = f_inv_out(1 - np.array(targets), params)
|
323
|
-
steps[np.isnan(steps)] = 0
|
324
|
-
return np.ceil(steps)
|
325
|
-
|
326
|
-
|
327
|
-
def get_curve_params(measures: dict[str, NDArray[Any]], ranges: NDArray[Any], niter: int) -> dict[str, NDArray[Any]]:
|
328
|
-
"""Calculates and aggregates parameters for both single and multi-class metrics"""
|
329
|
-
output = {}
|
330
|
-
for name, measure in measures.items():
|
331
|
-
measure = cast(np.ndarray, measure)
|
332
|
-
if measure.ndim > 1:
|
333
|
-
result = []
|
334
|
-
for value in measure:
|
335
|
-
result.append(calc_params(1 - value, ranges, niter))
|
336
|
-
output[name] = np.array(result)
|
337
|
-
else:
|
338
|
-
output[name] = calc_params(1 - measure, ranges, niter)
|
339
|
-
return output
|
340
|
-
|
341
|
-
|
342
|
-
def plot_measure(
|
343
|
-
name: str,
|
344
|
-
steps: NDArray[Any],
|
345
|
-
measure: NDArray[Any],
|
346
|
-
params: NDArray[Any],
|
347
|
-
projection: NDArray[Any],
|
348
|
-
) -> Figure:
|
349
|
-
import matplotlib.pyplot
|
350
|
-
|
351
|
-
fig = matplotlib.pyplot.figure()
|
352
|
-
fig = cast(Figure, fig)
|
353
|
-
fig.tight_layout()
|
354
|
-
|
355
|
-
ax = fig.add_subplot(111)
|
356
|
-
|
357
|
-
ax.set_title(f"{name} Sufficiency")
|
358
|
-
ax.set_ylabel(f"{name}")
|
359
|
-
ax.set_xlabel("Steps")
|
360
|
-
|
361
|
-
# Plot measure over each step
|
362
|
-
ax.scatter(steps, measure, label=f"Model Results ({name})", s=15, c="black")
|
363
|
-
|
364
|
-
# Plot extrapolation
|
365
|
-
ax.plot(
|
366
|
-
projection,
|
367
|
-
project_steps(params, projection),
|
368
|
-
linestyle="dashed",
|
369
|
-
label=f"Potential Model Results ({name})",
|
370
|
-
)
|
371
|
-
|
372
|
-
ax.legend()
|
373
|
-
return fig
|
374
|
-
|
375
|
-
|
376
44
|
T = TypeVar("T")
|
377
45
|
|
378
46
|
|
@@ -460,13 +128,13 @@ class Sufficiency(Generic[T]):
|
|
460
128
|
@property
|
461
129
|
def eval_fn(
|
462
130
|
self,
|
463
|
-
) -> Callable[[nn.Module, Dataset[T]],
|
131
|
+
) -> Callable[[nn.Module, Dataset[T]], Mapping[str, float] | Mapping[str, ArrayLike]]:
|
464
132
|
return self._eval_fn
|
465
133
|
|
466
134
|
@eval_fn.setter
|
467
135
|
def eval_fn(
|
468
136
|
self,
|
469
|
-
value: Callable[[nn.Module, Dataset[T]],
|
137
|
+
value: Callable[[nn.Module, Dataset[T]], Mapping[str, float] | Mapping[str, ArrayLike]],
|
470
138
|
) -> None:
|
471
139
|
if not callable(value):
|
472
140
|
raise TypeError("Must provide a callable for eval_fn.")
|
@@ -489,7 +157,7 @@ class Sufficiency(Generic[T]):
|
|
489
157
|
self._eval_kwargs = {} if value is None else value
|
490
158
|
|
491
159
|
@set_metadata(state=["runs", "substeps"])
|
492
|
-
def evaluate(self, eval_at: int | Iterable[int] | None = None
|
160
|
+
def evaluate(self, eval_at: int | Iterable[int] | None = None) -> SufficiencyOutput:
|
493
161
|
"""
|
494
162
|
Creates data indices, trains models, and returns plotting data
|
495
163
|
|
@@ -498,8 +166,6 @@ class Sufficiency(Generic[T]):
|
|
498
166
|
eval_at : int | Iterable[int] | None, default None
|
499
167
|
Specify this to collect accuracies over a specific set of dataset lengths, rather
|
500
168
|
than letting :term:`sufficiency<Sufficiency>` internally create the lengths to evaluate at.
|
501
|
-
niter : int, default 1000
|
502
|
-
Iterations to perform when using the basin-hopping method to curve-fit measure(s).
|
503
169
|
|
504
170
|
Returns
|
505
171
|
-------
|
@@ -523,7 +189,7 @@ class Sufficiency(Generic[T]):
|
|
523
189
|
... substeps=5,
|
524
190
|
... )
|
525
191
|
>>> suff.evaluate()
|
526
|
-
SufficiencyOutput(steps=array([ 1, 3, 10, 31, 100], dtype=uint32),
|
192
|
+
SufficiencyOutput(steps=array([ 1, 3, 10, 31, 100], dtype=uint32), measures={'test': array([1., 1., 1., 1., 1.])}, n_iter=1000)
|
527
193
|
""" # noqa: E501
|
528
194
|
if eval_at is not None:
|
529
195
|
ranges = np.asarray(list(eval_at) if isinstance(eval_at, Iterable) else [eval_at])
|
@@ -568,5 +234,4 @@ class Sufficiency(Generic[T]):
|
|
568
234
|
|
569
235
|
# The mean for each measure must be calculated before being returned
|
570
236
|
measures = {k: (v / self.runs).T for k, v in measures.items()}
|
571
|
-
|
572
|
-
return SufficiencyOutput(ranges, params_output, measures)
|
237
|
+
return SufficiencyOutput(ranges, measures)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: dataeval
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.82.0
|
4
4
|
Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
|
5
5
|
Home-page: https://dataeval.ai/
|
6
6
|
License: MIT
|
@@ -21,7 +21,10 @@ Classifier: Programming Language :: Python :: 3.12
|
|
21
21
|
Classifier: Programming Language :: Python :: 3 :: Only
|
22
22
|
Classifier: Topic :: Scientific/Engineering
|
23
23
|
Provides-Extra: all
|
24
|
+
Requires-Dist: defusedxml (>=0.7.1)
|
25
|
+
Requires-Dist: fast_hdbscan (==0.2.0)
|
24
26
|
Requires-Dist: matplotlib (>=3.7.1) ; extra == "all"
|
27
|
+
Requires-Dist: numba (>=0.59.1)
|
25
28
|
Requires-Dist: numpy (>=1.24.2)
|
26
29
|
Requires-Dist: pandas (>=2.0) ; extra == "all"
|
27
30
|
Requires-Dist: pillow (>=10.3.0)
|
@@ -71,7 +74,7 @@ DataEval is easy to install, supports a wide range of Python versions, and is
|
|
71
74
|
compatible with many of the most popular packages in the scientific and T&E
|
72
75
|
communities.
|
73
76
|
|
74
|
-
DataEval also has native
|
77
|
+
DataEval also has native interoperability between JATIC's suite of tools when
|
75
78
|
using MAITE-compliant datasets and models.
|
76
79
|
<!-- end JATIC interop -->
|
77
80
|
|