dataeval 0.63.0__py3-none-any.whl → 0.65.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataeval/__init__.py +4 -4
- dataeval/_internal/detectors/clusterer.py +47 -34
- dataeval/_internal/detectors/drift/base.py +53 -35
- dataeval/_internal/detectors/drift/cvm.py +5 -4
- dataeval/_internal/detectors/drift/ks.py +7 -6
- dataeval/_internal/detectors/drift/mmd.py +39 -19
- dataeval/_internal/detectors/drift/torch.py +6 -5
- dataeval/_internal/detectors/drift/uncertainty.py +7 -8
- dataeval/_internal/detectors/duplicates.py +57 -30
- dataeval/_internal/detectors/linter.py +40 -24
- dataeval/_internal/detectors/ood/ae.py +2 -1
- dataeval/_internal/detectors/ood/aegmm.py +2 -1
- dataeval/_internal/detectors/ood/base.py +37 -15
- dataeval/_internal/detectors/ood/llr.py +9 -8
- dataeval/_internal/detectors/ood/vae.py +2 -1
- dataeval/_internal/detectors/ood/vaegmm.py +2 -1
- dataeval/_internal/flags.py +42 -21
- dataeval/_internal/interop.py +3 -12
- dataeval/_internal/metrics/balance.py +188 -0
- dataeval/_internal/metrics/ber.py +123 -48
- dataeval/_internal/metrics/coverage.py +90 -74
- dataeval/_internal/metrics/divergence.py +101 -67
- dataeval/_internal/metrics/diversity.py +211 -0
- dataeval/_internal/metrics/parity.py +287 -155
- dataeval/_internal/metrics/stats.py +198 -317
- dataeval/_internal/metrics/uap.py +40 -29
- dataeval/_internal/metrics/utils.py +430 -0
- dataeval/_internal/models/tensorflow/losses.py +3 -3
- dataeval/_internal/models/tensorflow/trainer.py +3 -2
- dataeval/_internal/models/tensorflow/utils.py +4 -3
- dataeval/_internal/output.py +82 -0
- dataeval/_internal/utils.py +64 -0
- dataeval/_internal/workflows/sufficiency.py +96 -107
- dataeval/flags/__init__.py +2 -2
- dataeval/metrics/__init__.py +26 -7
- dataeval/utils/__init__.py +9 -0
- {dataeval-0.63.0.dist-info → dataeval-0.65.0.dist-info}/METADATA +1 -1
- dataeval-0.65.0.dist-info/RECORD +60 -0
- dataeval/_internal/functional/__init__.py +0 -0
- dataeval/_internal/functional/ber.py +0 -63
- dataeval/_internal/functional/coverage.py +0 -75
- dataeval/_internal/functional/divergence.py +0 -16
- dataeval/_internal/functional/hash.py +0 -79
- dataeval/_internal/functional/metadata.py +0 -136
- dataeval/_internal/functional/metadataparity.py +0 -190
- dataeval/_internal/functional/uap.py +0 -6
- dataeval/_internal/functional/utils.py +0 -158
- dataeval/_internal/maite/__init__.py +0 -0
- dataeval/_internal/maite/utils.py +0 -30
- dataeval/_internal/metrics/base.py +0 -92
- dataeval/_internal/metrics/metadata.py +0 -610
- dataeval/_internal/metrics/metadataparity.py +0 -67
- dataeval-0.63.0.dist-info/RECORD +0 -68
- {dataeval-0.63.0.dist-info → dataeval-0.65.0.dist-info}/LICENSE.txt +0 -0
- {dataeval-0.63.0.dist-info → dataeval-0.65.0.dist-info}/WHEEL +0 -0
@@ -1,4 +1,5 @@
|
|
1
1
|
import warnings
|
2
|
+
from dataclasses import dataclass
|
2
3
|
from typing import Any, Callable, Dict, List, Optional, Sequence, Union, cast
|
3
4
|
|
4
5
|
import matplotlib.pyplot as plt
|
@@ -6,57 +7,80 @@ import numpy as np
|
|
6
7
|
import torch
|
7
8
|
import torch.nn as nn
|
8
9
|
from matplotlib.figure import Figure
|
10
|
+
from numpy.typing import NDArray
|
9
11
|
from scipy.optimize import basinhopping
|
10
12
|
from torch.utils.data import Dataset
|
11
13
|
|
12
|
-
from dataeval._internal.
|
14
|
+
from dataeval._internal.output import OutputMetadata, set_metadata
|
13
15
|
|
14
|
-
STEPS_KEY = "_STEPS_"
|
15
|
-
PARAMS_KEY = "_CURVE_PARAMS_"
|
16
16
|
|
17
|
-
|
17
|
+
@dataclass(frozen=True)
|
18
|
+
class SufficiencyOutput(OutputMetadata):
|
19
|
+
"""
|
20
|
+
Attributes
|
21
|
+
----------
|
22
|
+
steps : NDArray[np.uint32]
|
23
|
+
Array of sample sizes
|
24
|
+
params : Dict[str, NDArray[np.float64]]
|
25
|
+
Inverse power curve coefficients for the line of best fit for each measure
|
26
|
+
measures : Dict[str, NDArray[np.float64]]
|
27
|
+
Average of values observed for each sample size step for each measure
|
28
|
+
"""
|
29
|
+
|
30
|
+
steps: NDArray[np.uint32]
|
31
|
+
params: Dict[str, NDArray[np.float64]]
|
32
|
+
measures: Dict[str, NDArray[np.float64]]
|
18
33
|
|
34
|
+
def __post_init__(self):
|
35
|
+
c = len(self.steps)
|
36
|
+
if set(self.params) != set(self.measures):
|
37
|
+
raise ValueError("params and measures have a key mismatch")
|
38
|
+
for m, v in self.measures.items():
|
39
|
+
c_v = v.shape[1] if v.ndim > 1 else len(v)
|
40
|
+
if c != c_v:
|
41
|
+
raise ValueError(f"{m} does not contain the expected number ({c}) of data points.")
|
19
42
|
|
20
|
-
|
43
|
+
|
44
|
+
def f_out(n_i: NDArray, x: NDArray) -> NDArray:
|
21
45
|
"""
|
22
46
|
Calculates the line of best fit based on its free parameters
|
23
47
|
|
24
48
|
Parameters
|
25
49
|
----------
|
26
|
-
n_i :
|
50
|
+
n_i : NDArray
|
27
51
|
Array of sample sizes
|
28
|
-
x :
|
52
|
+
x : NDArray
|
29
53
|
Array of inverse power curve coefficients
|
30
54
|
|
31
55
|
Returns
|
32
56
|
-------
|
33
|
-
|
57
|
+
NDArray
|
34
58
|
Data points for the line of best fit
|
35
59
|
"""
|
36
60
|
return x[0] * n_i ** (-x[1]) + x[2]
|
37
61
|
|
38
62
|
|
39
|
-
def f_inv_out(y_i:
|
63
|
+
def f_inv_out(y_i: NDArray, x: NDArray) -> NDArray[np.uint64]:
|
40
64
|
"""
|
41
65
|
Inverse function for f_out()
|
42
66
|
|
43
67
|
Parameters
|
44
68
|
----------
|
45
|
-
y_i :
|
69
|
+
y_i : NDArray
|
46
70
|
Data points for the line of best fit
|
47
|
-
x :
|
71
|
+
x : NDArray
|
48
72
|
Array of inverse power curve coefficients
|
49
73
|
|
50
74
|
Returns
|
51
75
|
-------
|
52
|
-
np.
|
76
|
+
NDArray[np.uint64]
|
53
77
|
Array of sample sizes
|
54
78
|
"""
|
55
79
|
n_i = ((y_i - x[2]) / x[0]) ** (-1 / x[1])
|
56
|
-
return n_i
|
80
|
+
return np.asarray(n_i, dtype=np.uint64)
|
57
81
|
|
58
82
|
|
59
|
-
def calc_params(p_i:
|
83
|
+
def calc_params(p_i: NDArray, n_i: NDArray, niter: int) -> NDArray:
|
60
84
|
"""
|
61
85
|
Retrieves the inverse power curve coefficients for the line of best fit.
|
62
86
|
Global minimization is done via basin hopping. More info on this algorithm
|
@@ -64,9 +88,9 @@ def calc_params(p_i: np.ndarray, n_i: np.ndarray, niter: int) -> np.ndarray:
|
|
64
88
|
|
65
89
|
Parameters
|
66
90
|
----------
|
67
|
-
p_i :
|
91
|
+
p_i : NDArray
|
68
92
|
Array of corresponding losses
|
69
|
-
n_i :
|
93
|
+
n_i : NDArray
|
70
94
|
Array of sample sizes
|
71
95
|
niter : int
|
72
96
|
Number of iterations to perform in the basin-hopping
|
@@ -74,7 +98,7 @@ def calc_params(p_i: np.ndarray, n_i: np.ndarray, niter: int) -> np.ndarray:
|
|
74
98
|
|
75
99
|
Returns
|
76
100
|
-------
|
77
|
-
|
101
|
+
NDArray
|
78
102
|
Array of parameters to recreate line of best fit
|
79
103
|
"""
|
80
104
|
|
@@ -128,60 +152,46 @@ def validate_dataset_len(dataset: Dataset) -> int:
|
|
128
152
|
return length
|
129
153
|
|
130
154
|
|
131
|
-
def
|
132
|
-
"""Ensure the sufficiency data used is not malformed"""
|
133
|
-
if not all(key in data for key in [STEPS_KEY, PARAMS_KEY]):
|
134
|
-
raise KeyError(f"{STEPS_KEY} and {PARAMS_KEY} are required keys for Sufficiency output.")
|
135
|
-
c = len(data[STEPS_KEY])
|
136
|
-
for m, v in data.items():
|
137
|
-
if m in [STEPS_KEY, PARAMS_KEY]:
|
138
|
-
continue
|
139
|
-
v = cast(np.ndarray, v)
|
140
|
-
c_v = v.shape[1] if v.ndim > 1 else len(v)
|
141
|
-
if c != c_v:
|
142
|
-
raise ValueError("f{m} does not contain the expected number ({c}) of data points.")
|
143
|
-
|
144
|
-
|
145
|
-
def project_steps(params: np.ndarray, projection: np.ndarray) -> np.ndarray:
|
155
|
+
def project_steps(params: NDArray, projection: NDArray) -> NDArray:
|
146
156
|
"""Projects the measures for each value of X
|
147
157
|
|
148
158
|
Parameters
|
149
159
|
----------
|
150
|
-
params :
|
160
|
+
params : NDArray
|
151
161
|
Inverse power curve coefficients used to calculate projection
|
152
|
-
projection :
|
162
|
+
projection : NDArray
|
153
163
|
Steps to extrapolate
|
154
164
|
|
155
165
|
Returns
|
156
166
|
-------
|
157
|
-
|
167
|
+
NDArray
|
158
168
|
Extrapolated measure values at each projection step
|
159
169
|
|
160
170
|
"""
|
161
171
|
return 1 - f_out(projection, params)
|
162
172
|
|
163
173
|
|
164
|
-
def inv_project_steps(params:
|
174
|
+
def inv_project_steps(params: NDArray, targets: NDArray) -> NDArray[np.uint64]:
|
165
175
|
"""Inverse function for project_steps()
|
166
176
|
|
167
177
|
Parameters
|
168
178
|
----------
|
169
|
-
params :
|
179
|
+
params : NDArray
|
170
180
|
Inverse power curve coefficients used to calculate projection
|
171
|
-
targets :
|
181
|
+
targets : NDArray
|
172
182
|
Desired measure values
|
173
183
|
|
174
184
|
Returns
|
175
185
|
-------
|
176
|
-
np.
|
186
|
+
NDArray[np.uint64]
|
177
187
|
Array of sample sizes, or 0 if overflow
|
178
188
|
"""
|
179
189
|
steps = f_inv_out(1 - np.array(targets), params)
|
180
190
|
steps[np.isnan(steps)] = 0
|
181
|
-
return np.ceil(steps)
|
191
|
+
return np.ceil(steps)
|
182
192
|
|
183
193
|
|
184
|
-
def get_curve_params(measures: Dict[str,
|
194
|
+
def get_curve_params(measures: Dict[str, NDArray], ranges: NDArray, niter: int) -> Dict[str, NDArray]:
|
185
195
|
"""Calculates and aggregates parameters for both single and multi-class metrics"""
|
186
196
|
output = {}
|
187
197
|
for name, measure in measures.items():
|
@@ -198,10 +208,10 @@ def get_curve_params(measures: Dict[str, np.ndarray], ranges: np.ndarray, niter:
|
|
198
208
|
|
199
209
|
def plot_measure(
|
200
210
|
name: str,
|
201
|
-
steps:
|
202
|
-
measure:
|
203
|
-
params:
|
204
|
-
projection:
|
211
|
+
steps: NDArray,
|
212
|
+
measure: NDArray,
|
213
|
+
params: NDArray,
|
214
|
+
projection: NDArray,
|
205
215
|
) -> Figure:
|
206
216
|
fig = plt.figure()
|
207
217
|
fig = cast(Figure, fig)
|
@@ -228,7 +238,7 @@ def plot_measure(
|
|
228
238
|
return fig
|
229
239
|
|
230
240
|
|
231
|
-
class Sufficiency
|
241
|
+
class Sufficiency:
|
232
242
|
"""
|
233
243
|
Project dataset sufficiency using given a model and evaluation criteria
|
234
244
|
|
@@ -265,7 +275,7 @@ class Sufficiency(EvaluateMixin):
|
|
265
275
|
train_ds: Dataset,
|
266
276
|
test_ds: Dataset,
|
267
277
|
train_fn: Callable[[nn.Module, Dataset, Sequence[int]], None],
|
268
|
-
eval_fn: Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str,
|
278
|
+
eval_fn: Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, NDArray]]],
|
269
279
|
runs: int = 1,
|
270
280
|
substeps: int = 5,
|
271
281
|
train_kwargs: Optional[Dict[str, Any]] = None,
|
@@ -312,13 +322,13 @@ class Sufficiency(EvaluateMixin):
|
|
312
322
|
@property
|
313
323
|
def eval_fn(
|
314
324
|
self,
|
315
|
-
) -> Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str,
|
325
|
+
) -> Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, NDArray]]]:
|
316
326
|
return self._eval_fn
|
317
327
|
|
318
328
|
@eval_fn.setter
|
319
329
|
def eval_fn(
|
320
330
|
self,
|
321
|
-
value: Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str,
|
331
|
+
value: Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, NDArray]]],
|
322
332
|
):
|
323
333
|
if not callable(value):
|
324
334
|
raise TypeError("Must provide a callable for eval_fn.")
|
@@ -340,13 +350,14 @@ class Sufficiency(EvaluateMixin):
|
|
340
350
|
def eval_kwargs(self, value: Optional[Dict[str, Any]]):
|
341
351
|
self._eval_kwargs = {} if value is None else value
|
342
352
|
|
343
|
-
|
353
|
+
@set_metadata("dataeval.workflows", ["runs", "substeps"])
|
354
|
+
def evaluate(self, eval_at: Optional[NDArray] = None, niter: int = 1000) -> SufficiencyOutput:
|
344
355
|
"""
|
345
356
|
Creates data indices, trains models, and returns plotting data
|
346
357
|
|
347
358
|
Parameters
|
348
359
|
----------
|
349
|
-
eval_at : Optional[
|
360
|
+
eval_at : Optional[NDArray]
|
350
361
|
Specify this to collect accuracies over a specific set of dataset lengths, rather
|
351
362
|
than letting Sufficiency internally create the lengths to evaluate at.
|
352
363
|
niter : int, default 1000
|
@@ -354,8 +365,8 @@ class Sufficiency(EvaluateMixin):
|
|
354
365
|
|
355
366
|
Returns
|
356
367
|
-------
|
357
|
-
|
358
|
-
|
368
|
+
SufficiencyOutput
|
369
|
+
Dataclass containing the average of each measure per substep
|
359
370
|
"""
|
360
371
|
if eval_at is not None:
|
361
372
|
ranges = eval_at
|
@@ -365,7 +376,7 @@ class Sufficiency(EvaluateMixin):
|
|
365
376
|
self._length,
|
366
377
|
self.substeps,
|
367
378
|
) # Start, Stop, Num steps
|
368
|
-
ranges = np.geomspace(*geomshape
|
379
|
+
ranges = np.geomspace(*geomshape, dtype=np.uint32)
|
369
380
|
substeps = len(ranges)
|
370
381
|
measures = {}
|
371
382
|
|
@@ -381,7 +392,7 @@ class Sufficiency(EvaluateMixin):
|
|
381
392
|
self.train_fn(
|
382
393
|
model,
|
383
394
|
self.train_ds,
|
384
|
-
indices[:substep].tolist(),
|
395
|
+
indices[: int(substep)].tolist(),
|
385
396
|
**self.train_kwargs,
|
386
397
|
)
|
387
398
|
|
@@ -390,9 +401,6 @@ class Sufficiency(EvaluateMixin):
|
|
390
401
|
|
391
402
|
# Keep track of each measures values
|
392
403
|
for name, value in measure.items():
|
393
|
-
if name in [STEPS_KEY, PARAMS_KEY]:
|
394
|
-
raise KeyError(f"Cannot use reserved name '{name}' as a metric name.")
|
395
|
-
|
396
404
|
# Sum result into current substep iteration to be averaged later
|
397
405
|
value = np.array(value).ravel()
|
398
406
|
if name not in measures:
|
@@ -402,57 +410,50 @@ class Sufficiency(EvaluateMixin):
|
|
402
410
|
# The mean for each measure must be calculated before being returned
|
403
411
|
measures = {k: (v / self.runs).T for k, v in measures.items()}
|
404
412
|
params_output = get_curve_params(measures, ranges, niter)
|
405
|
-
|
406
|
-
output.update(measures)
|
407
|
-
return output
|
413
|
+
return SufficiencyOutput(ranges, params_output, measures)
|
408
414
|
|
409
415
|
@classmethod
|
410
416
|
def project(
|
411
417
|
cls,
|
412
418
|
data: SufficiencyOutput,
|
413
|
-
projection: Union[int, Sequence[int], np.
|
414
|
-
) ->
|
419
|
+
projection: Union[int, Sequence[int], NDArray[np.uint]],
|
420
|
+
) -> SufficiencyOutput:
|
415
421
|
"""Projects the measures for each value of X
|
416
422
|
|
417
423
|
Parameters
|
418
424
|
----------
|
419
|
-
data :
|
425
|
+
data : SufficiencyOutput
|
420
426
|
Dataclass containing the average of each measure per substep
|
421
|
-
|
427
|
+
projection : Union[int, Sequence[int], NDArray[np.uint]]
|
422
428
|
Step or steps to project
|
423
|
-
|
424
|
-
|
425
|
-
|
429
|
+
|
430
|
+
Returns
|
431
|
+
-------
|
432
|
+
SufficiencyOutput
|
433
|
+
Dataclass containing the projected measures per projection
|
426
434
|
|
427
435
|
Raises
|
428
436
|
------
|
429
|
-
KeyError
|
430
|
-
If STEPS_KEY or measure is not a valid key
|
431
437
|
ValueError
|
432
438
|
If the length of data points in the measures do not match
|
433
439
|
If the steps are not int, Sequence[int] or an ndarray
|
434
440
|
"""
|
435
|
-
validate_output(data)
|
436
441
|
projection = [projection] if isinstance(projection, int) else projection
|
437
442
|
projection = np.array(projection) if isinstance(projection, Sequence) else projection
|
438
443
|
if not isinstance(projection, np.ndarray):
|
439
444
|
raise ValueError("'steps' must be an int, Sequence[int] or ndarray")
|
440
445
|
|
441
446
|
output = {}
|
442
|
-
|
443
|
-
for name, measures in data.items():
|
444
|
-
if name in [STEPS_KEY, PARAMS_KEY]:
|
445
|
-
continue
|
446
|
-
measures = cast(np.ndarray, measures)
|
447
|
+
for name, measures in data.measures.items():
|
447
448
|
if measures.ndim > 1:
|
448
449
|
result = []
|
449
450
|
for i in range(len(measures)):
|
450
|
-
projected = project_steps(data[
|
451
|
+
projected = project_steps(data.params[name][i], projection)
|
451
452
|
result.append(projected)
|
452
|
-
output[name] = np.array(result)
|
453
|
+
output[name] = np.array(result)
|
453
454
|
else:
|
454
|
-
output[name] = project_steps(data[
|
455
|
-
return output
|
455
|
+
output[name] = project_steps(data.params[name], projection)
|
456
|
+
return SufficiencyOutput(projection, data.params, output)
|
456
457
|
|
457
458
|
@classmethod
|
458
459
|
def plot(cls, data: SufficiencyOutput, class_names: Optional[Sequence[str]] = None) -> List[Figure]:
|
@@ -460,7 +461,7 @@ class Sufficiency(EvaluateMixin):
|
|
460
461
|
|
461
462
|
Parameters
|
462
463
|
----------
|
463
|
-
data :
|
464
|
+
data : SufficiencyOutput
|
464
465
|
Dataclass containing the average of each measure per substep
|
465
466
|
|
466
467
|
Returns
|
@@ -470,29 +471,19 @@ class Sufficiency(EvaluateMixin):
|
|
470
471
|
|
471
472
|
Raises
|
472
473
|
------
|
473
|
-
KeyError
|
474
|
-
If STEPS_KEY or measure is not a valid key
|
475
474
|
ValueError
|
476
475
|
If the length of data points in the measures do not match
|
477
476
|
"""
|
478
|
-
validate_output(data)
|
479
|
-
|
480
|
-
# X, y data
|
481
|
-
steps = cast(np.ndarray, data[STEPS_KEY])
|
482
|
-
|
483
477
|
# Extrapolation parameters
|
484
|
-
last_X = steps[-1]
|
485
|
-
geomshape = (0.01 * last_X, last_X * 4, len(steps))
|
478
|
+
last_X = data.steps[-1]
|
479
|
+
geomshape = (0.01 * last_X, last_X * 4, len(data.steps))
|
486
480
|
extrapolated = np.geomspace(*geomshape).astype(np.int64)
|
487
481
|
|
488
482
|
# Stores all plots
|
489
483
|
plots = []
|
490
484
|
|
491
485
|
# Create a plot for each measure on one figure
|
492
|
-
for name, measures in data.items():
|
493
|
-
if name in [STEPS_KEY, PARAMS_KEY]:
|
494
|
-
continue
|
495
|
-
measures = cast(np.ndarray, measures)
|
486
|
+
for name, measures in data.measures.items():
|
496
487
|
if measures.ndim > 1:
|
497
488
|
if class_names is not None and len(measures) != len(class_names):
|
498
489
|
raise IndexError("Class name count does not align with measures")
|
@@ -500,56 +491,54 @@ class Sufficiency(EvaluateMixin):
|
|
500
491
|
class_name = str(i) if class_names is None else class_names[i]
|
501
492
|
fig = plot_measure(
|
502
493
|
f"{name}_{class_name}",
|
503
|
-
steps,
|
494
|
+
data.steps,
|
504
495
|
measure,
|
505
|
-
data[
|
496
|
+
data.params[name][i],
|
506
497
|
extrapolated,
|
507
498
|
)
|
508
499
|
plots.append(fig)
|
509
500
|
|
510
501
|
else:
|
511
|
-
fig = plot_measure(name, steps, measures, data[
|
502
|
+
fig = plot_measure(name, data.steps, measures, data.params[name], extrapolated)
|
512
503
|
plots.append(fig)
|
513
504
|
|
514
505
|
return plots
|
515
506
|
|
516
507
|
@classmethod
|
517
|
-
def inv_project(cls, targets: Dict[str,
|
508
|
+
def inv_project(cls, targets: Dict[str, NDArray], data: SufficiencyOutput) -> Dict[str, NDArray]:
|
518
509
|
"""
|
519
510
|
Calculate training samples needed to achieve target model metric values.
|
520
511
|
|
521
512
|
Parameters
|
522
513
|
----------
|
523
|
-
targets : Dict[str,
|
514
|
+
targets : Dict[str, NDArray]
|
524
515
|
Dictionary of target metric scores (from 0.0 to 1.0) that we want
|
525
516
|
to achieve, where the key is the name of the metric.
|
526
517
|
|
527
|
-
data :
|
518
|
+
data : SufficiencyOutput
|
528
519
|
Dataclass containing the average of each measure per substep
|
529
520
|
|
530
521
|
Returns
|
531
522
|
-------
|
532
|
-
Dict[str,
|
523
|
+
Dict[str, NDArray]
|
533
524
|
List of the number of training samples needed to achieve each
|
534
525
|
corresponding entry in targets
|
535
526
|
"""
|
536
527
|
|
537
|
-
validate_output(data)
|
538
|
-
|
539
528
|
projection = {}
|
540
529
|
|
541
530
|
for name, target in targets.items():
|
542
|
-
if name not in data:
|
531
|
+
if name not in data.measures:
|
543
532
|
continue
|
544
533
|
|
545
|
-
measure =
|
534
|
+
measure = data.measures[name]
|
546
535
|
if measure.ndim > 1:
|
547
536
|
projection[name] = np.zeros((len(measure), len(target)))
|
548
537
|
for i in range(len(measure)):
|
549
538
|
projection[name][i] = inv_project_steps(
|
550
|
-
data[
|
539
|
+
data.params[name][i], target[i] if target.ndim == measure.ndim else target
|
551
540
|
)
|
552
541
|
else:
|
553
|
-
projection[name] = inv_project_steps(data[
|
542
|
+
projection[name] = inv_project_steps(data.params[name], target)
|
554
543
|
|
555
544
|
return projection
|
dataeval/flags/__init__.py
CHANGED
@@ -1,3 +1,3 @@
|
|
1
|
-
from dataeval._internal.flags import
|
1
|
+
from dataeval._internal.flags import ImageStat
|
2
2
|
|
3
|
-
__all__ = ["
|
3
|
+
__all__ = ["ImageStat"]
|
dataeval/metrics/__init__.py
CHANGED
@@ -1,8 +1,27 @@
|
|
1
|
-
from
|
2
|
-
from dataeval._internal.metrics.coverage import Coverage
|
3
|
-
from dataeval._internal.metrics.divergence import Divergence
|
4
|
-
from dataeval._internal.metrics.parity import Parity
|
5
|
-
from dataeval._internal.metrics.stats import ChannelStats, ImageStats
|
6
|
-
from dataeval._internal.metrics.uap import UAP
|
1
|
+
from typing import List
|
7
2
|
|
8
|
-
__all__ = [
|
3
|
+
__all__: List[str] = []
|
4
|
+
|
5
|
+
from dataeval._internal.metrics.balance import balance, balance_classwise
|
6
|
+
from dataeval._internal.metrics.ber import ber
|
7
|
+
from dataeval._internal.metrics.coverage import coverage
|
8
|
+
from dataeval._internal.metrics.divergence import divergence
|
9
|
+
from dataeval._internal.metrics.diversity import diversity, diversity_classwise
|
10
|
+
from dataeval._internal.metrics.parity import parity, parity_metadata
|
11
|
+
from dataeval._internal.metrics.stats import channelstats, imagestats
|
12
|
+
from dataeval._internal.metrics.uap import uap
|
13
|
+
|
14
|
+
__all__ += [
|
15
|
+
"balance",
|
16
|
+
"balance_classwise",
|
17
|
+
"ber",
|
18
|
+
"channelstats",
|
19
|
+
"coverage",
|
20
|
+
"divergence",
|
21
|
+
"diversity",
|
22
|
+
"diversity_classwise",
|
23
|
+
"imagestats",
|
24
|
+
"parity",
|
25
|
+
"parity_metadata",
|
26
|
+
"uap",
|
27
|
+
]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: dataeval
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.65.0
|
4
4
|
Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
|
5
5
|
Home-page: https://dataeval.ai/
|
6
6
|
License: MIT
|
@@ -0,0 +1,60 @@
|
|
1
|
+
dataeval/__init__.py,sha256=Uok47bAn3XhZppFB7u2BAVel5MDFpXC-1fEFlrWBIi8,424
|
2
|
+
dataeval/_internal/detectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
|
+
dataeval/_internal/detectors/clusterer.py,sha256=QHME6JQBqQe0xgEDuOGav6EDZFFq1hNiVde-DfUdNKU,20697
|
4
|
+
dataeval/_internal/detectors/drift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
|
+
dataeval/_internal/detectors/drift/base.py,sha256=0Vs69uctnC17pLhJ53rhDsQ8DLlZAj06LDCVGJhsMYo,9298
|
6
|
+
dataeval/_internal/detectors/drift/cvm.py,sha256=5tmD6uBumJgBS3u6e46Az13ytWSIScAAImvPHNk4PDA,4052
|
7
|
+
dataeval/_internal/detectors/drift/ks.py,sha256=FKHbJ4GR3kkADjPkf2CrjumIILEvuVUynjrOAFsK5no,4046
|
8
|
+
dataeval/_internal/detectors/drift/mmd.py,sha256=n5Z2tvpX41J5UPEE1JsJVBnVhGwZp3ADmLXEqFyXiek,7653
|
9
|
+
dataeval/_internal/detectors/drift/torch.py,sha256=zd2PcvSQ7j0rLwq8CvGpF9o8v5cVYZJFGv4jetU2PDo,10890
|
10
|
+
dataeval/_internal/detectors/drift/uncertainty.py,sha256=TZ1JIoZ9HRnrHupfPHvGT1P0gR-hYdH9bsigzJr_QcE,5311
|
11
|
+
dataeval/_internal/detectors/duplicates.py,sha256=VjunPZ9ffhVNFda9OIvnfE7QCvdA7OOfQMWEzDn2mdU,2973
|
12
|
+
dataeval/_internal/detectors/linter.py,sha256=gC-KDajxyNyBtCphtC2NULjeXmFx-d15GKF9I-MNOdI,5866
|
13
|
+
dataeval/_internal/detectors/ood/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
14
|
+
dataeval/_internal/detectors/ood/ae.py,sha256=FjqMucicFsDIKJMAOjWpKBohrPM4F1ubFLJk91GVqio,2681
|
15
|
+
dataeval/_internal/detectors/ood/aegmm.py,sha256=Kf9R5q-hoRg6RUHlJG-2oo52ZKeQmJQbxG0kFtHh6zA,2416
|
16
|
+
dataeval/_internal/detectors/ood/base.py,sha256=ybY4DJeQrzY62tEFnaFnAUL0ILkulOes63aA09jKynw,7633
|
17
|
+
dataeval/_internal/detectors/ood/llr.py,sha256=Amj3MFmoE0wi60qsHF7qMGRzO3ZeUW4ywUcLppx1ZQw,10160
|
18
|
+
dataeval/_internal/detectors/ood/vae.py,sha256=ntabTTTmPhJ18giZ7A64mxpJvTH9pIHmHPGGnu-gA8g,2987
|
19
|
+
dataeval/_internal/detectors/ood/vaegmm.py,sha256=opBfFLuXEAIMa8E6scwf-GWbZbuXnsqXlXTbLN4MoYg,2861
|
20
|
+
dataeval/_internal/flags.py,sha256=BKRYNvANpleVb1DGWqZASl_CB-BOa4UnamFY2So53Cc,2152
|
21
|
+
dataeval/_internal/interop.py,sha256=kX6lsEb7xoMD3iRG8GUp4uLbCCoC4Rb33PS6CHUl7BU,1036
|
22
|
+
dataeval/_internal/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
23
|
+
dataeval/_internal/metrics/balance.py,sha256=gD0Vm6Y55v6byfCJQi1c6cj9YzsDfCsHsEmFu8OrR1E,6758
|
24
|
+
dataeval/_internal/metrics/ber.py,sha256=bz7l3RqM54mYUwA8pqk9gq5GMFeRCNMD-65Prm9BV3w,4628
|
25
|
+
dataeval/_internal/metrics/coverage.py,sha256=9ZvcNjItE9rEyA2UHPE1K9zpTbbib4xqk8WpPpDN8ok,4037
|
26
|
+
dataeval/_internal/metrics/divergence.py,sha256=nmMUfr9FGnH798eb6xzEiMj4C42rQVthh5HeexiY6EE,4119
|
27
|
+
dataeval/_internal/metrics/diversity.py,sha256=dAgKaqY1J5cHy0JPUIN3kuW6NyS3wUb76w0EDipQaE4,7148
|
28
|
+
dataeval/_internal/metrics/parity.py,sha256=kOKw-B0RuralYxfBx5FiwyRn9kzR0uM9n8dpA7yebBI,11463
|
29
|
+
dataeval/_internal/metrics/stats.py,sha256=Fg1HZIQ1PWX4kgakN5wOOS627_vKt40nzvlzdjh7Q-E,8752
|
30
|
+
dataeval/_internal/metrics/uap.py,sha256=jv0ATJ5t5tEnKfcbuqB7KanO1n9dsqDsn42riJpG0M4,1307
|
31
|
+
dataeval/_internal/metrics/utils.py,sha256=izd1jimtLP0L__OERITqs0ppvYLL0PTlAC3kDfh3GOE,13037
|
32
|
+
dataeval/_internal/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
33
|
+
dataeval/_internal/models/pytorch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
34
|
+
dataeval/_internal/models/pytorch/autoencoder.py,sha256=iK3Z9claesU_pJkRaiFJIZ9zKZg-Qj8ugzVYTTokDbE,6123
|
35
|
+
dataeval/_internal/models/pytorch/blocks.py,sha256=pm2xwsDZjZJYXrhhiz8husvh2vHmrkFMSYEn-EDUD5Q,1354
|
36
|
+
dataeval/_internal/models/pytorch/utils.py,sha256=Qgwym1PxGuwxbXCKUT-8r6Iyrxqm7x94oj45Vf5_CjE,1675
|
37
|
+
dataeval/_internal/models/tensorflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
38
|
+
dataeval/_internal/models/tensorflow/autoencoder.py,sha256=rErnOfDFTd7e4brSGQ2Lr1x1kNjSEHdbOREOtUfIhIM,9975
|
39
|
+
dataeval/_internal/models/tensorflow/gmm.py,sha256=wnqQKm3fURuvBROUd2fitCqzKViDo-g0-Djr3TBHZ3U,3640
|
40
|
+
dataeval/_internal/models/tensorflow/losses.py,sha256=24gDqrA-EBg9J9tkLuXYZzbkwUjjAX8c1RAtN0o72xA,3774
|
41
|
+
dataeval/_internal/models/tensorflow/pixelcnn.py,sha256=B5cwB2IGPw-7b8klt82j_60g_IvqSiDELxvbiBYJtAo,48068
|
42
|
+
dataeval/_internal/models/tensorflow/trainer.py,sha256=rHWRHrX5hMj2iNZD9HqzfhPvqLwR2g-Aw-68Vq2_U94,4117
|
43
|
+
dataeval/_internal/models/tensorflow/utils.py,sha256=G7JpmjmWlJtqqHdxnbiBRVQjAXYFTqShii5Yz_Ici9U,8603
|
44
|
+
dataeval/_internal/output.py,sha256=L-EbpcdyZR2-NvsEKJkVpHXcyr3chdydbGNqoeXl-lI,2975
|
45
|
+
dataeval/_internal/utils.py,sha256=umvc_vN5c5IR0lz2F1U2YjA3VZloKTAEp9BQx8rSk6g,1561
|
46
|
+
dataeval/_internal/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
47
|
+
dataeval/_internal/workflows/sufficiency.py,sha256=BNl11OOrAjpyAIXeVhbGbJJ_tQDwZjXRW6dnlb6LYPM,17781
|
48
|
+
dataeval/detectors/__init__.py,sha256=I2e7YWb55RRlKQll85Z6KdN5wdBa53smn-_fcZIsCwA,1507
|
49
|
+
dataeval/flags/__init__.py,sha256=qo06_Tk0ul4lOhKSEs0HE2G6WBFvMwNJq77vRX1ynww,72
|
50
|
+
dataeval/metrics/__init__.py,sha256=eGL4LMxM1pTWwsT8HI-aIA7pZ2EPGFVIa86rG2LlVXs,787
|
51
|
+
dataeval/models/__init__.py,sha256=onevPb5wznCggowBnVT0OUa8uBJXZCbrkFuek1UFvOs,293
|
52
|
+
dataeval/models/tensorflow/__init__.py,sha256=A1XRxVGHefuvh_WpaKE1x95pRD1FecuFp66iuNPA_5U,424
|
53
|
+
dataeval/models/torch/__init__.py,sha256=su7P9DF9LChlVCNHWG6d7s_yeIfWQbhCYWIkzJe0Qig,190
|
54
|
+
dataeval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
55
|
+
dataeval/utils/__init__.py,sha256=bgUXeumTEspt2Q76YyEliGrnS-_incswY-pDexPdSCc,229
|
56
|
+
dataeval/workflows/__init__.py,sha256=ObgS1cVYFRzFZWbNzGs2OcU02IVkJkAMHNnlnSNTMCE,208
|
57
|
+
dataeval-0.65.0.dist-info/LICENSE.txt,sha256=Kpzcfobf1HlqafF-EX6dQLw9TlJiaJzfgvLQFukyXYw,1060
|
58
|
+
dataeval-0.65.0.dist-info/METADATA,sha256=-bjqI2XgbnRnGoHU3qf0NLfdHYY4bW5cm5tx__ux8zU,4217
|
59
|
+
dataeval-0.65.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
60
|
+
dataeval-0.65.0.dist-info/RECORD,,
|
File without changes
|
@@ -1,63 +0,0 @@
|
|
1
|
-
from typing import Tuple
|
2
|
-
|
3
|
-
import numpy as np
|
4
|
-
from scipy.sparse import coo_matrix
|
5
|
-
from scipy.stats import mode
|
6
|
-
|
7
|
-
from dataeval._internal.functional.utils import compute_neighbors, get_classes_counts, minimum_spanning_tree
|
8
|
-
|
9
|
-
|
10
|
-
def ber_mst(X: np.ndarray, y: np.ndarray, _: int) -> Tuple[float, float]:
|
11
|
-
"""Calculates the Bayes Error Rate using a minimum spanning tree
|
12
|
-
|
13
|
-
Parameters
|
14
|
-
----------
|
15
|
-
X : np.ndarray (N, :)
|
16
|
-
Data points with arbitrary dimensionality
|
17
|
-
y : np.ndarray (N, 1)
|
18
|
-
Labels for each data point
|
19
|
-
"""
|
20
|
-
|
21
|
-
M, N = get_classes_counts(y)
|
22
|
-
|
23
|
-
tree = coo_matrix(minimum_spanning_tree(X))
|
24
|
-
matches = np.sum([y[tree.row[i]] != y[tree.col[i]] for i in range(N - 1)])
|
25
|
-
deltas = matches / (2 * N)
|
26
|
-
upper = 2 * deltas
|
27
|
-
lower = ((M - 1) / (M)) * (1 - max(1 - 2 * ((M) / (M - 1)) * deltas, 0) ** 0.5)
|
28
|
-
return upper, lower
|
29
|
-
|
30
|
-
|
31
|
-
def ber_knn(X: np.ndarray, y: np.ndarray, k: int) -> Tuple[float, float]:
|
32
|
-
"""Calculates the Bayes Error Rate using K-nearest neighbors"""
|
33
|
-
|
34
|
-
M, N = get_classes_counts(y)
|
35
|
-
|
36
|
-
# All features belong on second dimension
|
37
|
-
X = X.reshape((X.shape[0], -1))
|
38
|
-
nn_indices = compute_neighbors(X, X, k=k)
|
39
|
-
nn_indices = np.expand_dims(nn_indices, axis=1) if nn_indices.ndim == 1 else nn_indices
|
40
|
-
modal_class = mode(y[nn_indices], axis=1, keepdims=True).mode.squeeze()
|
41
|
-
upper = float(np.count_nonzero(modal_class - y) / N)
|
42
|
-
lower = _knn_lowerbound(upper, M, k)
|
43
|
-
return upper, lower
|
44
|
-
|
45
|
-
|
46
|
-
def _knn_lowerbound(value: float, classes: int, k: int) -> float:
|
47
|
-
"""Several cases for computing the BER lower bound"""
|
48
|
-
if value <= 1e-10:
|
49
|
-
return 0.0
|
50
|
-
|
51
|
-
if classes == 2 and k != 1:
|
52
|
-
if k > 5:
|
53
|
-
# Property 2 (Devroye, 1981) cited in Snoopy paper, not in snoopy repo
|
54
|
-
alpha = 0.3399
|
55
|
-
beta = 0.9749
|
56
|
-
a_k = alpha * np.sqrt(k) / (k - 3.25) * (1 + beta / (np.sqrt(k - 3)))
|
57
|
-
return value / (1 + a_k)
|
58
|
-
if k > 2:
|
59
|
-
return value / (1 + (1 / np.sqrt(k)))
|
60
|
-
# k == 2:
|
61
|
-
return value / 2
|
62
|
-
|
63
|
-
return ((classes - 1) / classes) * (1 - np.sqrt(max(0, 1 - ((classes / (classes - 1)) * value))))
|