dataeval 0.63.0__py3-none-any.whl → 0.65.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. dataeval/__init__.py +4 -4
  2. dataeval/_internal/detectors/clusterer.py +47 -34
  3. dataeval/_internal/detectors/drift/base.py +53 -35
  4. dataeval/_internal/detectors/drift/cvm.py +5 -4
  5. dataeval/_internal/detectors/drift/ks.py +7 -6
  6. dataeval/_internal/detectors/drift/mmd.py +39 -19
  7. dataeval/_internal/detectors/drift/torch.py +6 -5
  8. dataeval/_internal/detectors/drift/uncertainty.py +7 -8
  9. dataeval/_internal/detectors/duplicates.py +57 -30
  10. dataeval/_internal/detectors/linter.py +40 -24
  11. dataeval/_internal/detectors/ood/ae.py +2 -1
  12. dataeval/_internal/detectors/ood/aegmm.py +2 -1
  13. dataeval/_internal/detectors/ood/base.py +37 -15
  14. dataeval/_internal/detectors/ood/llr.py +9 -8
  15. dataeval/_internal/detectors/ood/vae.py +2 -1
  16. dataeval/_internal/detectors/ood/vaegmm.py +2 -1
  17. dataeval/_internal/flags.py +42 -21
  18. dataeval/_internal/interop.py +3 -12
  19. dataeval/_internal/metrics/balance.py +188 -0
  20. dataeval/_internal/metrics/ber.py +123 -48
  21. dataeval/_internal/metrics/coverage.py +90 -74
  22. dataeval/_internal/metrics/divergence.py +101 -67
  23. dataeval/_internal/metrics/diversity.py +211 -0
  24. dataeval/_internal/metrics/parity.py +287 -155
  25. dataeval/_internal/metrics/stats.py +198 -317
  26. dataeval/_internal/metrics/uap.py +40 -29
  27. dataeval/_internal/metrics/utils.py +430 -0
  28. dataeval/_internal/models/tensorflow/losses.py +3 -3
  29. dataeval/_internal/models/tensorflow/trainer.py +3 -2
  30. dataeval/_internal/models/tensorflow/utils.py +4 -3
  31. dataeval/_internal/output.py +82 -0
  32. dataeval/_internal/utils.py +64 -0
  33. dataeval/_internal/workflows/sufficiency.py +96 -107
  34. dataeval/flags/__init__.py +2 -2
  35. dataeval/metrics/__init__.py +26 -7
  36. dataeval/utils/__init__.py +9 -0
  37. {dataeval-0.63.0.dist-info → dataeval-0.65.0.dist-info}/METADATA +1 -1
  38. dataeval-0.65.0.dist-info/RECORD +60 -0
  39. dataeval/_internal/functional/__init__.py +0 -0
  40. dataeval/_internal/functional/ber.py +0 -63
  41. dataeval/_internal/functional/coverage.py +0 -75
  42. dataeval/_internal/functional/divergence.py +0 -16
  43. dataeval/_internal/functional/hash.py +0 -79
  44. dataeval/_internal/functional/metadata.py +0 -136
  45. dataeval/_internal/functional/metadataparity.py +0 -190
  46. dataeval/_internal/functional/uap.py +0 -6
  47. dataeval/_internal/functional/utils.py +0 -158
  48. dataeval/_internal/maite/__init__.py +0 -0
  49. dataeval/_internal/maite/utils.py +0 -30
  50. dataeval/_internal/metrics/base.py +0 -92
  51. dataeval/_internal/metrics/metadata.py +0 -610
  52. dataeval/_internal/metrics/metadataparity.py +0 -67
  53. dataeval-0.63.0.dist-info/RECORD +0 -68
  54. {dataeval-0.63.0.dist-info → dataeval-0.65.0.dist-info}/LICENSE.txt +0 -0
  55. {dataeval-0.63.0.dist-info → dataeval-0.65.0.dist-info}/WHEEL +0 -0
@@ -1,4 +1,5 @@
1
1
  import warnings
2
+ from dataclasses import dataclass
2
3
  from typing import Any, Callable, Dict, List, Optional, Sequence, Union, cast
3
4
 
4
5
  import matplotlib.pyplot as plt
@@ -6,57 +7,80 @@ import numpy as np
6
7
  import torch
7
8
  import torch.nn as nn
8
9
  from matplotlib.figure import Figure
10
+ from numpy.typing import NDArray
9
11
  from scipy.optimize import basinhopping
10
12
  from torch.utils.data import Dataset
11
13
 
12
- from dataeval._internal.metrics.base import EvaluateMixin
14
+ from dataeval._internal.output import OutputMetadata, set_metadata
13
15
 
14
- STEPS_KEY = "_STEPS_"
15
- PARAMS_KEY = "_CURVE_PARAMS_"
16
16
 
17
- SufficiencyOutput = Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
17
+ @dataclass(frozen=True)
18
+ class SufficiencyOutput(OutputMetadata):
19
+ """
20
+ Attributes
21
+ ----------
22
+ steps : NDArray[np.uint32]
23
+ Array of sample sizes
24
+ params : Dict[str, NDArray[np.float64]]
25
+ Inverse power curve coefficients for the line of best fit for each measure
26
+ measures : Dict[str, NDArray[np.float64]]
27
+ Average of values observed for each sample size step for each measure
28
+ """
29
+
30
+ steps: NDArray[np.uint32]
31
+ params: Dict[str, NDArray[np.float64]]
32
+ measures: Dict[str, NDArray[np.float64]]
18
33
 
34
+ def __post_init__(self):
35
+ c = len(self.steps)
36
+ if set(self.params) != set(self.measures):
37
+ raise ValueError("params and measures have a key mismatch")
38
+ for m, v in self.measures.items():
39
+ c_v = v.shape[1] if v.ndim > 1 else len(v)
40
+ if c != c_v:
41
+ raise ValueError(f"{m} does not contain the expected number ({c}) of data points.")
19
42
 
20
- def f_out(n_i: np.ndarray, x: np.ndarray) -> np.ndarray:
43
+
44
+ def f_out(n_i: NDArray, x: NDArray) -> NDArray:
21
45
  """
22
46
  Calculates the line of best fit based on its free parameters
23
47
 
24
48
  Parameters
25
49
  ----------
26
- n_i : np.ndarray
50
+ n_i : NDArray
27
51
  Array of sample sizes
28
- x : np.ndarray
52
+ x : NDArray
29
53
  Array of inverse power curve coefficients
30
54
 
31
55
  Returns
32
56
  -------
33
- np.ndarray
57
+ NDArray
34
58
  Data points for the line of best fit
35
59
  """
36
60
  return x[0] * n_i ** (-x[1]) + x[2]
37
61
 
38
62
 
39
- def f_inv_out(y_i: np.ndarray, x: np.ndarray) -> np.ndarray:
63
+ def f_inv_out(y_i: NDArray, x: NDArray) -> NDArray[np.uint64]:
40
64
  """
41
65
  Inverse function for f_out()
42
66
 
43
67
  Parameters
44
68
  ----------
45
- y_i : np.ndarray
69
+ y_i : NDArray
46
70
  Data points for the line of best fit
47
- x : np.ndarray
71
+ x : NDArray
48
72
  Array of inverse power curve coefficients
49
73
 
50
74
  Returns
51
75
  -------
52
- np.ndarray
76
+ NDArray[np.uint64]
53
77
  Array of sample sizes
54
78
  """
55
79
  n_i = ((y_i - x[2]) / x[0]) ** (-1 / x[1])
56
- return n_i
80
+ return np.asarray(n_i, dtype=np.uint64)
57
81
 
58
82
 
59
- def calc_params(p_i: np.ndarray, n_i: np.ndarray, niter: int) -> np.ndarray:
83
+ def calc_params(p_i: NDArray, n_i: NDArray, niter: int) -> NDArray:
60
84
  """
61
85
  Retrieves the inverse power curve coefficients for the line of best fit.
62
86
  Global minimization is done via basin hopping. More info on this algorithm
@@ -64,9 +88,9 @@ def calc_params(p_i: np.ndarray, n_i: np.ndarray, niter: int) -> np.ndarray:
64
88
 
65
89
  Parameters
66
90
  ----------
67
- p_i : np.ndarray
91
+ p_i : NDArray
68
92
  Array of corresponding losses
69
- n_i : np.ndarray
93
+ n_i : NDArray
70
94
  Array of sample sizes
71
95
  niter : int
72
96
  Number of iterations to perform in the basin-hopping
@@ -74,7 +98,7 @@ def calc_params(p_i: np.ndarray, n_i: np.ndarray, niter: int) -> np.ndarray:
74
98
 
75
99
  Returns
76
100
  -------
77
- np.ndarray
101
+ NDArray
78
102
  Array of parameters to recreate line of best fit
79
103
  """
80
104
 
@@ -128,60 +152,46 @@ def validate_dataset_len(dataset: Dataset) -> int:
128
152
  return length
129
153
 
130
154
 
131
- def validate_output(data: SufficiencyOutput):
132
- """Ensure the sufficiency data used is not malformed"""
133
- if not all(key in data for key in [STEPS_KEY, PARAMS_KEY]):
134
- raise KeyError(f"{STEPS_KEY} and {PARAMS_KEY} are required keys for Sufficiency output.")
135
- c = len(data[STEPS_KEY])
136
- for m, v in data.items():
137
- if m in [STEPS_KEY, PARAMS_KEY]:
138
- continue
139
- v = cast(np.ndarray, v)
140
- c_v = v.shape[1] if v.ndim > 1 else len(v)
141
- if c != c_v:
142
- raise ValueError("f{m} does not contain the expected number ({c}) of data points.")
143
-
144
-
145
- def project_steps(params: np.ndarray, projection: np.ndarray) -> np.ndarray:
155
+ def project_steps(params: NDArray, projection: NDArray) -> NDArray:
146
156
  """Projects the measures for each value of X
147
157
 
148
158
  Parameters
149
159
  ----------
150
- params : np.ndarray
160
+ params : NDArray
151
161
  Inverse power curve coefficients used to calculate projection
152
- projection : np.ndarray
162
+ projection : NDArray
153
163
  Steps to extrapolate
154
164
 
155
165
  Returns
156
166
  -------
157
- np.ndarray
167
+ NDArray
158
168
  Extrapolated measure values at each projection step
159
169
 
160
170
  """
161
171
  return 1 - f_out(projection, params)
162
172
 
163
173
 
164
- def inv_project_steps(params: np.ndarray, targets: np.ndarray) -> np.ndarray:
174
+ def inv_project_steps(params: NDArray, targets: NDArray) -> NDArray[np.uint64]:
165
175
  """Inverse function for project_steps()
166
176
 
167
177
  Parameters
168
178
  ----------
169
- params : np.ndarray
179
+ params : NDArray
170
180
  Inverse power curve coefficients used to calculate projection
171
- targets : np.ndarray
181
+ targets : NDArray
172
182
  Desired measure values
173
183
 
174
184
  Returns
175
185
  -------
176
- np.ndarray
186
+ NDArray[np.uint64]
177
187
  Array of sample sizes, or 0 if overflow
178
188
  """
179
189
  steps = f_inv_out(1 - np.array(targets), params)
180
190
  steps[np.isnan(steps)] = 0
181
- return np.ceil(steps).astype(np.int64)
191
+ return np.ceil(steps)
182
192
 
183
193
 
184
- def get_curve_params(measures: Dict[str, np.ndarray], ranges: np.ndarray, niter: int) -> Dict[str, np.ndarray]:
194
+ def get_curve_params(measures: Dict[str, NDArray], ranges: NDArray, niter: int) -> Dict[str, NDArray]:
185
195
  """Calculates and aggregates parameters for both single and multi-class metrics"""
186
196
  output = {}
187
197
  for name, measure in measures.items():
@@ -198,10 +208,10 @@ def get_curve_params(measures: Dict[str, np.ndarray], ranges: np.ndarray, niter:
198
208
 
199
209
  def plot_measure(
200
210
  name: str,
201
- steps: np.ndarray,
202
- measure: np.ndarray,
203
- params: np.ndarray,
204
- projection: np.ndarray,
211
+ steps: NDArray,
212
+ measure: NDArray,
213
+ params: NDArray,
214
+ projection: NDArray,
205
215
  ) -> Figure:
206
216
  fig = plt.figure()
207
217
  fig = cast(Figure, fig)
@@ -228,7 +238,7 @@ def plot_measure(
228
238
  return fig
229
239
 
230
240
 
231
- class Sufficiency(EvaluateMixin):
241
+ class Sufficiency:
232
242
  """
233
243
  Project dataset sufficiency using given a model and evaluation criteria
234
244
 
@@ -265,7 +275,7 @@ class Sufficiency(EvaluateMixin):
265
275
  train_ds: Dataset,
266
276
  test_ds: Dataset,
267
277
  train_fn: Callable[[nn.Module, Dataset, Sequence[int]], None],
268
- eval_fn: Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, np.ndarray]]],
278
+ eval_fn: Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, NDArray]]],
269
279
  runs: int = 1,
270
280
  substeps: int = 5,
271
281
  train_kwargs: Optional[Dict[str, Any]] = None,
@@ -312,13 +322,13 @@ class Sufficiency(EvaluateMixin):
312
322
  @property
313
323
  def eval_fn(
314
324
  self,
315
- ) -> Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, np.ndarray]]]:
325
+ ) -> Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, NDArray]]]:
316
326
  return self._eval_fn
317
327
 
318
328
  @eval_fn.setter
319
329
  def eval_fn(
320
330
  self,
321
- value: Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, np.ndarray]]],
331
+ value: Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, NDArray]]],
322
332
  ):
323
333
  if not callable(value):
324
334
  raise TypeError("Must provide a callable for eval_fn.")
@@ -340,13 +350,14 @@ class Sufficiency(EvaluateMixin):
340
350
  def eval_kwargs(self, value: Optional[Dict[str, Any]]):
341
351
  self._eval_kwargs = {} if value is None else value
342
352
 
343
- def evaluate(self, eval_at: Optional[np.ndarray] = None, niter: int = 1000) -> SufficiencyOutput:
353
+ @set_metadata("dataeval.workflows", ["runs", "substeps"])
354
+ def evaluate(self, eval_at: Optional[NDArray] = None, niter: int = 1000) -> SufficiencyOutput:
344
355
  """
345
356
  Creates data indices, trains models, and returns plotting data
346
357
 
347
358
  Parameters
348
359
  ----------
349
- eval_at : Optional[np.ndarray]
360
+ eval_at : Optional[NDArray]
350
361
  Specify this to collect accuracies over a specific set of dataset lengths, rather
351
362
  than letting Sufficiency internally create the lengths to evaluate at.
352
363
  niter : int, default 1000
@@ -354,8 +365,8 @@ class Sufficiency(EvaluateMixin):
354
365
 
355
366
  Returns
356
367
  -------
357
- Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
358
- Dictionary containing the average of each measure per substep
368
+ SufficiencyOutput
369
+ Dataclass containing the average of each measure per substep
359
370
  """
360
371
  if eval_at is not None:
361
372
  ranges = eval_at
@@ -365,7 +376,7 @@ class Sufficiency(EvaluateMixin):
365
376
  self._length,
366
377
  self.substeps,
367
378
  ) # Start, Stop, Num steps
368
- ranges = np.geomspace(*geomshape).astype(np.int64)
379
+ ranges = np.geomspace(*geomshape, dtype=np.uint32)
369
380
  substeps = len(ranges)
370
381
  measures = {}
371
382
 
@@ -381,7 +392,7 @@ class Sufficiency(EvaluateMixin):
381
392
  self.train_fn(
382
393
  model,
383
394
  self.train_ds,
384
- indices[:substep].tolist(),
395
+ indices[: int(substep)].tolist(),
385
396
  **self.train_kwargs,
386
397
  )
387
398
 
@@ -390,9 +401,6 @@ class Sufficiency(EvaluateMixin):
390
401
 
391
402
  # Keep track of each measures values
392
403
  for name, value in measure.items():
393
- if name in [STEPS_KEY, PARAMS_KEY]:
394
- raise KeyError(f"Cannot use reserved name '{name}' as a metric name.")
395
-
396
404
  # Sum result into current substep iteration to be averaged later
397
405
  value = np.array(value).ravel()
398
406
  if name not in measures:
@@ -402,57 +410,50 @@ class Sufficiency(EvaluateMixin):
402
410
  # The mean for each measure must be calculated before being returned
403
411
  measures = {k: (v / self.runs).T for k, v in measures.items()}
404
412
  params_output = get_curve_params(measures, ranges, niter)
405
- output = {STEPS_KEY: ranges, PARAMS_KEY: params_output}
406
- output.update(measures)
407
- return output
413
+ return SufficiencyOutput(ranges, params_output, measures)
408
414
 
409
415
  @classmethod
410
416
  def project(
411
417
  cls,
412
418
  data: SufficiencyOutput,
413
- projection: Union[int, Sequence[int], np.ndarray],
414
- ) -> Dict[str, np.ndarray]:
419
+ projection: Union[int, Sequence[int], NDArray[np.uint]],
420
+ ) -> SufficiencyOutput:
415
421
  """Projects the measures for each value of X
416
422
 
417
423
  Parameters
418
424
  ----------
419
- data : Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
425
+ data : SufficiencyOutput
420
426
  Dataclass containing the average of each measure per substep
421
- steps : Union[int, np.ndarray]
427
+ projection : Union[int, Sequence[int], NDArray[np.uint]]
422
428
  Step or steps to project
423
- niter : int, default 200
424
- Number of iterations to perform in the basin-hopping
425
- numerical process to curve-fit data
429
+
430
+ Returns
431
+ -------
432
+ SufficiencyOutput
433
+ Dataclass containing the projected measures per projection
426
434
 
427
435
  Raises
428
436
  ------
429
- KeyError
430
- If STEPS_KEY or measure is not a valid key
431
437
  ValueError
432
438
  If the length of data points in the measures do not match
433
439
  If the steps are not int, Sequence[int] or an ndarray
434
440
  """
435
- validate_output(data)
436
441
  projection = [projection] if isinstance(projection, int) else projection
437
442
  projection = np.array(projection) if isinstance(projection, Sequence) else projection
438
443
  if not isinstance(projection, np.ndarray):
439
444
  raise ValueError("'steps' must be an int, Sequence[int] or ndarray")
440
445
 
441
446
  output = {}
442
- output[STEPS_KEY] = projection
443
- for name, measures in data.items():
444
- if name in [STEPS_KEY, PARAMS_KEY]:
445
- continue
446
- measures = cast(np.ndarray, measures)
447
+ for name, measures in data.measures.items():
447
448
  if measures.ndim > 1:
448
449
  result = []
449
450
  for i in range(len(measures)):
450
- projected = project_steps(data[PARAMS_KEY][name][i], projection)
451
+ projected = project_steps(data.params[name][i], projection)
451
452
  result.append(projected)
452
- output[name] = np.array(result).T
453
+ output[name] = np.array(result)
453
454
  else:
454
- output[name] = project_steps(data[PARAMS_KEY][name], projection)
455
- return output
455
+ output[name] = project_steps(data.params[name], projection)
456
+ return SufficiencyOutput(projection, data.params, output)
456
457
 
457
458
  @classmethod
458
459
  def plot(cls, data: SufficiencyOutput, class_names: Optional[Sequence[str]] = None) -> List[Figure]:
@@ -460,7 +461,7 @@ class Sufficiency(EvaluateMixin):
460
461
 
461
462
  Parameters
462
463
  ----------
463
- data : Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
464
+ data : SufficiencyOutput
464
465
  Dataclass containing the average of each measure per substep
465
466
 
466
467
  Returns
@@ -470,29 +471,19 @@ class Sufficiency(EvaluateMixin):
470
471
 
471
472
  Raises
472
473
  ------
473
- KeyError
474
- If STEPS_KEY or measure is not a valid key
475
474
  ValueError
476
475
  If the length of data points in the measures do not match
477
476
  """
478
- validate_output(data)
479
-
480
- # X, y data
481
- steps = cast(np.ndarray, data[STEPS_KEY])
482
-
483
477
  # Extrapolation parameters
484
- last_X = steps[-1]
485
- geomshape = (0.01 * last_X, last_X * 4, len(steps))
478
+ last_X = data.steps[-1]
479
+ geomshape = (0.01 * last_X, last_X * 4, len(data.steps))
486
480
  extrapolated = np.geomspace(*geomshape).astype(np.int64)
487
481
 
488
482
  # Stores all plots
489
483
  plots = []
490
484
 
491
485
  # Create a plot for each measure on one figure
492
- for name, measures in data.items():
493
- if name in [STEPS_KEY, PARAMS_KEY]:
494
- continue
495
- measures = cast(np.ndarray, measures)
486
+ for name, measures in data.measures.items():
496
487
  if measures.ndim > 1:
497
488
  if class_names is not None and len(measures) != len(class_names):
498
489
  raise IndexError("Class name count does not align with measures")
@@ -500,56 +491,54 @@ class Sufficiency(EvaluateMixin):
500
491
  class_name = str(i) if class_names is None else class_names[i]
501
492
  fig = plot_measure(
502
493
  f"{name}_{class_name}",
503
- steps,
494
+ data.steps,
504
495
  measure,
505
- data[PARAMS_KEY][name][i],
496
+ data.params[name][i],
506
497
  extrapolated,
507
498
  )
508
499
  plots.append(fig)
509
500
 
510
501
  else:
511
- fig = plot_measure(name, steps, measures, data[PARAMS_KEY][name], extrapolated)
502
+ fig = plot_measure(name, data.steps, measures, data.params[name], extrapolated)
512
503
  plots.append(fig)
513
504
 
514
505
  return plots
515
506
 
516
507
  @classmethod
517
- def inv_project(cls, targets: Dict[str, np.ndarray], data: SufficiencyOutput) -> Dict[str, np.ndarray]:
508
+ def inv_project(cls, targets: Dict[str, NDArray], data: SufficiencyOutput) -> Dict[str, NDArray]:
518
509
  """
519
510
  Calculate training samples needed to achieve target model metric values.
520
511
 
521
512
  Parameters
522
513
  ----------
523
- targets : Dict[str, np.ndarray]
514
+ targets : Dict[str, NDArray]
524
515
  Dictionary of target metric scores (from 0.0 to 1.0) that we want
525
516
  to achieve, where the key is the name of the metric.
526
517
 
527
- data : Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
518
+ data : SufficiencyOutput
528
519
  Dataclass containing the average of each measure per substep
529
520
 
530
521
  Returns
531
522
  -------
532
- Dict[str, np.ndarray]
523
+ Dict[str, NDArray]
533
524
  List of the number of training samples needed to achieve each
534
525
  corresponding entry in targets
535
526
  """
536
527
 
537
- validate_output(data)
538
-
539
528
  projection = {}
540
529
 
541
530
  for name, target in targets.items():
542
- if name not in data:
531
+ if name not in data.measures:
543
532
  continue
544
533
 
545
- measure = cast(np.ndarray, data[name])
534
+ measure = data.measures[name]
546
535
  if measure.ndim > 1:
547
536
  projection[name] = np.zeros((len(measure), len(target)))
548
537
  for i in range(len(measure)):
549
538
  projection[name][i] = inv_project_steps(
550
- data[PARAMS_KEY][name][i], target[i] if target.ndim == measure.ndim else target
539
+ data.params[name][i], target[i] if target.ndim == measure.ndim else target
551
540
  )
552
541
  else:
553
- projection[name] = inv_project_steps(data[PARAMS_KEY][name], target)
542
+ projection[name] = inv_project_steps(data.params[name], target)
554
543
 
555
544
  return projection
@@ -1,3 +1,3 @@
1
- from dataeval._internal.flags import ImageHash, ImageProperty, ImageStatistics, ImageVisuals
1
+ from dataeval._internal.flags import ImageStat
2
2
 
3
- __all__ = ["ImageHash", "ImageProperty", "ImageStatistics", "ImageVisuals"]
3
+ __all__ = ["ImageStat"]
@@ -1,8 +1,27 @@
1
- from dataeval._internal.metrics.ber import BER
2
- from dataeval._internal.metrics.coverage import Coverage
3
- from dataeval._internal.metrics.divergence import Divergence
4
- from dataeval._internal.metrics.parity import Parity
5
- from dataeval._internal.metrics.stats import ChannelStats, ImageStats
6
- from dataeval._internal.metrics.uap import UAP
1
+ from typing import List
7
2
 
8
- __all__ = ["BER", "Coverage", "Divergence", "Parity", "ChannelStats", "ImageStats", "UAP"]
3
+ __all__: List[str] = []
4
+
5
+ from dataeval._internal.metrics.balance import balance, balance_classwise
6
+ from dataeval._internal.metrics.ber import ber
7
+ from dataeval._internal.metrics.coverage import coverage
8
+ from dataeval._internal.metrics.divergence import divergence
9
+ from dataeval._internal.metrics.diversity import diversity, diversity_classwise
10
+ from dataeval._internal.metrics.parity import parity, parity_metadata
11
+ from dataeval._internal.metrics.stats import channelstats, imagestats
12
+ from dataeval._internal.metrics.uap import uap
13
+
14
+ __all__ += [
15
+ "balance",
16
+ "balance_classwise",
17
+ "ber",
18
+ "channelstats",
19
+ "coverage",
20
+ "divergence",
21
+ "diversity",
22
+ "diversity_classwise",
23
+ "imagestats",
24
+ "parity",
25
+ "parity_metadata",
26
+ "uap",
27
+ ]
@@ -0,0 +1,9 @@
1
+ from importlib.util import find_spec
2
+ from typing import List
3
+
4
+ __all__: List[str] = []
5
+
6
+ if find_spec("torch") is not None: # pragma: no cover
7
+ from dataeval._internal.utils import read_dataset
8
+
9
+ __all__ += ["read_dataset"]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dataeval
3
- Version: 0.63.0
3
+ Version: 0.65.0
4
4
  Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
5
5
  Home-page: https://dataeval.ai/
6
6
  License: MIT
@@ -0,0 +1,60 @@
1
+ dataeval/__init__.py,sha256=Uok47bAn3XhZppFB7u2BAVel5MDFpXC-1fEFlrWBIi8,424
2
+ dataeval/_internal/detectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
+ dataeval/_internal/detectors/clusterer.py,sha256=QHME6JQBqQe0xgEDuOGav6EDZFFq1hNiVde-DfUdNKU,20697
4
+ dataeval/_internal/detectors/drift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ dataeval/_internal/detectors/drift/base.py,sha256=0Vs69uctnC17pLhJ53rhDsQ8DLlZAj06LDCVGJhsMYo,9298
6
+ dataeval/_internal/detectors/drift/cvm.py,sha256=5tmD6uBumJgBS3u6e46Az13ytWSIScAAImvPHNk4PDA,4052
7
+ dataeval/_internal/detectors/drift/ks.py,sha256=FKHbJ4GR3kkADjPkf2CrjumIILEvuVUynjrOAFsK5no,4046
8
+ dataeval/_internal/detectors/drift/mmd.py,sha256=n5Z2tvpX41J5UPEE1JsJVBnVhGwZp3ADmLXEqFyXiek,7653
9
+ dataeval/_internal/detectors/drift/torch.py,sha256=zd2PcvSQ7j0rLwq8CvGpF9o8v5cVYZJFGv4jetU2PDo,10890
10
+ dataeval/_internal/detectors/drift/uncertainty.py,sha256=TZ1JIoZ9HRnrHupfPHvGT1P0gR-hYdH9bsigzJr_QcE,5311
11
+ dataeval/_internal/detectors/duplicates.py,sha256=VjunPZ9ffhVNFda9OIvnfE7QCvdA7OOfQMWEzDn2mdU,2973
12
+ dataeval/_internal/detectors/linter.py,sha256=gC-KDajxyNyBtCphtC2NULjeXmFx-d15GKF9I-MNOdI,5866
13
+ dataeval/_internal/detectors/ood/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
14
+ dataeval/_internal/detectors/ood/ae.py,sha256=FjqMucicFsDIKJMAOjWpKBohrPM4F1ubFLJk91GVqio,2681
15
+ dataeval/_internal/detectors/ood/aegmm.py,sha256=Kf9R5q-hoRg6RUHlJG-2oo52ZKeQmJQbxG0kFtHh6zA,2416
16
+ dataeval/_internal/detectors/ood/base.py,sha256=ybY4DJeQrzY62tEFnaFnAUL0ILkulOes63aA09jKynw,7633
17
+ dataeval/_internal/detectors/ood/llr.py,sha256=Amj3MFmoE0wi60qsHF7qMGRzO3ZeUW4ywUcLppx1ZQw,10160
18
+ dataeval/_internal/detectors/ood/vae.py,sha256=ntabTTTmPhJ18giZ7A64mxpJvTH9pIHmHPGGnu-gA8g,2987
19
+ dataeval/_internal/detectors/ood/vaegmm.py,sha256=opBfFLuXEAIMa8E6scwf-GWbZbuXnsqXlXTbLN4MoYg,2861
20
+ dataeval/_internal/flags.py,sha256=BKRYNvANpleVb1DGWqZASl_CB-BOa4UnamFY2So53Cc,2152
21
+ dataeval/_internal/interop.py,sha256=kX6lsEb7xoMD3iRG8GUp4uLbCCoC4Rb33PS6CHUl7BU,1036
22
+ dataeval/_internal/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
23
+ dataeval/_internal/metrics/balance.py,sha256=gD0Vm6Y55v6byfCJQi1c6cj9YzsDfCsHsEmFu8OrR1E,6758
24
+ dataeval/_internal/metrics/ber.py,sha256=bz7l3RqM54mYUwA8pqk9gq5GMFeRCNMD-65Prm9BV3w,4628
25
+ dataeval/_internal/metrics/coverage.py,sha256=9ZvcNjItE9rEyA2UHPE1K9zpTbbib4xqk8WpPpDN8ok,4037
26
+ dataeval/_internal/metrics/divergence.py,sha256=nmMUfr9FGnH798eb6xzEiMj4C42rQVthh5HeexiY6EE,4119
27
+ dataeval/_internal/metrics/diversity.py,sha256=dAgKaqY1J5cHy0JPUIN3kuW6NyS3wUb76w0EDipQaE4,7148
28
+ dataeval/_internal/metrics/parity.py,sha256=kOKw-B0RuralYxfBx5FiwyRn9kzR0uM9n8dpA7yebBI,11463
29
+ dataeval/_internal/metrics/stats.py,sha256=Fg1HZIQ1PWX4kgakN5wOOS627_vKt40nzvlzdjh7Q-E,8752
30
+ dataeval/_internal/metrics/uap.py,sha256=jv0ATJ5t5tEnKfcbuqB7KanO1n9dsqDsn42riJpG0M4,1307
31
+ dataeval/_internal/metrics/utils.py,sha256=izd1jimtLP0L__OERITqs0ppvYLL0PTlAC3kDfh3GOE,13037
32
+ dataeval/_internal/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
33
+ dataeval/_internal/models/pytorch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
34
+ dataeval/_internal/models/pytorch/autoencoder.py,sha256=iK3Z9claesU_pJkRaiFJIZ9zKZg-Qj8ugzVYTTokDbE,6123
35
+ dataeval/_internal/models/pytorch/blocks.py,sha256=pm2xwsDZjZJYXrhhiz8husvh2vHmrkFMSYEn-EDUD5Q,1354
36
+ dataeval/_internal/models/pytorch/utils.py,sha256=Qgwym1PxGuwxbXCKUT-8r6Iyrxqm7x94oj45Vf5_CjE,1675
37
+ dataeval/_internal/models/tensorflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
38
+ dataeval/_internal/models/tensorflow/autoencoder.py,sha256=rErnOfDFTd7e4brSGQ2Lr1x1kNjSEHdbOREOtUfIhIM,9975
39
+ dataeval/_internal/models/tensorflow/gmm.py,sha256=wnqQKm3fURuvBROUd2fitCqzKViDo-g0-Djr3TBHZ3U,3640
40
+ dataeval/_internal/models/tensorflow/losses.py,sha256=24gDqrA-EBg9J9tkLuXYZzbkwUjjAX8c1RAtN0o72xA,3774
41
+ dataeval/_internal/models/tensorflow/pixelcnn.py,sha256=B5cwB2IGPw-7b8klt82j_60g_IvqSiDELxvbiBYJtAo,48068
42
+ dataeval/_internal/models/tensorflow/trainer.py,sha256=rHWRHrX5hMj2iNZD9HqzfhPvqLwR2g-Aw-68Vq2_U94,4117
43
+ dataeval/_internal/models/tensorflow/utils.py,sha256=G7JpmjmWlJtqqHdxnbiBRVQjAXYFTqShii5Yz_Ici9U,8603
44
+ dataeval/_internal/output.py,sha256=L-EbpcdyZR2-NvsEKJkVpHXcyr3chdydbGNqoeXl-lI,2975
45
+ dataeval/_internal/utils.py,sha256=umvc_vN5c5IR0lz2F1U2YjA3VZloKTAEp9BQx8rSk6g,1561
46
+ dataeval/_internal/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
47
+ dataeval/_internal/workflows/sufficiency.py,sha256=BNl11OOrAjpyAIXeVhbGbJJ_tQDwZjXRW6dnlb6LYPM,17781
48
+ dataeval/detectors/__init__.py,sha256=I2e7YWb55RRlKQll85Z6KdN5wdBa53smn-_fcZIsCwA,1507
49
+ dataeval/flags/__init__.py,sha256=qo06_Tk0ul4lOhKSEs0HE2G6WBFvMwNJq77vRX1ynww,72
50
+ dataeval/metrics/__init__.py,sha256=eGL4LMxM1pTWwsT8HI-aIA7pZ2EPGFVIa86rG2LlVXs,787
51
+ dataeval/models/__init__.py,sha256=onevPb5wznCggowBnVT0OUa8uBJXZCbrkFuek1UFvOs,293
52
+ dataeval/models/tensorflow/__init__.py,sha256=A1XRxVGHefuvh_WpaKE1x95pRD1FecuFp66iuNPA_5U,424
53
+ dataeval/models/torch/__init__.py,sha256=su7P9DF9LChlVCNHWG6d7s_yeIfWQbhCYWIkzJe0Qig,190
54
+ dataeval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
55
+ dataeval/utils/__init__.py,sha256=bgUXeumTEspt2Q76YyEliGrnS-_incswY-pDexPdSCc,229
56
+ dataeval/workflows/__init__.py,sha256=ObgS1cVYFRzFZWbNzGs2OcU02IVkJkAMHNnlnSNTMCE,208
57
+ dataeval-0.65.0.dist-info/LICENSE.txt,sha256=Kpzcfobf1HlqafF-EX6dQLw9TlJiaJzfgvLQFukyXYw,1060
58
+ dataeval-0.65.0.dist-info/METADATA,sha256=-bjqI2XgbnRnGoHU3qf0NLfdHYY4bW5cm5tx__ux8zU,4217
59
+ dataeval-0.65.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
60
+ dataeval-0.65.0.dist-info/RECORD,,
File without changes
@@ -1,63 +0,0 @@
1
- from typing import Tuple
2
-
3
- import numpy as np
4
- from scipy.sparse import coo_matrix
5
- from scipy.stats import mode
6
-
7
- from dataeval._internal.functional.utils import compute_neighbors, get_classes_counts, minimum_spanning_tree
8
-
9
-
10
- def ber_mst(X: np.ndarray, y: np.ndarray, _: int) -> Tuple[float, float]:
11
- """Calculates the Bayes Error Rate using a minimum spanning tree
12
-
13
- Parameters
14
- ----------
15
- X : np.ndarray (N, :)
16
- Data points with arbitrary dimensionality
17
- y : np.ndarray (N, 1)
18
- Labels for each data point
19
- """
20
-
21
- M, N = get_classes_counts(y)
22
-
23
- tree = coo_matrix(minimum_spanning_tree(X))
24
- matches = np.sum([y[tree.row[i]] != y[tree.col[i]] for i in range(N - 1)])
25
- deltas = matches / (2 * N)
26
- upper = 2 * deltas
27
- lower = ((M - 1) / (M)) * (1 - max(1 - 2 * ((M) / (M - 1)) * deltas, 0) ** 0.5)
28
- return upper, lower
29
-
30
-
31
- def ber_knn(X: np.ndarray, y: np.ndarray, k: int) -> Tuple[float, float]:
32
- """Calculates the Bayes Error Rate using K-nearest neighbors"""
33
-
34
- M, N = get_classes_counts(y)
35
-
36
- # All features belong on second dimension
37
- X = X.reshape((X.shape[0], -1))
38
- nn_indices = compute_neighbors(X, X, k=k)
39
- nn_indices = np.expand_dims(nn_indices, axis=1) if nn_indices.ndim == 1 else nn_indices
40
- modal_class = mode(y[nn_indices], axis=1, keepdims=True).mode.squeeze()
41
- upper = float(np.count_nonzero(modal_class - y) / N)
42
- lower = _knn_lowerbound(upper, M, k)
43
- return upper, lower
44
-
45
-
46
- def _knn_lowerbound(value: float, classes: int, k: int) -> float:
47
- """Several cases for computing the BER lower bound"""
48
- if value <= 1e-10:
49
- return 0.0
50
-
51
- if classes == 2 and k != 1:
52
- if k > 5:
53
- # Property 2 (Devroye, 1981) cited in Snoopy paper, not in snoopy repo
54
- alpha = 0.3399
55
- beta = 0.9749
56
- a_k = alpha * np.sqrt(k) / (k - 3.25) * (1 + beta / (np.sqrt(k - 3)))
57
- return value / (1 + a_k)
58
- if k > 2:
59
- return value / (1 + (1 / np.sqrt(k)))
60
- # k == 2:
61
- return value / 2
62
-
63
- return ((classes - 1) / classes) * (1 - np.sqrt(max(0, 1 - ((classes / (classes - 1)) * value))))