dataeval 0.76.1__py3-none-any.whl → 0.82.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. dataeval/__init__.py +3 -3
  2. dataeval/config.py +77 -0
  3. dataeval/detectors/__init__.py +1 -1
  4. dataeval/detectors/drift/__init__.py +6 -6
  5. dataeval/detectors/drift/{base.py → _base.py} +40 -85
  6. dataeval/detectors/drift/{cvm.py → _cvm.py} +21 -28
  7. dataeval/detectors/drift/{ks.py → _ks.py} +20 -26
  8. dataeval/detectors/drift/{mmd.py → _mmd.py} +31 -43
  9. dataeval/detectors/drift/{torch.py → _torch.py} +2 -1
  10. dataeval/detectors/drift/{uncertainty.py → _uncertainty.py} +24 -7
  11. dataeval/detectors/drift/updates.py +20 -3
  12. dataeval/detectors/linters/__init__.py +3 -5
  13. dataeval/detectors/linters/duplicates.py +13 -36
  14. dataeval/detectors/linters/outliers.py +23 -148
  15. dataeval/detectors/ood/__init__.py +1 -1
  16. dataeval/detectors/ood/ae.py +30 -9
  17. dataeval/detectors/ood/base.py +5 -4
  18. dataeval/detectors/ood/mixin.py +21 -7
  19. dataeval/detectors/ood/vae.py +73 -0
  20. dataeval/metadata/__init__.py +6 -0
  21. dataeval/metadata/_distance.py +167 -0
  22. dataeval/metadata/_ood.py +217 -0
  23. dataeval/metadata/_utils.py +44 -0
  24. dataeval/metrics/__init__.py +1 -1
  25. dataeval/metrics/bias/__init__.py +6 -4
  26. dataeval/metrics/bias/{balance.py → _balance.py} +15 -101
  27. dataeval/metrics/bias/_coverage.py +98 -0
  28. dataeval/metrics/bias/{diversity.py → _diversity.py} +18 -111
  29. dataeval/metrics/bias/{parity.py → _parity.py} +39 -77
  30. dataeval/metrics/estimators/__init__.py +15 -4
  31. dataeval/metrics/estimators/{ber.py → _ber.py} +42 -29
  32. dataeval/metrics/estimators/_clusterer.py +44 -0
  33. dataeval/metrics/estimators/{divergence.py → _divergence.py} +18 -30
  34. dataeval/metrics/estimators/{uap.py → _uap.py} +4 -18
  35. dataeval/metrics/stats/__init__.py +16 -13
  36. dataeval/metrics/stats/{base.py → _base.py} +82 -133
  37. dataeval/metrics/stats/{boxratiostats.py → _boxratiostats.py} +15 -18
  38. dataeval/metrics/stats/_dimensionstats.py +75 -0
  39. dataeval/metrics/stats/{hashstats.py → _hashstats.py} +21 -37
  40. dataeval/metrics/stats/_imagestats.py +94 -0
  41. dataeval/metrics/stats/_labelstats.py +131 -0
  42. dataeval/metrics/stats/{pixelstats.py → _pixelstats.py} +19 -50
  43. dataeval/metrics/stats/{visualstats.py → _visualstats.py} +23 -54
  44. dataeval/outputs/__init__.py +53 -0
  45. dataeval/{output.py → outputs/_base.py} +55 -25
  46. dataeval/outputs/_bias.py +381 -0
  47. dataeval/outputs/_drift.py +83 -0
  48. dataeval/outputs/_estimators.py +114 -0
  49. dataeval/outputs/_linters.py +184 -0
  50. dataeval/{detectors/ood/output.py → outputs/_ood.py} +22 -22
  51. dataeval/outputs/_stats.py +387 -0
  52. dataeval/outputs/_utils.py +44 -0
  53. dataeval/outputs/_workflows.py +364 -0
  54. dataeval/typing.py +234 -0
  55. dataeval/utils/__init__.py +2 -2
  56. dataeval/utils/_array.py +169 -0
  57. dataeval/utils/_bin.py +199 -0
  58. dataeval/utils/_clusterer.py +144 -0
  59. dataeval/utils/_fast_mst.py +189 -0
  60. dataeval/utils/{image.py → _image.py} +6 -4
  61. dataeval/utils/_method.py +14 -0
  62. dataeval/utils/{shared.py → _mst.py} +3 -65
  63. dataeval/utils/{plot.py → _plot.py} +6 -6
  64. dataeval/utils/data/__init__.py +26 -0
  65. dataeval/utils/data/_dataset.py +217 -0
  66. dataeval/utils/data/_embeddings.py +104 -0
  67. dataeval/utils/data/_images.py +68 -0
  68. dataeval/utils/data/_metadata.py +360 -0
  69. dataeval/utils/data/_selection.py +126 -0
  70. dataeval/utils/{dataset/split.py → data/_split.py} +12 -38
  71. dataeval/utils/data/_targets.py +85 -0
  72. dataeval/utils/data/collate.py +103 -0
  73. dataeval/utils/data/datasets/__init__.py +17 -0
  74. dataeval/utils/data/datasets/_base.py +254 -0
  75. dataeval/utils/data/datasets/_cifar10.py +134 -0
  76. dataeval/utils/data/datasets/_fileio.py +168 -0
  77. dataeval/utils/data/datasets/_milco.py +153 -0
  78. dataeval/utils/data/datasets/_mixin.py +56 -0
  79. dataeval/utils/data/datasets/_mnist.py +183 -0
  80. dataeval/utils/data/datasets/_ships.py +123 -0
  81. dataeval/utils/data/datasets/_types.py +52 -0
  82. dataeval/utils/data/datasets/_voc.py +352 -0
  83. dataeval/utils/data/selections/__init__.py +15 -0
  84. dataeval/utils/data/selections/_classfilter.py +57 -0
  85. dataeval/utils/data/selections/_indices.py +26 -0
  86. dataeval/utils/data/selections/_limit.py +26 -0
  87. dataeval/utils/data/selections/_reverse.py +18 -0
  88. dataeval/utils/data/selections/_shuffle.py +29 -0
  89. dataeval/utils/metadata.py +51 -376
  90. dataeval/utils/torch/{gmm.py → _gmm.py} +4 -2
  91. dataeval/utils/torch/{internal.py → _internal.py} +21 -51
  92. dataeval/utils/torch/models.py +43 -2
  93. dataeval/workflows/__init__.py +2 -1
  94. dataeval/workflows/sufficiency.py +11 -346
  95. {dataeval-0.76.1.dist-info → dataeval-0.82.0.dist-info}/METADATA +5 -2
  96. dataeval-0.82.0.dist-info/RECORD +104 -0
  97. dataeval/detectors/linters/clusterer.py +0 -512
  98. dataeval/detectors/linters/merged_stats.py +0 -49
  99. dataeval/detectors/ood/metadata_ks_compare.py +0 -129
  100. dataeval/detectors/ood/metadata_least_likely.py +0 -119
  101. dataeval/interop.py +0 -69
  102. dataeval/metrics/bias/coverage.py +0 -194
  103. dataeval/metrics/stats/datasetstats.py +0 -202
  104. dataeval/metrics/stats/dimensionstats.py +0 -115
  105. dataeval/metrics/stats/labelstats.py +0 -210
  106. dataeval/utils/dataset/__init__.py +0 -7
  107. dataeval/utils/dataset/datasets.py +0 -412
  108. dataeval/utils/dataset/read.py +0 -63
  109. dataeval-0.76.1.dist-info/RECORD +0 -67
  110. /dataeval/{log.py → _log.py} +0 -0
  111. /dataeval/utils/torch/{blocks.py → _blocks.py} +0 -0
  112. {dataeval-0.76.1.dist-info → dataeval-0.82.0.dist-info}/LICENSE.txt +0 -0
  113. {dataeval-0.76.1.dist-info → dataeval-0.82.0.dist-info}/WHEEL +0 -0
@@ -11,30 +11,7 @@ from numpy.typing import NDArray
11
11
  from torch.utils.data import DataLoader, TensorDataset
12
12
  from tqdm import tqdm
13
13
 
14
-
15
- def get_device(device: str | torch.device | None = None) -> torch.device:
16
- """
17
- Instantiates a PyTorch device object.
18
-
19
- Parameters
20
- ----------
21
- device : str | torch.device | None, default None
22
- Either ``None``, a str ('gpu' or 'cpu') indicating the device to choose, or an
23
- already instantiated device object. If ``None``, the GPU is selected if it is
24
- detected, otherwise the CPU is used as a fallback.
25
-
26
- Returns
27
- -------
28
- The instantiated device object.
29
- """
30
- if isinstance(device, torch.device): # Already a torch device
31
- return device
32
- else: # Instantiate device
33
- if device is None or device.lower() in ["gpu", "cuda"]:
34
- torch_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
35
- else:
36
- torch_device = torch.device("cpu")
37
- return torch_device
14
+ from dataeval.config import get_device
38
15
 
39
16
 
40
17
  def predict_batch(
@@ -42,7 +19,7 @@ def predict_batch(
42
19
  model: Callable | torch.nn.Module | torch.nn.Sequential,
43
20
  device: torch.device | None = None,
44
21
  batch_size: int = int(1e10),
45
- preprocess_fn: Callable | None = None,
22
+ preprocess_fn: Callable[[torch.Tensor], torch.Tensor] | None = None,
46
23
  dtype: type[np.generic] | torch.dtype = np.float32,
47
24
  ) -> NDArray[Any] | torch.Tensor | tuple[Any, ...]:
48
25
  """
@@ -71,11 +48,12 @@ def predict_batch(
71
48
  """
72
49
  device = get_device(device)
73
50
  if isinstance(x, np.ndarray):
74
- x = torch.from_numpy(x).to(device)
51
+ x = torch.tensor(x, device=device)
75
52
  n = len(x)
76
53
  n_minibatch = int(np.ceil(n / batch_size))
77
54
  return_np = not isinstance(dtype, torch.dtype)
78
- preds = []
55
+ preds_tuple = None
56
+ preds_array = []
79
57
  with torch.no_grad():
80
58
  for i in range(n_minibatch):
81
59
  istart, istop = i * batch_size, min((i + 1) * batch_size, n)
@@ -83,23 +61,17 @@ def predict_batch(
83
61
  if isinstance(preprocess_fn, Callable):
84
62
  x_batch = preprocess_fn(x_batch)
85
63
 
86
- preds_tmp = model(x_batch.to(torch.float32).to(device))
64
+ preds_tmp = model(x_batch.to(dtype=torch.float32))
87
65
  if isinstance(preds_tmp, (list, tuple)):
88
- if len(preds) == 0: # init tuple with lists to store predictions
89
- preds = tuple([] for _ in range(len(preds_tmp)))
66
+ if preds_tuple is None: # init tuple with lists to store predictions
67
+ preds_tuple = tuple([] for _ in range(len(preds_tmp)))
90
68
  for j, p in enumerate(preds_tmp):
91
- if isinstance(p, torch.Tensor):
92
- p = p.cpu()
93
- preds[j].append(p if not return_np or isinstance(p, np.ndarray) else p.numpy())
69
+ p = p.cpu() if isinstance(p, torch.Tensor) else p
70
+ preds_tuple[j].append(p if not return_np or isinstance(p, np.ndarray) else p.numpy())
94
71
  elif isinstance(preds_tmp, (np.ndarray, torch.Tensor)):
95
- if isinstance(preds_tmp, torch.Tensor):
96
- preds_tmp = preds_tmp.cpu()
97
- if isinstance(preds, tuple):
98
- preds = list(preds)
99
- preds.append(
100
- preds_tmp
101
- if not return_np or isinstance(preds_tmp, np.ndarray) # type: ignore
102
- else preds_tmp.numpy()
72
+ preds_tmp = preds_tmp.cpu() if isinstance(preds_tmp, torch.Tensor) else preds_tmp
73
+ preds_array.append(
74
+ preds_tmp if not return_np or isinstance(preds_tmp, np.ndarray) else preds_tmp.numpy()
103
75
  )
104
76
  else:
105
77
  raise TypeError(
@@ -108,9 +80,7 @@ def predict_batch(
108
80
  torch.Tensor."
109
81
  )
110
82
  concat = partial(np.concatenate, axis=0) if return_np else partial(torch.cat, dim=0)
111
- out: tuple | np.ndarray | torch.Tensor = (
112
- tuple(concat(p) for p in preds) if isinstance(preds, tuple) else concat(preds) # type: ignore
113
- )
83
+ out = tuple(concat(p) for p in preds_tuple) if preds_tuple is not None else concat(preds_array)
114
84
  return out
115
85
 
116
86
 
@@ -154,18 +124,18 @@ def trainer(
154
124
  verbose
155
125
  Whether to print training progress.
156
126
  """
127
+ if loss_fn is None:
128
+ loss_fn = torch.nn.MSELoss()
129
+
157
130
  if optimizer is None:
158
131
  optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
159
132
 
160
133
  if y_train is None:
161
- dataset = TensorDataset(torch.from_numpy(x_train).to(torch.float32))
162
-
134
+ dataset = TensorDataset(torch.tensor(x_train, dtype=torch.float32))
163
135
  else:
164
- dataset = TensorDataset(
165
- torch.from_numpy(x_train).to(torch.float32), torch.from_numpy(y_train).to(torch.float32)
166
- )
136
+ dataset = TensorDataset(torch.tensor(x_train, dtype=torch.float32), torch.tensor(y_train, dtype=torch.float32))
167
137
 
168
- loader = DataLoader(dataset=dataset)
138
+ loader = DataLoader(dataset=dataset, batch_size=batch_size)
169
139
 
170
140
  model = model.to(device)
171
141
 
@@ -186,7 +156,7 @@ def trainer(
186
156
  y_hat = model(x)
187
157
  y = x if y is None else y
188
158
 
189
- loss = loss_fn(y, y_hat) # type: ignore
159
+ loss = loss_fn(y, *y_hat) if isinstance(y_hat, tuple) else loss_fn(y, y_hat) # type: ignore
190
160
 
191
161
  optimizer.zero_grad()
192
162
  loss.backward()
@@ -2,13 +2,19 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- __all__ = ["Autoencoder", "Encoder", "Decoder"]
5
+ __all__ = ["Autoencoder", "Encoder", "Decoder", "ResNet18"]
6
6
 
7
7
  import math
8
- from typing import Any
8
+ from typing import Any, Protocol, runtime_checkable
9
9
 
10
10
  import torch
11
11
  import torch.nn as nn
12
+ from torchvision.models import ResNet18_Weights, resnet18
13
+
14
+
15
+ @runtime_checkable
16
+ class SupportsEncode(Protocol):
17
+ def encode(self, x: Any) -> Any: ...
12
18
 
13
19
 
14
20
  class Autoencoder(nn.Module):
@@ -330,3 +336,38 @@ class Decoder_AE(nn.Module):
330
336
  x = self.decoder(x)
331
337
  x = x.reshape((-1, *self.input_shape))
332
338
  return x
339
+
340
+
341
+ class ResNet18(nn.Module):
342
+ """
343
+ A wrapper class for the torchvision.models.resnet18 model
344
+
345
+
346
+ Note
347
+ ----
348
+ This class is provided for the use of DataEval documentation and excludes many features
349
+ of the torchvision implementation.
350
+
351
+ Warning
352
+ -------
353
+ This class has been thoroughly tested for the purposes
354
+ of DataEval's documentation but not for operational use.
355
+ Please use with caution if deploying this class or subclasses.
356
+ """
357
+
358
+ def __init__(self, embedding_size: int = 128):
359
+ super().__init__()
360
+ self.model: nn.Module = resnet18(weights=ResNet18_Weights.DEFAULT, progress=False)
361
+ self.model.fc = nn.Linear(self.model.fc.in_features, embedding_size)
362
+
363
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
364
+ return self.model(x)
365
+
366
+ @staticmethod
367
+ def transforms() -> Any:
368
+ """(Returns) the default ResNet18 IMAGENET1K_V1 transforms"""
369
+
370
+ return ResNet18_Weights.DEFAULT.transforms()
371
+
372
+ def __str__(self) -> str:
373
+ return str(self.model)
@@ -4,4 +4,5 @@ Workflows perform a sequence of actions to analyze the dataset and make predicti
4
4
 
5
5
  __all__ = ["Sufficiency", "SufficiencyOutput"]
6
6
 
7
- from dataeval.workflows.sufficiency import Sufficiency, SufficiencyOutput
7
+ from dataeval.outputs._workflows import SufficiencyOutput
8
+ from dataeval.workflows.sufficiency import Sufficiency
@@ -2,260 +2,16 @@ from __future__ import annotations
2
2
 
3
3
  __all__ = []
4
4
 
5
- import contextlib
6
- import warnings
7
- from dataclasses import dataclass
8
- from typing import Any, Callable, Generic, Iterable, Mapping, Sequence, TypeVar, cast
5
+ from typing import Any, Callable, Generic, Iterable, Mapping, Sequence, Sized, TypeVar
9
6
 
10
7
  import numpy as np
11
8
  import torch
12
9
  import torch.nn as nn
13
- from numpy.typing import ArrayLike, NDArray
14
- from scipy.optimize import basinhopping
15
10
  from torch.utils.data import Dataset
16
11
 
17
- from dataeval.interop import as_numpy
18
- from dataeval.output import Output, set_metadata
19
-
20
- with contextlib.suppress(ImportError):
21
- from matplotlib.figure import Figure
22
-
23
-
24
- @dataclass(frozen=True)
25
- class SufficiencyOutput(Output):
26
- """
27
- Output class for :class:`Sufficiency` workflow.
28
-
29
- Attributes
30
- ----------
31
- steps : NDArray
32
- Array of sample sizes
33
- params : Dict[str, NDArray]
34
- Inverse power curve coefficients for the line of best fit for each measure
35
- measures : Dict[str, NDArray]
36
- Average of values observed for each sample size step for each measure
37
- """
38
-
39
- steps: NDArray[np.uint32]
40
- params: dict[str, NDArray[np.float64]]
41
- measures: dict[str, NDArray[np.float64]]
42
-
43
- def __post_init__(self) -> None:
44
- c = len(self.steps)
45
- if set(self.params) != set(self.measures):
46
- raise ValueError("params and measures have a key mismatch")
47
- for m, v in self.measures.items():
48
- c_v = v.shape[1] if v.ndim > 1 else len(v)
49
- if c != c_v:
50
- raise ValueError(f"{m} does not contain the expected number ({c}) of data points.")
51
-
52
- @set_metadata
53
- def project(
54
- self,
55
- projection: int | Iterable[int],
56
- ) -> SufficiencyOutput:
57
- """Projects the measures for each value of X
58
-
59
- Parameters
60
- ----------
61
- projection : int | Iterable[int]
62
- Step or steps to project
63
-
64
- Returns
65
- -------
66
- SufficiencyOutput
67
- Dataclass containing the projected measures per projection
68
-
69
- Raises
70
- ------
71
- ValueError
72
- If the length of data points in the measures do not match
73
- If `projection` is not numerical
74
- """
75
- projection = np.asarray(list(projection) if isinstance(projection, Iterable) else [projection])
76
-
77
- if not np.issubdtype(projection.dtype, np.number):
78
- raise ValueError("'projection' must consist of numerical values")
79
-
80
- output = {}
81
- for name, measures in self.measures.items():
82
- if measures.ndim > 1:
83
- result = []
84
- for i in range(len(measures)):
85
- projected = project_steps(self.params[name][i], projection)
86
- result.append(projected)
87
- output[name] = np.array(result)
88
- else:
89
- output[name] = project_steps(self.params[name], projection)
90
- return SufficiencyOutput(projection, self.params, output)
91
-
92
- def plot(self, class_names: Sequence[str] | None = None) -> list[Figure]:
93
- """Plotting function for data :term:`sufficience<Sufficiency>` tasks
94
-
95
- Parameters
96
- ----------
97
- class_names : Sequence[str] | None, default None
98
- List of class names
99
-
100
- Returns
101
- -------
102
- list[Figure]
103
- List of Figures for each measure
104
-
105
- Raises
106
- ------
107
- ValueError
108
- If the length of data points in the measures do not match
109
- """
110
- # Extrapolation parameters
111
- last_X = self.steps[-1]
112
- geomshape = (0.01 * last_X, last_X * 4, len(self.steps))
113
- extrapolated = np.geomspace(*geomshape).astype(np.int64)
114
-
115
- # Stores all plots
116
- plots = []
117
-
118
- # Create a plot for each measure on one figure
119
- for name, measures in self.measures.items():
120
- if measures.ndim > 1:
121
- if class_names is not None and len(measures) != len(class_names):
122
- raise IndexError("Class name count does not align with measures")
123
- for i, measure in enumerate(measures):
124
- class_name = str(i) if class_names is None else class_names[i]
125
- fig = plot_measure(
126
- f"{name}_{class_name}",
127
- self.steps,
128
- measure,
129
- self.params[name][i],
130
- extrapolated,
131
- )
132
- plots.append(fig)
133
-
134
- else:
135
- fig = plot_measure(name, self.steps, measures, self.params[name], extrapolated)
136
- plots.append(fig)
137
-
138
- return plots
139
-
140
- def inv_project(self, targets: Mapping[str, ArrayLike]) -> dict[str, NDArray[np.float64]]:
141
- """
142
- Calculate training samples needed to achieve target model metric values.
143
-
144
- Parameters
145
- ----------
146
- targets : Mapping[str, ArrayLike]
147
- Mapping of target metric scores (from 0.0 to 1.0) that we want
148
- to achieve, where the key is the name of the metric.
149
-
150
- Returns
151
- -------
152
- dict[str, NDArray]
153
- List of the number of training samples needed to achieve each
154
- corresponding entry in targets
155
- """
156
-
157
- projection = {}
158
-
159
- for name, target in targets.items():
160
- tarray = as_numpy(target)
161
- if name not in self.measures:
162
- continue
163
-
164
- measure = self.measures[name]
165
- if measure.ndim > 1:
166
- projection[name] = np.zeros((len(measure), len(tarray)))
167
- for i in range(len(measure)):
168
- projection[name][i] = inv_project_steps(
169
- self.params[name][i], tarray[i] if tarray.ndim == measure.ndim else tarray
170
- )
171
- else:
172
- projection[name] = inv_project_steps(self.params[name], tarray)
173
-
174
- return projection
175
-
176
-
177
- def f_out(n_i: NDArray[Any], x: NDArray[Any]) -> NDArray[Any]:
178
- """
179
- Calculates the line of best fit based on its free parameters
180
-
181
- Parameters
182
- ----------
183
- n_i : NDArray
184
- Array of sample sizes
185
- x : NDArray
186
- Array of inverse power curve coefficients
187
-
188
- Returns
189
- -------
190
- NDArray
191
- Data points for the line of best fit
192
- """
193
- return x[0] * n_i ** (-x[1]) + x[2]
194
-
195
-
196
- def f_inv_out(y_i: NDArray[Any], x: NDArray[Any]) -> NDArray[np.uint64]:
197
- """
198
- Inverse function for f_out()
199
-
200
- Parameters
201
- ----------
202
- y_i : NDArray
203
- Data points for the line of best fit
204
- x : NDArray
205
- Array of inverse power curve coefficients
206
-
207
- Returns
208
- -------
209
- NDArray
210
- Array of sample sizes
211
- """
212
- n_i = ((y_i - x[2]) / x[0]) ** (-1 / x[1])
213
- return np.asarray(n_i, dtype=np.uint64)
214
-
215
-
216
- def calc_params(p_i: NDArray[Any], n_i: NDArray[Any], niter: int) -> NDArray[Any]:
217
- """
218
- Retrieves the inverse power curve coefficients for the line of best fit.
219
- Global minimization is done via basin hopping. More info on this algorithm
220
- can be found here: https://arxiv.org/abs/cond-mat/9803344 .
221
-
222
- Parameters
223
- ----------
224
- p_i : NDArray
225
- Array of corresponding losses
226
- n_i : NDArray
227
- Array of sample sizes
228
- niter : int
229
- Number of iterations to perform in the basin-hopping
230
- numerical process to curve-fit p_i
231
-
232
- Returns
233
- -------
234
- NDArray
235
- Array of parameters to recreate line of best fit
236
- """
237
-
238
- def is_valid(f_new, x_new, f_old, x_old):
239
- return f_new != np.nan
240
-
241
- def f(x):
242
- try:
243
- return np.sum(np.square(p_i - f_out(n_i, x)))
244
- except RuntimeWarning:
245
- return np.nan
246
-
247
- with warnings.catch_warnings():
248
- warnings.filterwarnings("error", category=RuntimeWarning)
249
- res = basinhopping(
250
- f,
251
- np.array([0.5, 0.5, 0.1]),
252
- niter=niter,
253
- stepsize=1.0,
254
- minimizer_kwargs={"method": "Powell"},
255
- accept_test=is_valid,
256
- niter_success=200,
257
- )
258
- return res.x
12
+ from dataeval.outputs import SufficiencyOutput
13
+ from dataeval.outputs._base import set_metadata
14
+ from dataeval.typing import ArrayLike
259
15
 
260
16
 
261
17
  def reset_parameters(model: nn.Module) -> nn.Module:
@@ -277,102 +33,14 @@ def reset_parameters(model: nn.Module) -> nn.Module:
277
33
 
278
34
 
279
35
  def validate_dataset_len(dataset: Dataset[Any]) -> int:
280
- if not hasattr(dataset, "__len__"):
36
+ if not isinstance(dataset, Sized):
281
37
  raise TypeError("Must provide a dataset with a length attribute")
282
- length: int = dataset.__len__() # type: ignore
38
+ length: int = len(dataset)
283
39
  if length <= 0:
284
40
  raise ValueError("Dataset length must be greater than 0")
285
41
  return length
286
42
 
287
43
 
288
- def project_steps(params: NDArray[Any], projection: NDArray[Any]) -> NDArray[Any]:
289
- """Projects the measures for each value of X
290
-
291
- Parameters
292
- ----------
293
- params : NDArray
294
- Inverse power curve coefficients used to calculate projection
295
- projection : NDArray
296
- Steps to extrapolate
297
-
298
- Returns
299
- -------
300
- NDArray
301
- Extrapolated measure values at each projection step
302
-
303
- """
304
- return 1 - f_out(projection, params)
305
-
306
-
307
- def inv_project_steps(params: NDArray[Any], targets: NDArray[Any]) -> NDArray[np.uint64]:
308
- """Inverse function for project_steps()
309
-
310
- Parameters
311
- ----------
312
- params : NDArray
313
- Inverse power curve coefficients used to calculate projection
314
- targets : NDArray
315
- Desired measure values
316
-
317
- Returns
318
- -------
319
- NDArray
320
- Array of sample sizes, or 0 if overflow
321
- """
322
- steps = f_inv_out(1 - np.array(targets), params)
323
- steps[np.isnan(steps)] = 0
324
- return np.ceil(steps)
325
-
326
-
327
- def get_curve_params(measures: dict[str, NDArray[Any]], ranges: NDArray[Any], niter: int) -> dict[str, NDArray[Any]]:
328
- """Calculates and aggregates parameters for both single and multi-class metrics"""
329
- output = {}
330
- for name, measure in measures.items():
331
- measure = cast(np.ndarray, measure)
332
- if measure.ndim > 1:
333
- result = []
334
- for value in measure:
335
- result.append(calc_params(1 - value, ranges, niter))
336
- output[name] = np.array(result)
337
- else:
338
- output[name] = calc_params(1 - measure, ranges, niter)
339
- return output
340
-
341
-
342
- def plot_measure(
343
- name: str,
344
- steps: NDArray[Any],
345
- measure: NDArray[Any],
346
- params: NDArray[Any],
347
- projection: NDArray[Any],
348
- ) -> Figure:
349
- import matplotlib.pyplot
350
-
351
- fig = matplotlib.pyplot.figure()
352
- fig = cast(Figure, fig)
353
- fig.tight_layout()
354
-
355
- ax = fig.add_subplot(111)
356
-
357
- ax.set_title(f"{name} Sufficiency")
358
- ax.set_ylabel(f"{name}")
359
- ax.set_xlabel("Steps")
360
-
361
- # Plot measure over each step
362
- ax.scatter(steps, measure, label=f"Model Results ({name})", s=15, c="black")
363
-
364
- # Plot extrapolation
365
- ax.plot(
366
- projection,
367
- project_steps(params, projection),
368
- linestyle="dashed",
369
- label=f"Potential Model Results ({name})",
370
- )
371
-
372
- ax.legend()
373
- return fig
374
-
375
-
376
44
  T = TypeVar("T")
377
45
 
378
46
 
@@ -460,13 +128,13 @@ class Sufficiency(Generic[T]):
460
128
  @property
461
129
  def eval_fn(
462
130
  self,
463
- ) -> Callable[[nn.Module, Dataset[T]], dict[str, float] | Mapping[str, ArrayLike]]:
131
+ ) -> Callable[[nn.Module, Dataset[T]], Mapping[str, float] | Mapping[str, ArrayLike]]:
464
132
  return self._eval_fn
465
133
 
466
134
  @eval_fn.setter
467
135
  def eval_fn(
468
136
  self,
469
- value: Callable[[nn.Module, Dataset[T]], dict[str, float] | Mapping[str, ArrayLike]],
137
+ value: Callable[[nn.Module, Dataset[T]], Mapping[str, float] | Mapping[str, ArrayLike]],
470
138
  ) -> None:
471
139
  if not callable(value):
472
140
  raise TypeError("Must provide a callable for eval_fn.")
@@ -489,7 +157,7 @@ class Sufficiency(Generic[T]):
489
157
  self._eval_kwargs = {} if value is None else value
490
158
 
491
159
  @set_metadata(state=["runs", "substeps"])
492
- def evaluate(self, eval_at: int | Iterable[int] | None = None, niter: int = 1000) -> SufficiencyOutput:
160
+ def evaluate(self, eval_at: int | Iterable[int] | None = None) -> SufficiencyOutput:
493
161
  """
494
162
  Creates data indices, trains models, and returns plotting data
495
163
 
@@ -498,8 +166,6 @@ class Sufficiency(Generic[T]):
498
166
  eval_at : int | Iterable[int] | None, default None
499
167
  Specify this to collect accuracies over a specific set of dataset lengths, rather
500
168
  than letting :term:`sufficiency<Sufficiency>` internally create the lengths to evaluate at.
501
- niter : int, default 1000
502
- Iterations to perform when using the basin-hopping method to curve-fit measure(s).
503
169
 
504
170
  Returns
505
171
  -------
@@ -523,7 +189,7 @@ class Sufficiency(Generic[T]):
523
189
  ... substeps=5,
524
190
  ... )
525
191
  >>> suff.evaluate()
526
- SufficiencyOutput(steps=array([ 1, 3, 10, 31, 100], dtype=uint32), params={'test': array([ 0., 42., 0.])}, measures={'test': array([1., 1., 1., 1., 1.])})
192
+ SufficiencyOutput(steps=array([ 1, 3, 10, 31, 100], dtype=uint32), measures={'test': array([1., 1., 1., 1., 1.])}, n_iter=1000)
527
193
  """ # noqa: E501
528
194
  if eval_at is not None:
529
195
  ranges = np.asarray(list(eval_at) if isinstance(eval_at, Iterable) else [eval_at])
@@ -568,5 +234,4 @@ class Sufficiency(Generic[T]):
568
234
 
569
235
  # The mean for each measure must be calculated before being returned
570
236
  measures = {k: (v / self.runs).T for k, v in measures.items()}
571
- params_output = get_curve_params(measures, ranges, niter)
572
- return SufficiencyOutput(ranges, params_output, measures)
237
+ return SufficiencyOutput(ranges, measures)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dataeval
3
- Version: 0.76.1
3
+ Version: 0.82.0
4
4
  Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
5
5
  Home-page: https://dataeval.ai/
6
6
  License: MIT
@@ -21,7 +21,10 @@ Classifier: Programming Language :: Python :: 3.12
21
21
  Classifier: Programming Language :: Python :: 3 :: Only
22
22
  Classifier: Topic :: Scientific/Engineering
23
23
  Provides-Extra: all
24
+ Requires-Dist: defusedxml (>=0.7.1)
25
+ Requires-Dist: fast_hdbscan (==0.2.0)
24
26
  Requires-Dist: matplotlib (>=3.7.1) ; extra == "all"
27
+ Requires-Dist: numba (>=0.59.1)
25
28
  Requires-Dist: numpy (>=1.24.2)
26
29
  Requires-Dist: pandas (>=2.0) ; extra == "all"
27
30
  Requires-Dist: pillow (>=10.3.0)
@@ -71,7 +74,7 @@ DataEval is easy to install, supports a wide range of Python versions, and is
71
74
  compatible with many of the most popular packages in the scientific and T&E
72
75
  communities.
73
76
 
74
- DataEval also has native interopability between JATIC's suite of tools when
77
+ DataEval also has native interoperability between JATIC's suite of tools when
75
78
  using MAITE-compliant datasets and models.
76
79
  <!-- end JATIC interop -->
77
80