dataeval 0.76.1__py3-none-any.whl → 0.82.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataeval/__init__.py +3 -3
- dataeval/config.py +77 -0
- dataeval/detectors/__init__.py +1 -1
- dataeval/detectors/drift/__init__.py +6 -6
- dataeval/detectors/drift/{base.py → _base.py} +40 -85
- dataeval/detectors/drift/{cvm.py → _cvm.py} +21 -28
- dataeval/detectors/drift/{ks.py → _ks.py} +20 -26
- dataeval/detectors/drift/{mmd.py → _mmd.py} +31 -43
- dataeval/detectors/drift/{torch.py → _torch.py} +2 -1
- dataeval/detectors/drift/{uncertainty.py → _uncertainty.py} +24 -7
- dataeval/detectors/drift/updates.py +20 -3
- dataeval/detectors/linters/__init__.py +3 -5
- dataeval/detectors/linters/duplicates.py +13 -36
- dataeval/detectors/linters/outliers.py +23 -148
- dataeval/detectors/ood/__init__.py +1 -1
- dataeval/detectors/ood/ae.py +30 -9
- dataeval/detectors/ood/base.py +5 -4
- dataeval/detectors/ood/mixin.py +21 -7
- dataeval/detectors/ood/vae.py +73 -0
- dataeval/metadata/__init__.py +6 -0
- dataeval/metadata/_distance.py +167 -0
- dataeval/metadata/_ood.py +217 -0
- dataeval/metadata/_utils.py +44 -0
- dataeval/metrics/__init__.py +1 -1
- dataeval/metrics/bias/__init__.py +6 -4
- dataeval/metrics/bias/{balance.py → _balance.py} +15 -101
- dataeval/metrics/bias/_coverage.py +98 -0
- dataeval/metrics/bias/{diversity.py → _diversity.py} +18 -111
- dataeval/metrics/bias/{parity.py → _parity.py} +39 -77
- dataeval/metrics/estimators/__init__.py +15 -4
- dataeval/metrics/estimators/{ber.py → _ber.py} +42 -29
- dataeval/metrics/estimators/_clusterer.py +44 -0
- dataeval/metrics/estimators/{divergence.py → _divergence.py} +18 -30
- dataeval/metrics/estimators/{uap.py → _uap.py} +4 -18
- dataeval/metrics/stats/__init__.py +16 -13
- dataeval/metrics/stats/{base.py → _base.py} +82 -133
- dataeval/metrics/stats/{boxratiostats.py → _boxratiostats.py} +15 -18
- dataeval/metrics/stats/_dimensionstats.py +75 -0
- dataeval/metrics/stats/{hashstats.py → _hashstats.py} +21 -37
- dataeval/metrics/stats/_imagestats.py +94 -0
- dataeval/metrics/stats/_labelstats.py +131 -0
- dataeval/metrics/stats/{pixelstats.py → _pixelstats.py} +19 -50
- dataeval/metrics/stats/{visualstats.py → _visualstats.py} +23 -54
- dataeval/outputs/__init__.py +53 -0
- dataeval/{output.py → outputs/_base.py} +55 -25
- dataeval/outputs/_bias.py +381 -0
- dataeval/outputs/_drift.py +83 -0
- dataeval/outputs/_estimators.py +114 -0
- dataeval/outputs/_linters.py +184 -0
- dataeval/{detectors/ood/output.py → outputs/_ood.py} +22 -22
- dataeval/outputs/_stats.py +387 -0
- dataeval/outputs/_utils.py +44 -0
- dataeval/outputs/_workflows.py +364 -0
- dataeval/typing.py +234 -0
- dataeval/utils/__init__.py +2 -2
- dataeval/utils/_array.py +169 -0
- dataeval/utils/_bin.py +199 -0
- dataeval/utils/_clusterer.py +144 -0
- dataeval/utils/_fast_mst.py +189 -0
- dataeval/utils/{image.py → _image.py} +6 -4
- dataeval/utils/_method.py +14 -0
- dataeval/utils/{shared.py → _mst.py} +3 -65
- dataeval/utils/{plot.py → _plot.py} +6 -6
- dataeval/utils/data/__init__.py +26 -0
- dataeval/utils/data/_dataset.py +217 -0
- dataeval/utils/data/_embeddings.py +104 -0
- dataeval/utils/data/_images.py +68 -0
- dataeval/utils/data/_metadata.py +360 -0
- dataeval/utils/data/_selection.py +126 -0
- dataeval/utils/{dataset/split.py → data/_split.py} +12 -38
- dataeval/utils/data/_targets.py +85 -0
- dataeval/utils/data/collate.py +103 -0
- dataeval/utils/data/datasets/__init__.py +17 -0
- dataeval/utils/data/datasets/_base.py +254 -0
- dataeval/utils/data/datasets/_cifar10.py +134 -0
- dataeval/utils/data/datasets/_fileio.py +168 -0
- dataeval/utils/data/datasets/_milco.py +153 -0
- dataeval/utils/data/datasets/_mixin.py +56 -0
- dataeval/utils/data/datasets/_mnist.py +183 -0
- dataeval/utils/data/datasets/_ships.py +123 -0
- dataeval/utils/data/datasets/_types.py +52 -0
- dataeval/utils/data/datasets/_voc.py +352 -0
- dataeval/utils/data/selections/__init__.py +15 -0
- dataeval/utils/data/selections/_classfilter.py +57 -0
- dataeval/utils/data/selections/_indices.py +26 -0
- dataeval/utils/data/selections/_limit.py +26 -0
- dataeval/utils/data/selections/_reverse.py +18 -0
- dataeval/utils/data/selections/_shuffle.py +29 -0
- dataeval/utils/metadata.py +51 -376
- dataeval/utils/torch/{gmm.py → _gmm.py} +4 -2
- dataeval/utils/torch/{internal.py → _internal.py} +21 -51
- dataeval/utils/torch/models.py +43 -2
- dataeval/workflows/__init__.py +2 -1
- dataeval/workflows/sufficiency.py +11 -346
- {dataeval-0.76.1.dist-info → dataeval-0.82.0.dist-info}/METADATA +5 -2
- dataeval-0.82.0.dist-info/RECORD +104 -0
- dataeval/detectors/linters/clusterer.py +0 -512
- dataeval/detectors/linters/merged_stats.py +0 -49
- dataeval/detectors/ood/metadata_ks_compare.py +0 -129
- dataeval/detectors/ood/metadata_least_likely.py +0 -119
- dataeval/interop.py +0 -69
- dataeval/metrics/bias/coverage.py +0 -194
- dataeval/metrics/stats/datasetstats.py +0 -202
- dataeval/metrics/stats/dimensionstats.py +0 -115
- dataeval/metrics/stats/labelstats.py +0 -210
- dataeval/utils/dataset/__init__.py +0 -7
- dataeval/utils/dataset/datasets.py +0 -412
- dataeval/utils/dataset/read.py +0 -63
- dataeval-0.76.1.dist-info/RECORD +0 -67
- /dataeval/{log.py → _log.py} +0 -0
- /dataeval/utils/torch/{blocks.py → _blocks.py} +0 -0
- {dataeval-0.76.1.dist-info → dataeval-0.82.0.dist-info}/LICENSE.txt +0 -0
- {dataeval-0.76.1.dist-info → dataeval-0.82.0.dist-info}/WHEEL +0 -0
@@ -0,0 +1,364 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
__all__ = []
|
4
|
+
|
5
|
+
import contextlib
|
6
|
+
import warnings
|
7
|
+
from dataclasses import dataclass
|
8
|
+
from typing import Any, Iterable, Mapping, Sequence, cast
|
9
|
+
|
10
|
+
import numpy as np
|
11
|
+
from numpy.typing import NDArray
|
12
|
+
|
13
|
+
with contextlib.suppress(ImportError):
|
14
|
+
from matplotlib.figure import Figure
|
15
|
+
|
16
|
+
from scipy.optimize import basinhopping
|
17
|
+
|
18
|
+
from dataeval.outputs._base import Output, set_metadata
|
19
|
+
from dataeval.typing import ArrayLike
|
20
|
+
from dataeval.utils._array import as_numpy
|
21
|
+
|
22
|
+
|
23
|
+
def f_out(n_i: NDArray[Any], x: NDArray[Any]) -> NDArray[Any]:
|
24
|
+
"""
|
25
|
+
Calculates the line of best fit based on its free parameters
|
26
|
+
|
27
|
+
Parameters
|
28
|
+
----------
|
29
|
+
n_i : NDArray
|
30
|
+
Array of sample sizes
|
31
|
+
x : NDArray
|
32
|
+
Array of inverse power curve coefficients
|
33
|
+
|
34
|
+
Returns
|
35
|
+
-------
|
36
|
+
NDArray
|
37
|
+
Data points for the line of best fit
|
38
|
+
"""
|
39
|
+
return x[0] * n_i ** (-x[1]) + x[2]
|
40
|
+
|
41
|
+
|
42
|
+
def project_steps(params: NDArray[Any], projection: NDArray[Any]) -> NDArray[Any]:
|
43
|
+
"""Projects the measures for each value of X
|
44
|
+
|
45
|
+
Parameters
|
46
|
+
----------
|
47
|
+
params : NDArray
|
48
|
+
Inverse power curve coefficients used to calculate projection
|
49
|
+
projection : NDArray
|
50
|
+
Steps to extrapolate
|
51
|
+
|
52
|
+
Returns
|
53
|
+
-------
|
54
|
+
NDArray
|
55
|
+
Extrapolated measure values at each projection step
|
56
|
+
|
57
|
+
"""
|
58
|
+
return 1 - f_out(projection, params)
|
59
|
+
|
60
|
+
|
61
|
+
def plot_measure(
|
62
|
+
name: str,
|
63
|
+
steps: NDArray[Any],
|
64
|
+
measure: NDArray[Any],
|
65
|
+
params: NDArray[Any],
|
66
|
+
projection: NDArray[Any],
|
67
|
+
) -> Figure:
|
68
|
+
import matplotlib.pyplot
|
69
|
+
|
70
|
+
fig = matplotlib.pyplot.figure()
|
71
|
+
fig = cast(Figure, fig)
|
72
|
+
fig.tight_layout()
|
73
|
+
|
74
|
+
ax = fig.add_subplot(111)
|
75
|
+
|
76
|
+
ax.set_title(f"{name} Sufficiency")
|
77
|
+
ax.set_ylabel(f"{name}")
|
78
|
+
ax.set_xlabel("Steps")
|
79
|
+
|
80
|
+
# Plot measure over each step
|
81
|
+
ax.scatter(steps, measure, label=f"Model Results ({name})", s=15, c="black")
|
82
|
+
|
83
|
+
# Plot extrapolation
|
84
|
+
ax.plot(
|
85
|
+
projection,
|
86
|
+
project_steps(params, projection),
|
87
|
+
linestyle="dashed",
|
88
|
+
label=f"Potential Model Results ({name})",
|
89
|
+
)
|
90
|
+
|
91
|
+
ax.legend()
|
92
|
+
return fig
|
93
|
+
|
94
|
+
|
95
|
+
def f_inv_out(y_i: NDArray[Any], x: NDArray[Any]) -> NDArray[np.uint64]:
|
96
|
+
"""
|
97
|
+
Inverse function for f_out()
|
98
|
+
|
99
|
+
Parameters
|
100
|
+
----------
|
101
|
+
y_i : NDArray
|
102
|
+
Data points for the line of best fit
|
103
|
+
x : NDArray
|
104
|
+
Array of inverse power curve coefficients
|
105
|
+
|
106
|
+
Returns
|
107
|
+
-------
|
108
|
+
NDArray
|
109
|
+
Array of sample sizes
|
110
|
+
"""
|
111
|
+
n_i = ((y_i - x[2]) / x[0]) ** (-1 / x[1])
|
112
|
+
return np.asarray(n_i, dtype=np.uint64)
|
113
|
+
|
114
|
+
|
115
|
+
def inv_project_steps(params: NDArray[Any], targets: NDArray[Any]) -> NDArray[np.uint64]:
|
116
|
+
"""Inverse function for project_steps()
|
117
|
+
|
118
|
+
Parameters
|
119
|
+
----------
|
120
|
+
params : NDArray
|
121
|
+
Inverse power curve coefficients used to calculate projection
|
122
|
+
targets : NDArray
|
123
|
+
Desired measure values
|
124
|
+
|
125
|
+
Returns
|
126
|
+
-------
|
127
|
+
NDArray
|
128
|
+
Array of sample sizes, or 0 if overflow
|
129
|
+
"""
|
130
|
+
steps = f_inv_out(1 - np.array(targets), params)
|
131
|
+
steps[np.isnan(steps)] = 0
|
132
|
+
return np.ceil(steps)
|
133
|
+
|
134
|
+
|
135
|
+
def calc_params(p_i: NDArray[Any], n_i: NDArray[Any], niter: int) -> NDArray[Any]:
|
136
|
+
"""
|
137
|
+
Retrieves the inverse power curve coefficients for the line of best fit.
|
138
|
+
Global minimization is done via basin hopping. More info on this algorithm
|
139
|
+
can be found here: https://arxiv.org/abs/cond-mat/9803344 .
|
140
|
+
|
141
|
+
Parameters
|
142
|
+
----------
|
143
|
+
p_i : NDArray
|
144
|
+
Array of corresponding losses
|
145
|
+
n_i : NDArray
|
146
|
+
Array of sample sizes
|
147
|
+
niter : int
|
148
|
+
Number of iterations to perform in the basin-hopping
|
149
|
+
numerical process to curve-fit p_i
|
150
|
+
|
151
|
+
Returns
|
152
|
+
-------
|
153
|
+
NDArray
|
154
|
+
Array of parameters to recreate line of best fit
|
155
|
+
"""
|
156
|
+
|
157
|
+
def is_valid(f_new, x_new, f_old, x_old):
|
158
|
+
return f_new != np.nan
|
159
|
+
|
160
|
+
def f(x):
|
161
|
+
try:
|
162
|
+
return np.sum(np.square(p_i - f_out(n_i, x)))
|
163
|
+
except RuntimeWarning:
|
164
|
+
return np.nan
|
165
|
+
|
166
|
+
with warnings.catch_warnings():
|
167
|
+
warnings.filterwarnings("error", category=RuntimeWarning)
|
168
|
+
res = basinhopping(
|
169
|
+
f,
|
170
|
+
np.array([0.5, 0.5, 0.1]),
|
171
|
+
niter=niter,
|
172
|
+
stepsize=1.0,
|
173
|
+
minimizer_kwargs={"method": "Powell"},
|
174
|
+
accept_test=is_valid,
|
175
|
+
niter_success=200,
|
176
|
+
)
|
177
|
+
return res.x
|
178
|
+
|
179
|
+
|
180
|
+
def get_curve_params(measures: dict[str, NDArray[Any]], ranges: NDArray[Any], niter: int) -> dict[str, NDArray[Any]]:
|
181
|
+
"""Calculates and aggregates parameters for both single and multi-class metrics"""
|
182
|
+
output = {}
|
183
|
+
for name, measure in measures.items():
|
184
|
+
measure = cast(np.ndarray, measure)
|
185
|
+
if measure.ndim > 1:
|
186
|
+
result = []
|
187
|
+
for value in measure:
|
188
|
+
result.append(calc_params(1 - value, ranges, niter))
|
189
|
+
output[name] = np.array(result)
|
190
|
+
else:
|
191
|
+
output[name] = calc_params(1 - measure, ranges, niter)
|
192
|
+
return output
|
193
|
+
|
194
|
+
|
195
|
+
@dataclass
|
196
|
+
class SufficiencyOutput(Output):
|
197
|
+
"""
|
198
|
+
Output class for :class:`.Sufficiency` workflow.
|
199
|
+
|
200
|
+
Attributes
|
201
|
+
----------
|
202
|
+
steps : NDArray
|
203
|
+
Array of sample sizes
|
204
|
+
measures : Dict[str, NDArray]
|
205
|
+
Average of values observed for each sample size step for each measure
|
206
|
+
n_iter : int, default 1000
|
207
|
+
Number of iterations to perform in the basin-hopping curve-fit process
|
208
|
+
"""
|
209
|
+
|
210
|
+
steps: NDArray[np.uint32]
|
211
|
+
measures: dict[str, NDArray[np.float64]]
|
212
|
+
n_iter: int = 1000
|
213
|
+
|
214
|
+
def __post_init__(self) -> None:
|
215
|
+
c = len(self.steps)
|
216
|
+
for m, v in self.measures.items():
|
217
|
+
c_v = v.shape[1] if v.ndim > 1 else len(v)
|
218
|
+
if c != c_v:
|
219
|
+
raise ValueError(f"{m} does not contain the expected number ({c}) of data points.")
|
220
|
+
self._params = None
|
221
|
+
|
222
|
+
@property
|
223
|
+
def params(self) -> dict[str, NDArray[Any]]:
|
224
|
+
if self._params is None:
|
225
|
+
self._params = {}
|
226
|
+
if self.n_iter not in self._params:
|
227
|
+
self._params[self.n_iter] = get_curve_params(self.measures, self.steps, self.n_iter)
|
228
|
+
return self._params[self.n_iter]
|
229
|
+
|
230
|
+
@set_metadata
|
231
|
+
def project(
|
232
|
+
self,
|
233
|
+
projection: int | Iterable[int],
|
234
|
+
) -> SufficiencyOutput:
|
235
|
+
"""
|
236
|
+
Projects the measures for each step.
|
237
|
+
|
238
|
+
Parameters
|
239
|
+
----------
|
240
|
+
projection : int | Iterable[int]
|
241
|
+
Step or steps to project
|
242
|
+
|
243
|
+
Returns
|
244
|
+
-------
|
245
|
+
SufficiencyOutput
|
246
|
+
Dataclass containing the projected measures per projection
|
247
|
+
|
248
|
+
Raises
|
249
|
+
------
|
250
|
+
ValueError
|
251
|
+
If the length of data points in the measures do not match
|
252
|
+
If `projection` is not numerical
|
253
|
+
"""
|
254
|
+
projection = np.asarray(list(projection) if isinstance(projection, Iterable) else [projection])
|
255
|
+
|
256
|
+
if not np.issubdtype(projection.dtype, np.number):
|
257
|
+
raise ValueError("'projection' must consist of numerical values")
|
258
|
+
|
259
|
+
output = {}
|
260
|
+
for name, measures in self.measures.items():
|
261
|
+
if measures.ndim > 1:
|
262
|
+
result = []
|
263
|
+
for i in range(len(measures)):
|
264
|
+
projected = project_steps(self.params[name][i], projection)
|
265
|
+
result.append(projected)
|
266
|
+
output[name] = np.array(result)
|
267
|
+
else:
|
268
|
+
output[name] = project_steps(self.params[name], projection)
|
269
|
+
proj = SufficiencyOutput(projection, output, self.n_iter)
|
270
|
+
proj._params = self._params
|
271
|
+
return proj
|
272
|
+
|
273
|
+
def plot(self, class_names: Sequence[str] | None = None) -> list[Figure]:
|
274
|
+
"""
|
275
|
+
Plotting function for data :term:`sufficience<Sufficiency>` tasks.
|
276
|
+
|
277
|
+
Parameters
|
278
|
+
----------
|
279
|
+
class_names : Sequence[str] | None, default None
|
280
|
+
List of class names
|
281
|
+
|
282
|
+
Returns
|
283
|
+
-------
|
284
|
+
list[Figure]
|
285
|
+
List of Figures for each measure
|
286
|
+
|
287
|
+
Raises
|
288
|
+
------
|
289
|
+
ValueError
|
290
|
+
If the length of data points in the measures do not match
|
291
|
+
|
292
|
+
Notes
|
293
|
+
-----
|
294
|
+
This method requires `matplotlib <https://matplotlib.org/>`_ to be installed.
|
295
|
+
"""
|
296
|
+
# Extrapolation parameters
|
297
|
+
last_X = self.steps[-1]
|
298
|
+
geomshape = (0.01 * last_X, last_X * 4, len(self.steps))
|
299
|
+
extrapolated = np.geomspace(*geomshape).astype(np.int64)
|
300
|
+
|
301
|
+
# Stores all plots
|
302
|
+
plots = []
|
303
|
+
|
304
|
+
# Create a plot for each measure on one figure
|
305
|
+
for name, measures in self.measures.items():
|
306
|
+
if measures.ndim > 1:
|
307
|
+
if class_names is not None and len(measures) != len(class_names):
|
308
|
+
raise IndexError("Class name count does not align with measures")
|
309
|
+
for i, measure in enumerate(measures):
|
310
|
+
class_name = str(i) if class_names is None else class_names[i]
|
311
|
+
fig = plot_measure(
|
312
|
+
f"{name}_{class_name}",
|
313
|
+
self.steps,
|
314
|
+
measure,
|
315
|
+
self.params[name][i],
|
316
|
+
extrapolated,
|
317
|
+
)
|
318
|
+
plots.append(fig)
|
319
|
+
|
320
|
+
else:
|
321
|
+
fig = plot_measure(name, self.steps, measures, self.params[name], extrapolated)
|
322
|
+
plots.append(fig)
|
323
|
+
|
324
|
+
return plots
|
325
|
+
|
326
|
+
def inv_project(
|
327
|
+
self, targets: Mapping[str, ArrayLike], n_iter: int | None = None
|
328
|
+
) -> dict[str, NDArray[np.float64]]:
|
329
|
+
"""
|
330
|
+
Calculate training samples needed to achieve target model metric values.
|
331
|
+
|
332
|
+
Parameters
|
333
|
+
----------
|
334
|
+
targets : Mapping[str, ArrayLike]
|
335
|
+
Mapping of target metric scores (from 0.0 to 1.0) that we want
|
336
|
+
to achieve, where the key is the name of the metric.
|
337
|
+
n_iter : int or None, default None
|
338
|
+
Iteration to use when calculating the inverse power curve, if None defaults to 1000
|
339
|
+
|
340
|
+
Returns
|
341
|
+
-------
|
342
|
+
dict[str, NDArray]
|
343
|
+
List of the number of training samples needed to achieve each
|
344
|
+
corresponding entry in targets
|
345
|
+
"""
|
346
|
+
|
347
|
+
projection = {}
|
348
|
+
|
349
|
+
for name, target in targets.items():
|
350
|
+
tarray = as_numpy(target)
|
351
|
+
if name not in self.measures:
|
352
|
+
continue
|
353
|
+
|
354
|
+
measure = self.measures[name]
|
355
|
+
if measure.ndim > 1:
|
356
|
+
projection[name] = np.zeros((len(measure), len(tarray)))
|
357
|
+
for i in range(len(measure)):
|
358
|
+
projection[name][i] = inv_project_steps(
|
359
|
+
self.params[name][i], tarray[i] if tarray.ndim == measure.ndim else tarray
|
360
|
+
)
|
361
|
+
else:
|
362
|
+
projection[name] = inv_project_steps(self.params[name], tarray)
|
363
|
+
|
364
|
+
return projection
|
dataeval/typing.py
ADDED
@@ -0,0 +1,234 @@
|
|
1
|
+
"""
|
2
|
+
Common type hints used for interoperability with DataEval.
|
3
|
+
"""
|
4
|
+
|
5
|
+
__all__ = [
|
6
|
+
"Array",
|
7
|
+
"ArrayLike",
|
8
|
+
"Dataset",
|
9
|
+
"AnnotatedDataset",
|
10
|
+
"DatasetMetadata",
|
11
|
+
"ImageClassificationDatum",
|
12
|
+
"ImageClassificationDataset",
|
13
|
+
"ObjectDetectionTarget",
|
14
|
+
"ObjectDetectionDatum",
|
15
|
+
"ObjectDetectionDataset",
|
16
|
+
"SegmentationTarget",
|
17
|
+
"SegmentationDatum",
|
18
|
+
"SegmentationDataset",
|
19
|
+
]
|
20
|
+
|
21
|
+
|
22
|
+
import sys
|
23
|
+
from typing import Any, Generic, Iterator, Protocol, Sequence, TypedDict, TypeVar, Union, runtime_checkable
|
24
|
+
|
25
|
+
from typing_extensions import NotRequired, Required
|
26
|
+
|
27
|
+
if sys.version_info >= (3, 10):
|
28
|
+
from typing import TypeAlias
|
29
|
+
else:
|
30
|
+
from typing_extensions import TypeAlias
|
31
|
+
|
32
|
+
|
33
|
+
@runtime_checkable
|
34
|
+
class Array(Protocol):
|
35
|
+
"""
|
36
|
+
Protocol for array objects providing interoperability with DataEval.
|
37
|
+
|
38
|
+
Supports common array representations with popular libraries like
|
39
|
+
PyTorch, Tensorflow and JAX, as well as NumPy arrays.
|
40
|
+
|
41
|
+
Example
|
42
|
+
-------
|
43
|
+
>>> import numpy as np
|
44
|
+
>>> import torch
|
45
|
+
>>> from dataeval.typing import Array
|
46
|
+
|
47
|
+
Create array objects
|
48
|
+
|
49
|
+
>>> ndarray = np.random.random((10, 10))
|
50
|
+
>>> tensor = torch.tensor([1, 2, 3])
|
51
|
+
|
52
|
+
Check type at runtime
|
53
|
+
|
54
|
+
>>> isinstance(ndarray, Array)
|
55
|
+
True
|
56
|
+
|
57
|
+
>>> isinstance(tensor, Array)
|
58
|
+
True
|
59
|
+
"""
|
60
|
+
|
61
|
+
@property
|
62
|
+
def shape(self) -> tuple[int, ...]: ...
|
63
|
+
def __array__(self) -> Any: ...
|
64
|
+
def __getitem__(self, key: Any, /) -> Any: ...
|
65
|
+
def __iter__(self) -> Iterator[Any]: ...
|
66
|
+
def __len__(self) -> int: ...
|
67
|
+
|
68
|
+
|
69
|
+
_T_co = TypeVar("_T_co", covariant=True)
|
70
|
+
_ScalarType = Union[int, float, bool, str]
|
71
|
+
ArrayLike: TypeAlias = Union[Sequence[_ScalarType], Sequence[Sequence[_ScalarType]], Sequence[Array], Array]
|
72
|
+
"""
|
73
|
+
Type alias for array-like objects used for interoperability with DataEval.
|
74
|
+
|
75
|
+
This includes native Python sequences, as well as objects that conform to
|
76
|
+
the :class:`Array` protocol.
|
77
|
+
"""
|
78
|
+
|
79
|
+
|
80
|
+
class DatasetMetadata(TypedDict, total=False):
|
81
|
+
"""
|
82
|
+
Dataset level metadata required for all `AnnotatedDataset` classes.
|
83
|
+
|
84
|
+
Attributes
|
85
|
+
----------
|
86
|
+
id : Required[str]
|
87
|
+
A unique identifier for the dataset
|
88
|
+
index2label : NotRequired[dict[int, str]]
|
89
|
+
A lookup table converting label value to class name
|
90
|
+
"""
|
91
|
+
|
92
|
+
id: Required[str]
|
93
|
+
index2label: NotRequired[dict[int, str]]
|
94
|
+
|
95
|
+
|
96
|
+
@runtime_checkable
|
97
|
+
class Dataset(Generic[_T_co], Protocol):
|
98
|
+
"""
|
99
|
+
Protocol for a generic `Dataset`.
|
100
|
+
|
101
|
+
Methods
|
102
|
+
-------
|
103
|
+
__getitem__(index: int)
|
104
|
+
Returns datum at specified index.
|
105
|
+
__len__()
|
106
|
+
Returns dataset length.
|
107
|
+
"""
|
108
|
+
|
109
|
+
def __getitem__(self, index: int, /) -> _T_co: ...
|
110
|
+
def __len__(self) -> int: ...
|
111
|
+
|
112
|
+
|
113
|
+
@runtime_checkable
|
114
|
+
class AnnotatedDataset(Dataset[_T_co], Generic[_T_co], Protocol):
|
115
|
+
"""
|
116
|
+
Protocol for a generic `AnnotatedDataset`.
|
117
|
+
|
118
|
+
Attributes
|
119
|
+
----------
|
120
|
+
metadata : :class:`.DatasetMetadata` or derivatives.
|
121
|
+
|
122
|
+
Methods
|
123
|
+
-------
|
124
|
+
__getitem__(index: int)
|
125
|
+
Returns datum at specified index.
|
126
|
+
__len__()
|
127
|
+
Returns dataset length.
|
128
|
+
|
129
|
+
Notes
|
130
|
+
-----
|
131
|
+
Inherits from :class:`.Dataset`.
|
132
|
+
"""
|
133
|
+
|
134
|
+
@property
|
135
|
+
def metadata(self) -> DatasetMetadata: ...
|
136
|
+
|
137
|
+
|
138
|
+
# ========== IMAGE CLASSIFICATION DATASETS ==========
|
139
|
+
|
140
|
+
|
141
|
+
ImageClassificationDatum: TypeAlias = tuple[Array, Array, dict[str, Any]]
|
142
|
+
"""
|
143
|
+
A type definition for an image classification datum tuple.
|
144
|
+
|
145
|
+
- :class:`Array` of shape (C, H, W) - Image data in channel, height, width format.
|
146
|
+
- :class:`Array` of shape (N,) - Class label as one-hot encoded ground-truth or prediction confidences.
|
147
|
+
- dict[str, Any] - Datum level metadata.
|
148
|
+
"""
|
149
|
+
|
150
|
+
|
151
|
+
ImageClassificationDataset: TypeAlias = AnnotatedDataset[ImageClassificationDatum]
|
152
|
+
"""
|
153
|
+
A type definition for an :class:`AnnotatedDataset` of :class:`ImageClassificationDatum` elements.
|
154
|
+
"""
|
155
|
+
|
156
|
+
# ========== OBJECT DETECTION DATASETS ==========
|
157
|
+
|
158
|
+
|
159
|
+
@runtime_checkable
|
160
|
+
class ObjectDetectionTarget(Protocol):
|
161
|
+
"""
|
162
|
+
A protocol for targets in an Object Detection dataset.
|
163
|
+
|
164
|
+
Attributes
|
165
|
+
----------
|
166
|
+
boxes : :class:`ArrayLike` of shape (N, 4)
|
167
|
+
labels : :class:`ArrayLike` of shape (N,)
|
168
|
+
scores : :class:`ArrayLike` of shape (N, M)
|
169
|
+
"""
|
170
|
+
|
171
|
+
@property
|
172
|
+
def boxes(self) -> ArrayLike: ...
|
173
|
+
|
174
|
+
@property
|
175
|
+
def labels(self) -> ArrayLike: ...
|
176
|
+
|
177
|
+
@property
|
178
|
+
def scores(self) -> ArrayLike: ...
|
179
|
+
|
180
|
+
|
181
|
+
ObjectDetectionDatum: TypeAlias = tuple[Array, ObjectDetectionTarget, dict[str, Any]]
|
182
|
+
"""
|
183
|
+
A type definition for an object detection datum tuple.
|
184
|
+
|
185
|
+
- :class:`Array` of shape (C, H, W) - Image data in channel, height, width format.
|
186
|
+
- :class:`ObjectDetectionTarget` - Object detection target information for the image.
|
187
|
+
- dict[str, Any] - Datum level metadata.
|
188
|
+
"""
|
189
|
+
|
190
|
+
|
191
|
+
ObjectDetectionDataset: TypeAlias = AnnotatedDataset[ObjectDetectionDatum]
|
192
|
+
"""
|
193
|
+
A type definition for an :class:`AnnotatedDataset` of :class:`ObjectDetectionDatum` elements.
|
194
|
+
"""
|
195
|
+
|
196
|
+
|
197
|
+
# ========== SEGMENTATION DATASETS ==========
|
198
|
+
|
199
|
+
|
200
|
+
@runtime_checkable
|
201
|
+
class SegmentationTarget(Protocol):
|
202
|
+
"""
|
203
|
+
A protocol for targets in a Segmentation dataset.
|
204
|
+
|
205
|
+
Attributes
|
206
|
+
----------
|
207
|
+
mask : :class:`ArrayLike`
|
208
|
+
labels : :class:`ArrayLike`
|
209
|
+
scores : :class:`ArrayLike`
|
210
|
+
"""
|
211
|
+
|
212
|
+
@property
|
213
|
+
def mask(self) -> ArrayLike: ...
|
214
|
+
|
215
|
+
@property
|
216
|
+
def labels(self) -> ArrayLike: ...
|
217
|
+
|
218
|
+
@property
|
219
|
+
def scores(self) -> ArrayLike: ...
|
220
|
+
|
221
|
+
|
222
|
+
SegmentationDatum: TypeAlias = tuple[Array, SegmentationTarget, dict[str, Any]]
|
223
|
+
"""
|
224
|
+
A type definition for an image classification datum tuple.
|
225
|
+
|
226
|
+
- :class:`Array` of shape (C, H, W) - Image data in channel, height, width format.
|
227
|
+
- :class:`SegmentationTarget` - Segmentation target information for the image.
|
228
|
+
- dict[str, Any] - Datum level metadata.
|
229
|
+
"""
|
230
|
+
|
231
|
+
SegmentationDataset: TypeAlias = AnnotatedDataset[SegmentationDatum]
|
232
|
+
"""
|
233
|
+
A type definition for an :class:`AnnotatedDataset` of :class:`SegmentationDatum` elements.
|
234
|
+
"""
|
dataeval/utils/__init__.py
CHANGED
@@ -4,6 +4,6 @@ in setting up data and architectures that are guaranteed to work with applicable
|
|
4
4
|
DataEval metrics.
|
5
5
|
"""
|
6
6
|
|
7
|
-
__all__ = ["
|
7
|
+
__all__ = ["data", "metadata", "torch"]
|
8
8
|
|
9
|
-
from
|
9
|
+
from . import data, metadata, torch
|