dataeval 0.69.3__py3-none-any.whl → 0.70.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataeval/__init__.py +3 -3
- dataeval/_internal/datasets.py +300 -0
- dataeval/_internal/detectors/drift/base.py +5 -6
- dataeval/_internal/detectors/drift/mmd.py +3 -3
- dataeval/_internal/detectors/duplicates.py +62 -45
- dataeval/_internal/detectors/merged_stats.py +23 -54
- dataeval/_internal/detectors/ood/ae.py +3 -3
- dataeval/_internal/detectors/outliers.py +133 -61
- dataeval/_internal/interop.py +11 -7
- dataeval/_internal/metrics/balance.py +9 -9
- dataeval/_internal/metrics/ber.py +3 -3
- dataeval/_internal/metrics/divergence.py +3 -3
- dataeval/_internal/metrics/diversity.py +6 -6
- dataeval/_internal/metrics/parity.py +24 -16
- dataeval/_internal/metrics/stats/base.py +231 -0
- dataeval/_internal/metrics/stats/boxratiostats.py +159 -0
- dataeval/_internal/metrics/stats/datasetstats.py +97 -0
- dataeval/_internal/metrics/stats/dimensionstats.py +111 -0
- dataeval/_internal/metrics/stats/hashstats.py +73 -0
- dataeval/_internal/metrics/stats/labelstats.py +125 -0
- dataeval/_internal/metrics/stats/pixelstats.py +117 -0
- dataeval/_internal/metrics/stats/visualstats.py +122 -0
- dataeval/_internal/metrics/uap.py +2 -2
- dataeval/_internal/metrics/utils.py +28 -13
- dataeval/_internal/output.py +3 -18
- dataeval/_internal/workflows/sufficiency.py +123 -133
- dataeval/metrics/stats/__init__.py +14 -3
- dataeval/workflows/__init__.py +2 -2
- {dataeval-0.69.3.dist-info → dataeval-0.70.0.dist-info}/METADATA +3 -2
- {dataeval-0.69.3.dist-info → dataeval-0.70.0.dist-info}/RECORD +32 -26
- {dataeval-0.69.3.dist-info → dataeval-0.70.0.dist-info}/WHEEL +1 -1
- dataeval/_internal/flags.py +0 -77
- dataeval/_internal/metrics/stats.py +0 -397
- dataeval/flags/__init__.py +0 -3
- {dataeval-0.69.3.dist-info → dataeval-0.70.0.dist-info}/LICENSE.txt +0 -0
dataeval/__init__.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
__version__ = "0.
|
1
|
+
__version__ = "0.70.0"
|
2
2
|
|
3
3
|
from importlib.util import find_spec
|
4
4
|
|
@@ -7,9 +7,9 @@ _IS_TENSORFLOW_AVAILABLE = find_spec("tensorflow") is not None and find_spec("te
|
|
7
7
|
|
8
8
|
del find_spec
|
9
9
|
|
10
|
-
from . import detectors,
|
10
|
+
from . import detectors, metrics # noqa: E402
|
11
11
|
|
12
|
-
__all__ = ["detectors", "
|
12
|
+
__all__ = ["detectors", "metrics"]
|
13
13
|
|
14
14
|
if _IS_TORCH_AVAILABLE: # pragma: no cover
|
15
15
|
from . import torch, utils, workflows
|
@@ -0,0 +1,300 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import hashlib
|
4
|
+
import os
|
5
|
+
import zipfile
|
6
|
+
from pathlib import Path
|
7
|
+
from typing import Literal
|
8
|
+
from urllib.error import HTTPError, URLError
|
9
|
+
from urllib.request import urlretrieve
|
10
|
+
|
11
|
+
import numpy as np
|
12
|
+
from numpy.typing import NDArray
|
13
|
+
from torch.utils.data import Dataset
|
14
|
+
from torchvision.datasets import CIFAR10, VOCDetection # noqa: F401
|
15
|
+
|
16
|
+
|
17
|
+
def _validate_file(fpath, file_md5, chunk_size=65535):
|
18
|
+
hasher = hashlib.md5()
|
19
|
+
with open(fpath, "rb") as fpath_file:
|
20
|
+
while chunk := fpath_file.read(chunk_size):
|
21
|
+
hasher.update(chunk)
|
22
|
+
return hasher.hexdigest() == file_md5
|
23
|
+
|
24
|
+
|
25
|
+
def _get_file(
|
26
|
+
root: str | Path,
|
27
|
+
fname: str,
|
28
|
+
origin: str,
|
29
|
+
file_md5: str | None = None,
|
30
|
+
):
|
31
|
+
fname = os.fspath(fname) if isinstance(fname, os.PathLike) else fname
|
32
|
+
fpath = os.path.join(root, fname)
|
33
|
+
|
34
|
+
download = False
|
35
|
+
if os.path.exists(fpath):
|
36
|
+
if file_md5 is not None and not _validate_file(fpath, file_md5):
|
37
|
+
download = True
|
38
|
+
else:
|
39
|
+
print("Files already downloaded and verified")
|
40
|
+
else:
|
41
|
+
download = True
|
42
|
+
|
43
|
+
if download:
|
44
|
+
try:
|
45
|
+
error_msg = "URL fetch failure on {}: {} -- {}"
|
46
|
+
try:
|
47
|
+
urlretrieve(origin, fpath)
|
48
|
+
except HTTPError as e:
|
49
|
+
raise Exception(error_msg.format(origin, e.code, e.msg)) from e
|
50
|
+
except URLError as e:
|
51
|
+
raise Exception(error_msg.format(origin, e.errno, e.reason)) from e
|
52
|
+
except (Exception, KeyboardInterrupt):
|
53
|
+
if os.path.exists(fpath):
|
54
|
+
os.remove(fpath)
|
55
|
+
raise
|
56
|
+
|
57
|
+
if os.path.exists(fpath) and file_md5 is not None and not _validate_file(fpath, file_md5):
|
58
|
+
raise ValueError(
|
59
|
+
"Incomplete or corrupted file detected. "
|
60
|
+
f"The md5 file hash does not match the provided value "
|
61
|
+
f"of {file_md5}.",
|
62
|
+
)
|
63
|
+
return fpath
|
64
|
+
|
65
|
+
|
66
|
+
def download_dataset(url: str, root: str | Path, fname: str, md5: str) -> str:
|
67
|
+
"""Code to download mnist and corruptions, originates from tensorflow_datasets (tfds):
|
68
|
+
https://github.com/tensorflow/datasets/blob/master/tensorflow_datasets/image_classification/mnist_corrupted.py
|
69
|
+
"""
|
70
|
+
name, _ = os.path.splitext(fname)
|
71
|
+
folder = os.path.join(root, name)
|
72
|
+
os.makedirs(folder, exist_ok=True)
|
73
|
+
|
74
|
+
path = _get_file(
|
75
|
+
root,
|
76
|
+
fname,
|
77
|
+
origin=url + fname,
|
78
|
+
file_md5=md5,
|
79
|
+
)
|
80
|
+
extract_archive(path, remove_finished=True)
|
81
|
+
return path
|
82
|
+
|
83
|
+
|
84
|
+
def extract_archive(
|
85
|
+
from_path: str | Path,
|
86
|
+
to_path: str | Path | None = None,
|
87
|
+
remove_finished: bool = False,
|
88
|
+
):
|
89
|
+
"""Extract an archive.
|
90
|
+
|
91
|
+
The archive type and a possible compression is automatically detected from the file name.
|
92
|
+
"""
|
93
|
+
from_path = Path(from_path)
|
94
|
+
if not from_path.is_absolute():
|
95
|
+
from_path = from_path.resolve()
|
96
|
+
|
97
|
+
if to_path is None:
|
98
|
+
to_path = os.path.dirname(from_path)
|
99
|
+
|
100
|
+
# Extracting zip
|
101
|
+
with zipfile.ZipFile(from_path, "r", compression=zipfile.ZIP_STORED) as zzip:
|
102
|
+
zzip.extractall(to_path)
|
103
|
+
|
104
|
+
if remove_finished:
|
105
|
+
os.remove(from_path)
|
106
|
+
|
107
|
+
|
108
|
+
class MNIST(Dataset):
|
109
|
+
"""MNIST Dataset and Corruptions.
|
110
|
+
|
111
|
+
Args:
|
112
|
+
root : str | ``pathlib.Path``
|
113
|
+
Root directory of dataset where the ``mnist_c/`` folder exists.
|
114
|
+
train : bool, default True
|
115
|
+
If True, creates dataset from ``train_images.npy`` and ``train_labels.npy``.
|
116
|
+
download : bool, default False
|
117
|
+
If True, downloads the dataset from the internet and puts it in root
|
118
|
+
directory. If dataset is already downloaded, it is not downloaded again.
|
119
|
+
size : int, default -1
|
120
|
+
Limit the dataset size, must be a value greater than 0.
|
121
|
+
unit_interval : bool, default False
|
122
|
+
Shift the data values to the unit interval [0-1].
|
123
|
+
dtype : type | None, default None
|
124
|
+
Change the numpy dtype - data is loaded as np.uint8
|
125
|
+
channels : Literal['channels_first' | 'channels_last'] | None, default None
|
126
|
+
Location of channel axis if desired, default has no channels (N, 28, 28)
|
127
|
+
flatten : bool, default False
|
128
|
+
Flatten data into single dimension (N, 784) - cannot use both channels and flatten,
|
129
|
+
channels takes priority over flatten.
|
130
|
+
normalize : tuple[mean, std] | None, default None
|
131
|
+
Normalize images acorrding to provided mean and standard deviation
|
132
|
+
corruption : Literal['identity' | 'shot_noise' | 'impulse_noise' | 'glass_blur' |
|
133
|
+
'motion_blur' | 'shear' | 'scale' | 'rotate' | 'brightness' | 'translate' | 'stripe' |
|
134
|
+
'fog' | 'spatter' | 'dotted_line' | 'zigzag' | 'canny_edges'] | None, default None
|
135
|
+
The desired corruption style or None.
|
136
|
+
"""
|
137
|
+
|
138
|
+
mirror = "https://zenodo.org/record/3239543/files/"
|
139
|
+
|
140
|
+
resources = ("mnist_c.zip", "4b34b33045869ee6d424616cd3a65da3")
|
141
|
+
|
142
|
+
classes = [
|
143
|
+
"0 - zero",
|
144
|
+
"1 - one",
|
145
|
+
"2 - two",
|
146
|
+
"3 - three",
|
147
|
+
"4 - four",
|
148
|
+
"5 - five",
|
149
|
+
"6 - six",
|
150
|
+
"7 - seven",
|
151
|
+
"8 - eight",
|
152
|
+
"9 - nine",
|
153
|
+
]
|
154
|
+
|
155
|
+
@property
|
156
|
+
def train_labels(self):
|
157
|
+
return self.targets
|
158
|
+
|
159
|
+
@property
|
160
|
+
def test_labels(self):
|
161
|
+
return self.targets
|
162
|
+
|
163
|
+
@property
|
164
|
+
def train_data(self):
|
165
|
+
return self.data
|
166
|
+
|
167
|
+
@property
|
168
|
+
def test_data(self):
|
169
|
+
return self.data
|
170
|
+
|
171
|
+
def __init__(
|
172
|
+
self,
|
173
|
+
root: str | Path,
|
174
|
+
train: bool = True,
|
175
|
+
download: bool = False,
|
176
|
+
size: int = -1,
|
177
|
+
unit_interval: bool = False,
|
178
|
+
dtype: type | None = None,
|
179
|
+
channels: Literal["channels_first", "channels_last"] | None = None,
|
180
|
+
flatten: bool = False,
|
181
|
+
normalize: tuple[float, float] | None = None,
|
182
|
+
corruption: Literal[
|
183
|
+
"identity",
|
184
|
+
"shot_noise",
|
185
|
+
"impulse_noise",
|
186
|
+
"glass_blur",
|
187
|
+
"motion_blur",
|
188
|
+
"shear",
|
189
|
+
"scale",
|
190
|
+
"rotate",
|
191
|
+
"brightness",
|
192
|
+
"translate",
|
193
|
+
"stripe",
|
194
|
+
"fog",
|
195
|
+
"spatter",
|
196
|
+
"dotted_line",
|
197
|
+
"zigzag",
|
198
|
+
"canny_edges",
|
199
|
+
]
|
200
|
+
| None = None,
|
201
|
+
) -> None:
|
202
|
+
if isinstance(root, str):
|
203
|
+
root = os.path.expanduser(root)
|
204
|
+
self.root = root # location of stored dataset
|
205
|
+
self.train = train # training set or test set
|
206
|
+
self.size = size
|
207
|
+
self.unit_interval = unit_interval
|
208
|
+
self.dtype = dtype
|
209
|
+
self.channels = channels
|
210
|
+
self.flatten = flatten
|
211
|
+
self.normalize = normalize
|
212
|
+
|
213
|
+
if corruption is None:
|
214
|
+
corruption = "identity"
|
215
|
+
elif corruption == "identity":
|
216
|
+
print("Identity is not a corrupted dataset but the original MNIST dataset")
|
217
|
+
self.corruption = corruption
|
218
|
+
|
219
|
+
if os.path.exists(self.mnist_folder):
|
220
|
+
print("Files already downloaded and verified")
|
221
|
+
elif download:
|
222
|
+
download_dataset(self.mirror, self.root, self.resources[0], self.resources[1])
|
223
|
+
else:
|
224
|
+
raise RuntimeError("Dataset not found. You can use download=True to download it")
|
225
|
+
|
226
|
+
self.data, self.targets = self._load_data()
|
227
|
+
|
228
|
+
def _load_data(self):
|
229
|
+
image_file = f"{'train' if self.train else 'test'}_images.npy"
|
230
|
+
data = self._read_image_file(os.path.join(self.mnist_folder, image_file))
|
231
|
+
|
232
|
+
label_file = f"{'train' if self.train else 'test'}_labels.npy"
|
233
|
+
targets = self._read_label_file(os.path.join(self.mnist_folder, label_file))
|
234
|
+
|
235
|
+
if self.size >= 1 and self.size >= len(self.classes):
|
236
|
+
final_data = []
|
237
|
+
final_targets = []
|
238
|
+
for label in range(len(self.classes)):
|
239
|
+
indices = np.where(targets == label)[0]
|
240
|
+
selected_indices = indices[: int(self.size / len(self.classes))]
|
241
|
+
final_data.append(data[selected_indices])
|
242
|
+
final_targets.append(targets[selected_indices])
|
243
|
+
data = np.concatenate(final_data)
|
244
|
+
targets = np.concatenate(final_targets)
|
245
|
+
shuffled_indices = np.random.permutation(data.shape[0])
|
246
|
+
data = data[shuffled_indices]
|
247
|
+
targets = targets[shuffled_indices]
|
248
|
+
elif self.size >= 1:
|
249
|
+
data = data[: self.size]
|
250
|
+
targets = targets[: self.size]
|
251
|
+
|
252
|
+
if self.unit_interval:
|
253
|
+
data = data / 255
|
254
|
+
|
255
|
+
if self.normalize:
|
256
|
+
data = (data - self.normalize[0]) / self.normalize[1]
|
257
|
+
|
258
|
+
if self.dtype:
|
259
|
+
data = data.astype(self.dtype)
|
260
|
+
|
261
|
+
if self.channels == "channels_first":
|
262
|
+
data = np.moveaxis(data, -1, 1)
|
263
|
+
elif self.channels is None:
|
264
|
+
data = data[:, :, :, 0]
|
265
|
+
|
266
|
+
if self.flatten and self.channels is None:
|
267
|
+
data = data.reshape(data.shape[0], -1)
|
268
|
+
|
269
|
+
return data, targets
|
270
|
+
|
271
|
+
def __getitem__(self, index: int) -> tuple[NDArray, int]:
|
272
|
+
"""
|
273
|
+
Args:
|
274
|
+
index (int): Index
|
275
|
+
|
276
|
+
Returns:
|
277
|
+
tuple: (image, target) where target is index of the target class.
|
278
|
+
"""
|
279
|
+
img, target = self.data[index], int(self.targets[index])
|
280
|
+
|
281
|
+
return img, target
|
282
|
+
|
283
|
+
def __len__(self) -> int:
|
284
|
+
return len(self.data)
|
285
|
+
|
286
|
+
@property
|
287
|
+
def mnist_folder(self) -> str:
|
288
|
+
return os.path.join(self.root, "mnist_c", self.corruption)
|
289
|
+
|
290
|
+
@property
|
291
|
+
def class_to_idx(self) -> dict[str, int]:
|
292
|
+
return {_class: i for i, _class in enumerate(self.classes)}
|
293
|
+
|
294
|
+
def _read_label_file(self, path: str) -> NDArray:
|
295
|
+
x = np.load(path, allow_pickle=False)
|
296
|
+
return x
|
297
|
+
|
298
|
+
def _read_image_file(self, path: str) -> NDArray:
|
299
|
+
x = np.load(path, allow_pickle=False)
|
300
|
+
return x
|
@@ -16,7 +16,7 @@ from typing import Callable, Literal
|
|
16
16
|
import numpy as np
|
17
17
|
from numpy.typing import ArrayLike, NDArray
|
18
18
|
|
19
|
-
from dataeval._internal.interop import to_numpy
|
19
|
+
from dataeval._internal.interop import as_numpy, to_numpy
|
20
20
|
from dataeval._internal.output import OutputMetadata, set_metadata
|
21
21
|
|
22
22
|
|
@@ -234,7 +234,7 @@ class BaseDrift:
|
|
234
234
|
if correction not in ["bonferroni", "fdr"]:
|
235
235
|
raise ValueError("`correction` must be `bonferroni` or `fdr`.")
|
236
236
|
|
237
|
-
self._x_ref = x_ref
|
237
|
+
self._x_ref = to_numpy(x_ref)
|
238
238
|
self.x_ref_preprocessed = x_ref_preprocessed
|
239
239
|
|
240
240
|
# Other attributes
|
@@ -242,7 +242,7 @@ class BaseDrift:
|
|
242
242
|
self.update_x_ref = update_x_ref
|
243
243
|
self.preprocess_fn = preprocess_fn
|
244
244
|
self.correction = correction
|
245
|
-
self.n = len(self._x_ref)
|
245
|
+
self.n = len(self._x_ref)
|
246
246
|
|
247
247
|
# Ref counter for preprocessed x
|
248
248
|
self._x_refcount = 0
|
@@ -260,9 +260,8 @@ class BaseDrift:
|
|
260
260
|
if not self.x_ref_preprocessed:
|
261
261
|
self.x_ref_preprocessed = True
|
262
262
|
if self.preprocess_fn is not None:
|
263
|
-
self._x_ref = self.preprocess_fn(self._x_ref)
|
263
|
+
self._x_ref = as_numpy(self.preprocess_fn(self._x_ref))
|
264
264
|
|
265
|
-
self._x_ref = to_numpy(self._x_ref)
|
266
265
|
return self._x_ref
|
267
266
|
|
268
267
|
def _preprocess(self, x: ArrayLike) -> ArrayLike:
|
@@ -380,7 +379,7 @@ class BaseDriftUnivariate(BaseDrift):
|
|
380
379
|
self._n_features = self.x_ref.reshape(self.x_ref.shape[0], -1).shape[-1]
|
381
380
|
else:
|
382
381
|
# infer number of features after applying preprocessing step
|
383
|
-
x =
|
382
|
+
x = as_numpy(self.preprocess_fn(self._x_ref[0:1])) # type: ignore
|
384
383
|
self._n_features = x.reshape(x.shape[0], -1).shape[-1]
|
385
384
|
|
386
385
|
return self._n_features
|
@@ -14,7 +14,7 @@ from typing import Callable
|
|
14
14
|
import torch
|
15
15
|
from numpy.typing import ArrayLike
|
16
16
|
|
17
|
-
from dataeval._internal.interop import
|
17
|
+
from dataeval._internal.interop import as_numpy
|
18
18
|
from dataeval._internal.output import set_metadata
|
19
19
|
|
20
20
|
from .base import BaseDrift, DriftBaseOutput, UpdateStrategy, preprocess_x, update_x_ref
|
@@ -110,7 +110,7 @@ class DriftMMD(BaseDrift):
|
|
110
110
|
self.device = get_device(device)
|
111
111
|
|
112
112
|
# initialize kernel
|
113
|
-
sigma_tensor = torch.from_numpy(
|
113
|
+
sigma_tensor = torch.from_numpy(as_numpy(sigma)).to(self.device) if sigma is not None else None
|
114
114
|
self.kernel = kernel(sigma_tensor).to(self.device) if kernel == GaussianRBF else kernel
|
115
115
|
|
116
116
|
# compute kernel matrix for the reference data
|
@@ -147,7 +147,7 @@ class DriftMMD(BaseDrift):
|
|
147
147
|
p-value obtained from the permutation test, MMD^2 between the reference and test set,
|
148
148
|
and MMD^2 threshold above which drift is flagged
|
149
149
|
"""
|
150
|
-
x =
|
150
|
+
x = as_numpy(x)
|
151
151
|
x_ref = torch.from_numpy(self.x_ref).to(self.device)
|
152
152
|
n = x.shape[0]
|
153
153
|
kernel_mat = self._kernel_matrix(x_ref, torch.from_numpy(x).to(self.device))
|
@@ -1,13 +1,12 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
3
|
from dataclasses import dataclass
|
4
|
-
from typing import Generic, Iterable, Sequence, TypeVar
|
4
|
+
from typing import Generic, Iterable, Sequence, TypeVar
|
5
5
|
|
6
6
|
from numpy.typing import ArrayLike
|
7
7
|
|
8
8
|
from dataeval._internal.detectors.merged_stats import combine_stats, get_dataset_step_from_idx
|
9
|
-
from dataeval._internal.
|
10
|
-
from dataeval._internal.metrics.stats import StatsOutput, imagestats
|
9
|
+
from dataeval._internal.metrics.stats.hashstats import HashStatsOutput, hashstats
|
11
10
|
from dataeval._internal.output import OutputMetadata, set_metadata
|
12
11
|
|
13
12
|
DuplicateGroup = list[int]
|
@@ -53,26 +52,23 @@ class Duplicates:
|
|
53
52
|
-------
|
54
53
|
Initialize the Duplicates class:
|
55
54
|
|
56
|
-
>>>
|
55
|
+
>>> all_dupes = Duplicates()
|
56
|
+
>>> exact_dupes = Duplicates(only_exact=True)
|
57
57
|
"""
|
58
58
|
|
59
59
|
def __init__(self, only_exact: bool = False):
|
60
|
-
self.stats:
|
60
|
+
self.stats: HashStatsOutput
|
61
61
|
self.only_exact = only_exact
|
62
62
|
|
63
|
-
def _get_duplicates(self) -> dict[str, list[list[int]]]:
|
64
|
-
|
65
|
-
|
66
|
-
exact_dict
|
67
|
-
|
68
|
-
exact_dict.setdefault(value, []).append(i)
|
69
|
-
exact = [sorted(v) for v in exact_dict.values() if len(v) > 1]
|
70
|
-
else:
|
71
|
-
exact = []
|
63
|
+
def _get_duplicates(self, stats: dict) -> dict[str, list[list[int]]]:
|
64
|
+
exact_dict: dict[int, list] = {}
|
65
|
+
for i, value in enumerate(stats["xxhash"]):
|
66
|
+
exact_dict.setdefault(value, []).append(i)
|
67
|
+
exact = [sorted(v) for v in exact_dict.values() if len(v) > 1]
|
72
68
|
|
73
|
-
if
|
69
|
+
if not self.only_exact:
|
74
70
|
near_dict: dict[int, list] = {}
|
75
|
-
for i, value in enumerate(
|
71
|
+
for i, value in enumerate(stats["pchash"]):
|
76
72
|
near_dict.setdefault(value, []).append(i)
|
77
73
|
near = [sorted(v) for v in near_dict.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
|
78
74
|
else:
|
@@ -84,14 +80,14 @@ class Duplicates:
|
|
84
80
|
}
|
85
81
|
|
86
82
|
@set_metadata("dataeval.detectors", ["only_exact"])
|
87
|
-
def
|
83
|
+
def from_stats(self, hashes: HashStatsOutput | Sequence[HashStatsOutput]) -> DuplicatesOutput:
|
88
84
|
"""
|
89
85
|
Returns duplicate image indices for both exact matches and near matches
|
90
86
|
|
91
87
|
Parameters
|
92
88
|
----------
|
93
|
-
data :
|
94
|
-
|
89
|
+
data : HashStatsOutput | Sequence[HashStatsOutput]
|
90
|
+
The output(s) from a hashstats analysis
|
95
91
|
|
96
92
|
Returns
|
97
93
|
-------
|
@@ -100,39 +96,60 @@ class Duplicates:
|
|
100
96
|
|
101
97
|
See Also
|
102
98
|
--------
|
103
|
-
|
99
|
+
hashstats
|
104
100
|
|
105
101
|
Example
|
106
102
|
-------
|
107
|
-
>>>
|
108
|
-
DuplicatesOutput(exact=[[3, 20], [16
|
109
|
-
"""
|
103
|
+
>>> exact_dupes.from_stats([hashes1, hashes2])
|
104
|
+
DuplicatesOutput(exact=[{0: [3, 20]}, {0: [16], 1: [12]}], near=[])
|
105
|
+
"""
|
110
106
|
|
111
|
-
|
107
|
+
if isinstance(hashes, HashStatsOutput):
|
108
|
+
return DuplicatesOutput(**self._get_duplicates(hashes.dict()))
|
112
109
|
|
113
|
-
if isinstance(
|
114
|
-
|
115
|
-
raise ValueError("StatsOutput must include xxhash information of the images.")
|
116
|
-
if not self.only_exact and not stats.pchash:
|
117
|
-
raise ValueError("StatsOutput must include pchash information of the images for near matches.")
|
118
|
-
self.stats = stats
|
119
|
-
else:
|
120
|
-
flags = ImageStat.XXHASH | (ImageStat(0) if self.only_exact else ImageStat.PCHASH)
|
121
|
-
self.stats = imagestats(cast(Iterable[ArrayLike], data), flags)
|
110
|
+
if not isinstance(hashes, Sequence):
|
111
|
+
raise TypeError("Invalid stats output type; only use output from hashstats.")
|
122
112
|
|
123
|
-
|
113
|
+
combined, dataset_steps = combine_stats(hashes)
|
114
|
+
duplicates = self._get_duplicates(combined.dict())
|
124
115
|
|
125
116
|
# split up results from combined dataset into individual dataset buckets
|
126
|
-
|
127
|
-
|
128
|
-
for
|
129
|
-
|
130
|
-
for
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
117
|
+
for dup_type, dup_list in duplicates.items():
|
118
|
+
dup_list_dict = []
|
119
|
+
for idxs in dup_list:
|
120
|
+
dup_dict = {}
|
121
|
+
for idx in idxs:
|
122
|
+
k, v = get_dataset_step_from_idx(idx, dataset_steps)
|
123
|
+
dup_dict.setdefault(k, []).append(v)
|
124
|
+
dup_list_dict.append(dup_dict)
|
125
|
+
duplicates[dup_type] = dup_list_dict
|
126
|
+
|
127
|
+
return DuplicatesOutput(**duplicates)
|
128
|
+
|
129
|
+
@set_metadata("dataeval.detectors", ["only_exact"])
|
130
|
+
def evaluate(self, data: Iterable[ArrayLike]) -> DuplicatesOutput:
|
131
|
+
"""
|
132
|
+
Returns duplicate image indices for both exact matches and near matches
|
133
|
+
|
134
|
+
Parameters
|
135
|
+
----------
|
136
|
+
data : Iterable[ArrayLike], shape - (N, C, H, W) | StatsOutput | Sequence[StatsOutput]
|
137
|
+
A dataset of images in an ArrayLike format or the output(s) from a hashstats analysis
|
138
|
+
|
139
|
+
Returns
|
140
|
+
-------
|
141
|
+
DuplicatesOutput
|
142
|
+
List of groups of indices that are exact and near matches
|
137
143
|
|
144
|
+
See Also
|
145
|
+
--------
|
146
|
+
hashstats
|
147
|
+
|
148
|
+
Example
|
149
|
+
-------
|
150
|
+
>>> all_dupes.evaluate(images)
|
151
|
+
DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
|
152
|
+
""" # noqa: E501
|
153
|
+
self.stats = hashstats(data)
|
154
|
+
duplicates = self._get_duplicates(self.stats.dict())
|
138
155
|
return DuplicatesOutput(**duplicates)
|
@@ -1,71 +1,40 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
from
|
4
|
-
from
|
3
|
+
from copy import deepcopy
|
4
|
+
from typing import Sequence, TypeVar
|
5
5
|
|
6
6
|
import numpy as np
|
7
7
|
|
8
|
-
from dataeval._internal.metrics.stats import
|
9
|
-
from dataeval._internal.output import populate_defaults
|
8
|
+
from dataeval._internal.metrics.stats.base import BaseStatsOutput
|
10
9
|
|
10
|
+
TStatsOutput = TypeVar("TStatsOutput", bound=BaseStatsOutput)
|
11
11
|
|
12
|
-
def add_stats(a: StatsOutput, b: StatsOutput) -> StatsOutput:
|
13
|
-
if not isinstance(a, StatsOutput) or not isinstance(b, StatsOutput):
|
14
|
-
raise TypeError(f"Cannot add object of type {type(a)} and type {type(b)}.")
|
15
12
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
b_keys = set(b_dict)
|
13
|
+
def add_stats(a: TStatsOutput, b: TStatsOutput) -> TStatsOutput:
|
14
|
+
if type(a) is not type(b):
|
15
|
+
raise TypeError(f"Types {type(a)} and {type(b)} cannot be added.")
|
20
16
|
|
21
|
-
|
22
|
-
if missing_keys:
|
23
|
-
raise ValueError(f"Required keys are missing: {missing_keys}.")
|
17
|
+
sum_dict = deepcopy(a.dict())
|
24
18
|
|
25
|
-
|
26
|
-
|
27
|
-
|
19
|
+
for k in sum_dict:
|
20
|
+
if isinstance(sum_dict[k], list):
|
21
|
+
sum_dict[k].extend(b.dict()[k])
|
22
|
+
else:
|
23
|
+
sum_dict[k] = np.concatenate((sum_dict[k], b.dict()[k]))
|
28
24
|
|
29
|
-
|
30
|
-
if "ch_idx_map" in a_dict:
|
31
|
-
for k, v in a_dict.items():
|
32
|
-
if k == "ch_idx_map":
|
33
|
-
offset = sum([len(idxs) for idxs in v.values()])
|
34
|
-
for ch_k, ch_v in b_dict[k].items():
|
35
|
-
if ch_k not in v:
|
36
|
-
v[ch_k] = []
|
37
|
-
a_dict[k][ch_k].extend([idx + offset for idx in ch_v])
|
38
|
-
else:
|
39
|
-
for ch_k in b_dict[k]:
|
40
|
-
if ch_k not in v:
|
41
|
-
v[ch_k] = b_dict[k][ch_k]
|
42
|
-
else:
|
43
|
-
v[ch_k] = np.concatenate((v[ch_k], b_dict[k][ch_k]), axis=1)
|
44
|
-
else:
|
45
|
-
for k in a_dict:
|
46
|
-
if isinstance(a_dict[k], list):
|
47
|
-
a_dict[k].extend(b_dict[k])
|
48
|
-
else:
|
49
|
-
a_dict[k] = np.concatenate((a_dict[k], b_dict[k]))
|
25
|
+
return type(a)(**sum_dict)
|
50
26
|
|
51
|
-
return StatsOutput(**populate_defaults(a_dict, StatsOutput))
|
52
|
-
|
53
|
-
|
54
|
-
def combine_stats(stats) -> tuple[StatsOutput | None, list[int]]:
|
55
|
-
dataset_steps = []
|
56
|
-
|
57
|
-
if isinstance(stats, StatsOutput):
|
58
|
-
return stats, dataset_steps
|
59
27
|
|
28
|
+
def combine_stats(stats: Sequence[TStatsOutput]) -> tuple[TStatsOutput, list[int]]:
|
60
29
|
output = None
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
30
|
+
dataset_steps = []
|
31
|
+
cur_len = 0
|
32
|
+
for s in stats:
|
33
|
+
output = s if output is None else add_stats(output, s)
|
34
|
+
cur_len += len(s)
|
35
|
+
dataset_steps.append(cur_len)
|
36
|
+
if output is None:
|
37
|
+
raise TypeError("Cannot combine empty sequence of stats.")
|
69
38
|
return output, dataset_steps
|
70
39
|
|
71
40
|
|
@@ -16,7 +16,7 @@ import tensorflow as tf
|
|
16
16
|
from numpy.typing import ArrayLike
|
17
17
|
|
18
18
|
from dataeval._internal.detectors.ood.base import OODBase, OODScore
|
19
|
-
from dataeval._internal.interop import
|
19
|
+
from dataeval._internal.interop import as_numpy
|
20
20
|
from dataeval._internal.models.tensorflow.autoencoder import AE
|
21
21
|
from dataeval._internal.models.tensorflow.utils import predict_batch
|
22
22
|
|
@@ -46,10 +46,10 @@ class OOD_AE(OODBase):
|
|
46
46
|
) -> None:
|
47
47
|
if loss_fn is None:
|
48
48
|
loss_fn = keras.losses.MeanSquaredError()
|
49
|
-
super().fit(
|
49
|
+
super().fit(as_numpy(x_ref), threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
|
50
50
|
|
51
51
|
def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
|
52
|
-
self._validate(X :=
|
52
|
+
self._validate(X := as_numpy(X))
|
53
53
|
|
54
54
|
# reconstruct instances
|
55
55
|
X_recon = predict_batch(X, self.model, batch_size=batch_size)
|