dataeval 0.69.3__py3-none-any.whl → 0.70.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. dataeval/__init__.py +3 -3
  2. dataeval/_internal/datasets.py +300 -0
  3. dataeval/_internal/detectors/drift/base.py +5 -6
  4. dataeval/_internal/detectors/drift/mmd.py +3 -3
  5. dataeval/_internal/detectors/duplicates.py +62 -45
  6. dataeval/_internal/detectors/merged_stats.py +23 -54
  7. dataeval/_internal/detectors/ood/ae.py +3 -3
  8. dataeval/_internal/detectors/outliers.py +133 -61
  9. dataeval/_internal/interop.py +11 -7
  10. dataeval/_internal/metrics/balance.py +9 -9
  11. dataeval/_internal/metrics/ber.py +3 -3
  12. dataeval/_internal/metrics/divergence.py +3 -3
  13. dataeval/_internal/metrics/diversity.py +6 -6
  14. dataeval/_internal/metrics/parity.py +24 -16
  15. dataeval/_internal/metrics/stats/base.py +231 -0
  16. dataeval/_internal/metrics/stats/boxratiostats.py +159 -0
  17. dataeval/_internal/metrics/stats/datasetstats.py +97 -0
  18. dataeval/_internal/metrics/stats/dimensionstats.py +111 -0
  19. dataeval/_internal/metrics/stats/hashstats.py +73 -0
  20. dataeval/_internal/metrics/stats/labelstats.py +125 -0
  21. dataeval/_internal/metrics/stats/pixelstats.py +117 -0
  22. dataeval/_internal/metrics/stats/visualstats.py +122 -0
  23. dataeval/_internal/metrics/uap.py +2 -2
  24. dataeval/_internal/metrics/utils.py +28 -13
  25. dataeval/_internal/output.py +3 -18
  26. dataeval/_internal/workflows/sufficiency.py +123 -133
  27. dataeval/metrics/stats/__init__.py +14 -3
  28. dataeval/workflows/__init__.py +2 -2
  29. {dataeval-0.69.3.dist-info → dataeval-0.70.0.dist-info}/METADATA +3 -2
  30. {dataeval-0.69.3.dist-info → dataeval-0.70.0.dist-info}/RECORD +32 -26
  31. {dataeval-0.69.3.dist-info → dataeval-0.70.0.dist-info}/WHEEL +1 -1
  32. dataeval/_internal/flags.py +0 -77
  33. dataeval/_internal/metrics/stats.py +0 -397
  34. dataeval/flags/__init__.py +0 -3
  35. {dataeval-0.69.3.dist-info → dataeval-0.70.0.dist-info}/LICENSE.txt +0 -0
dataeval/__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- __version__ = "0.69.3"
1
+ __version__ = "0.70.0"
2
2
 
3
3
  from importlib.util import find_spec
4
4
 
@@ -7,9 +7,9 @@ _IS_TENSORFLOW_AVAILABLE = find_spec("tensorflow") is not None and find_spec("te
7
7
 
8
8
  del find_spec
9
9
 
10
- from . import detectors, flags, metrics # noqa: E402
10
+ from . import detectors, metrics # noqa: E402
11
11
 
12
- __all__ = ["detectors", "flags", "metrics"]
12
+ __all__ = ["detectors", "metrics"]
13
13
 
14
14
  if _IS_TORCH_AVAILABLE: # pragma: no cover
15
15
  from . import torch, utils, workflows
@@ -0,0 +1,300 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import os
5
+ import zipfile
6
+ from pathlib import Path
7
+ from typing import Literal
8
+ from urllib.error import HTTPError, URLError
9
+ from urllib.request import urlretrieve
10
+
11
+ import numpy as np
12
+ from numpy.typing import NDArray
13
+ from torch.utils.data import Dataset
14
+ from torchvision.datasets import CIFAR10, VOCDetection # noqa: F401
15
+
16
+
17
+ def _validate_file(fpath, file_md5, chunk_size=65535):
18
+ hasher = hashlib.md5()
19
+ with open(fpath, "rb") as fpath_file:
20
+ while chunk := fpath_file.read(chunk_size):
21
+ hasher.update(chunk)
22
+ return hasher.hexdigest() == file_md5
23
+
24
+
25
+ def _get_file(
26
+ root: str | Path,
27
+ fname: str,
28
+ origin: str,
29
+ file_md5: str | None = None,
30
+ ):
31
+ fname = os.fspath(fname) if isinstance(fname, os.PathLike) else fname
32
+ fpath = os.path.join(root, fname)
33
+
34
+ download = False
35
+ if os.path.exists(fpath):
36
+ if file_md5 is not None and not _validate_file(fpath, file_md5):
37
+ download = True
38
+ else:
39
+ print("Files already downloaded and verified")
40
+ else:
41
+ download = True
42
+
43
+ if download:
44
+ try:
45
+ error_msg = "URL fetch failure on {}: {} -- {}"
46
+ try:
47
+ urlretrieve(origin, fpath)
48
+ except HTTPError as e:
49
+ raise Exception(error_msg.format(origin, e.code, e.msg)) from e
50
+ except URLError as e:
51
+ raise Exception(error_msg.format(origin, e.errno, e.reason)) from e
52
+ except (Exception, KeyboardInterrupt):
53
+ if os.path.exists(fpath):
54
+ os.remove(fpath)
55
+ raise
56
+
57
+ if os.path.exists(fpath) and file_md5 is not None and not _validate_file(fpath, file_md5):
58
+ raise ValueError(
59
+ "Incomplete or corrupted file detected. "
60
+ f"The md5 file hash does not match the provided value "
61
+ f"of {file_md5}.",
62
+ )
63
+ return fpath
64
+
65
+
66
+ def download_dataset(url: str, root: str | Path, fname: str, md5: str) -> str:
67
+ """Code to download mnist and corruptions, originates from tensorflow_datasets (tfds):
68
+ https://github.com/tensorflow/datasets/blob/master/tensorflow_datasets/image_classification/mnist_corrupted.py
69
+ """
70
+ name, _ = os.path.splitext(fname)
71
+ folder = os.path.join(root, name)
72
+ os.makedirs(folder, exist_ok=True)
73
+
74
+ path = _get_file(
75
+ root,
76
+ fname,
77
+ origin=url + fname,
78
+ file_md5=md5,
79
+ )
80
+ extract_archive(path, remove_finished=True)
81
+ return path
82
+
83
+
84
+ def extract_archive(
85
+ from_path: str | Path,
86
+ to_path: str | Path | None = None,
87
+ remove_finished: bool = False,
88
+ ):
89
+ """Extract an archive.
90
+
91
+ The archive type and a possible compression is automatically detected from the file name.
92
+ """
93
+ from_path = Path(from_path)
94
+ if not from_path.is_absolute():
95
+ from_path = from_path.resolve()
96
+
97
+ if to_path is None:
98
+ to_path = os.path.dirname(from_path)
99
+
100
+ # Extracting zip
101
+ with zipfile.ZipFile(from_path, "r", compression=zipfile.ZIP_STORED) as zzip:
102
+ zzip.extractall(to_path)
103
+
104
+ if remove_finished:
105
+ os.remove(from_path)
106
+
107
+
108
+ class MNIST(Dataset):
109
+ """MNIST Dataset and Corruptions.
110
+
111
+ Args:
112
+ root : str | ``pathlib.Path``
113
+ Root directory of dataset where the ``mnist_c/`` folder exists.
114
+ train : bool, default True
115
+ If True, creates dataset from ``train_images.npy`` and ``train_labels.npy``.
116
+ download : bool, default False
117
+ If True, downloads the dataset from the internet and puts it in root
118
+ directory. If dataset is already downloaded, it is not downloaded again.
119
+ size : int, default -1
120
+ Limit the dataset size, must be a value greater than 0.
121
+ unit_interval : bool, default False
122
+ Shift the data values to the unit interval [0-1].
123
+ dtype : type | None, default None
124
+ Change the numpy dtype - data is loaded as np.uint8
125
+ channels : Literal['channels_first' | 'channels_last'] | None, default None
126
+ Location of channel axis if desired, default has no channels (N, 28, 28)
127
+ flatten : bool, default False
128
+ Flatten data into single dimension (N, 784) - cannot use both channels and flatten,
129
+ channels takes priority over flatten.
130
+ normalize : tuple[mean, std] | None, default None
131
+ Normalize images acorrding to provided mean and standard deviation
132
+ corruption : Literal['identity' | 'shot_noise' | 'impulse_noise' | 'glass_blur' |
133
+ 'motion_blur' | 'shear' | 'scale' | 'rotate' | 'brightness' | 'translate' | 'stripe' |
134
+ 'fog' | 'spatter' | 'dotted_line' | 'zigzag' | 'canny_edges'] | None, default None
135
+ The desired corruption style or None.
136
+ """
137
+
138
+ mirror = "https://zenodo.org/record/3239543/files/"
139
+
140
+ resources = ("mnist_c.zip", "4b34b33045869ee6d424616cd3a65da3")
141
+
142
+ classes = [
143
+ "0 - zero",
144
+ "1 - one",
145
+ "2 - two",
146
+ "3 - three",
147
+ "4 - four",
148
+ "5 - five",
149
+ "6 - six",
150
+ "7 - seven",
151
+ "8 - eight",
152
+ "9 - nine",
153
+ ]
154
+
155
+ @property
156
+ def train_labels(self):
157
+ return self.targets
158
+
159
+ @property
160
+ def test_labels(self):
161
+ return self.targets
162
+
163
+ @property
164
+ def train_data(self):
165
+ return self.data
166
+
167
+ @property
168
+ def test_data(self):
169
+ return self.data
170
+
171
+ def __init__(
172
+ self,
173
+ root: str | Path,
174
+ train: bool = True,
175
+ download: bool = False,
176
+ size: int = -1,
177
+ unit_interval: bool = False,
178
+ dtype: type | None = None,
179
+ channels: Literal["channels_first", "channels_last"] | None = None,
180
+ flatten: bool = False,
181
+ normalize: tuple[float, float] | None = None,
182
+ corruption: Literal[
183
+ "identity",
184
+ "shot_noise",
185
+ "impulse_noise",
186
+ "glass_blur",
187
+ "motion_blur",
188
+ "shear",
189
+ "scale",
190
+ "rotate",
191
+ "brightness",
192
+ "translate",
193
+ "stripe",
194
+ "fog",
195
+ "spatter",
196
+ "dotted_line",
197
+ "zigzag",
198
+ "canny_edges",
199
+ ]
200
+ | None = None,
201
+ ) -> None:
202
+ if isinstance(root, str):
203
+ root = os.path.expanduser(root)
204
+ self.root = root # location of stored dataset
205
+ self.train = train # training set or test set
206
+ self.size = size
207
+ self.unit_interval = unit_interval
208
+ self.dtype = dtype
209
+ self.channels = channels
210
+ self.flatten = flatten
211
+ self.normalize = normalize
212
+
213
+ if corruption is None:
214
+ corruption = "identity"
215
+ elif corruption == "identity":
216
+ print("Identity is not a corrupted dataset but the original MNIST dataset")
217
+ self.corruption = corruption
218
+
219
+ if os.path.exists(self.mnist_folder):
220
+ print("Files already downloaded and verified")
221
+ elif download:
222
+ download_dataset(self.mirror, self.root, self.resources[0], self.resources[1])
223
+ else:
224
+ raise RuntimeError("Dataset not found. You can use download=True to download it")
225
+
226
+ self.data, self.targets = self._load_data()
227
+
228
+ def _load_data(self):
229
+ image_file = f"{'train' if self.train else 'test'}_images.npy"
230
+ data = self._read_image_file(os.path.join(self.mnist_folder, image_file))
231
+
232
+ label_file = f"{'train' if self.train else 'test'}_labels.npy"
233
+ targets = self._read_label_file(os.path.join(self.mnist_folder, label_file))
234
+
235
+ if self.size >= 1 and self.size >= len(self.classes):
236
+ final_data = []
237
+ final_targets = []
238
+ for label in range(len(self.classes)):
239
+ indices = np.where(targets == label)[0]
240
+ selected_indices = indices[: int(self.size / len(self.classes))]
241
+ final_data.append(data[selected_indices])
242
+ final_targets.append(targets[selected_indices])
243
+ data = np.concatenate(final_data)
244
+ targets = np.concatenate(final_targets)
245
+ shuffled_indices = np.random.permutation(data.shape[0])
246
+ data = data[shuffled_indices]
247
+ targets = targets[shuffled_indices]
248
+ elif self.size >= 1:
249
+ data = data[: self.size]
250
+ targets = targets[: self.size]
251
+
252
+ if self.unit_interval:
253
+ data = data / 255
254
+
255
+ if self.normalize:
256
+ data = (data - self.normalize[0]) / self.normalize[1]
257
+
258
+ if self.dtype:
259
+ data = data.astype(self.dtype)
260
+
261
+ if self.channels == "channels_first":
262
+ data = np.moveaxis(data, -1, 1)
263
+ elif self.channels is None:
264
+ data = data[:, :, :, 0]
265
+
266
+ if self.flatten and self.channels is None:
267
+ data = data.reshape(data.shape[0], -1)
268
+
269
+ return data, targets
270
+
271
+ def __getitem__(self, index: int) -> tuple[NDArray, int]:
272
+ """
273
+ Args:
274
+ index (int): Index
275
+
276
+ Returns:
277
+ tuple: (image, target) where target is index of the target class.
278
+ """
279
+ img, target = self.data[index], int(self.targets[index])
280
+
281
+ return img, target
282
+
283
+ def __len__(self) -> int:
284
+ return len(self.data)
285
+
286
+ @property
287
+ def mnist_folder(self) -> str:
288
+ return os.path.join(self.root, "mnist_c", self.corruption)
289
+
290
+ @property
291
+ def class_to_idx(self) -> dict[str, int]:
292
+ return {_class: i for i, _class in enumerate(self.classes)}
293
+
294
+ def _read_label_file(self, path: str) -> NDArray:
295
+ x = np.load(path, allow_pickle=False)
296
+ return x
297
+
298
+ def _read_image_file(self, path: str) -> NDArray:
299
+ x = np.load(path, allow_pickle=False)
300
+ return x
@@ -16,7 +16,7 @@ from typing import Callable, Literal
16
16
  import numpy as np
17
17
  from numpy.typing import ArrayLike, NDArray
18
18
 
19
- from dataeval._internal.interop import to_numpy
19
+ from dataeval._internal.interop import as_numpy, to_numpy
20
20
  from dataeval._internal.output import OutputMetadata, set_metadata
21
21
 
22
22
 
@@ -234,7 +234,7 @@ class BaseDrift:
234
234
  if correction not in ["bonferroni", "fdr"]:
235
235
  raise ValueError("`correction` must be `bonferroni` or `fdr`.")
236
236
 
237
- self._x_ref = x_ref
237
+ self._x_ref = to_numpy(x_ref)
238
238
  self.x_ref_preprocessed = x_ref_preprocessed
239
239
 
240
240
  # Other attributes
@@ -242,7 +242,7 @@ class BaseDrift:
242
242
  self.update_x_ref = update_x_ref
243
243
  self.preprocess_fn = preprocess_fn
244
244
  self.correction = correction
245
- self.n = len(self._x_ref) # type: ignore
245
+ self.n = len(self._x_ref)
246
246
 
247
247
  # Ref counter for preprocessed x
248
248
  self._x_refcount = 0
@@ -260,9 +260,8 @@ class BaseDrift:
260
260
  if not self.x_ref_preprocessed:
261
261
  self.x_ref_preprocessed = True
262
262
  if self.preprocess_fn is not None:
263
- self._x_ref = self.preprocess_fn(self._x_ref)
263
+ self._x_ref = as_numpy(self.preprocess_fn(self._x_ref))
264
264
 
265
- self._x_ref = to_numpy(self._x_ref)
266
265
  return self._x_ref
267
266
 
268
267
  def _preprocess(self, x: ArrayLike) -> ArrayLike:
@@ -380,7 +379,7 @@ class BaseDriftUnivariate(BaseDrift):
380
379
  self._n_features = self.x_ref.reshape(self.x_ref.shape[0], -1).shape[-1]
381
380
  else:
382
381
  # infer number of features after applying preprocessing step
383
- x = to_numpy(self.preprocess_fn(self._x_ref[0:1])) # type: ignore
382
+ x = as_numpy(self.preprocess_fn(self._x_ref[0:1])) # type: ignore
384
383
  self._n_features = x.reshape(x.shape[0], -1).shape[-1]
385
384
 
386
385
  return self._n_features
@@ -14,7 +14,7 @@ from typing import Callable
14
14
  import torch
15
15
  from numpy.typing import ArrayLike
16
16
 
17
- from dataeval._internal.interop import to_numpy
17
+ from dataeval._internal.interop import as_numpy
18
18
  from dataeval._internal.output import set_metadata
19
19
 
20
20
  from .base import BaseDrift, DriftBaseOutput, UpdateStrategy, preprocess_x, update_x_ref
@@ -110,7 +110,7 @@ class DriftMMD(BaseDrift):
110
110
  self.device = get_device(device)
111
111
 
112
112
  # initialize kernel
113
- sigma_tensor = torch.from_numpy(to_numpy(sigma)).to(self.device) if sigma is not None else None
113
+ sigma_tensor = torch.from_numpy(as_numpy(sigma)).to(self.device) if sigma is not None else None
114
114
  self.kernel = kernel(sigma_tensor).to(self.device) if kernel == GaussianRBF else kernel
115
115
 
116
116
  # compute kernel matrix for the reference data
@@ -147,7 +147,7 @@ class DriftMMD(BaseDrift):
147
147
  p-value obtained from the permutation test, MMD^2 between the reference and test set,
148
148
  and MMD^2 threshold above which drift is flagged
149
149
  """
150
- x = to_numpy(x)
150
+ x = as_numpy(x)
151
151
  x_ref = torch.from_numpy(self.x_ref).to(self.device)
152
152
  n = x.shape[0]
153
153
  kernel_mat = self._kernel_matrix(x_ref, torch.from_numpy(x).to(self.device))
@@ -1,13 +1,12 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  from dataclasses import dataclass
4
- from typing import Generic, Iterable, Sequence, TypeVar, cast
4
+ from typing import Generic, Iterable, Sequence, TypeVar
5
5
 
6
6
  from numpy.typing import ArrayLike
7
7
 
8
8
  from dataeval._internal.detectors.merged_stats import combine_stats, get_dataset_step_from_idx
9
- from dataeval._internal.flags import ImageStat
10
- from dataeval._internal.metrics.stats import StatsOutput, imagestats
9
+ from dataeval._internal.metrics.stats.hashstats import HashStatsOutput, hashstats
11
10
  from dataeval._internal.output import OutputMetadata, set_metadata
12
11
 
13
12
  DuplicateGroup = list[int]
@@ -53,26 +52,23 @@ class Duplicates:
53
52
  -------
54
53
  Initialize the Duplicates class:
55
54
 
56
- >>> dups = Duplicates()
55
+ >>> all_dupes = Duplicates()
56
+ >>> exact_dupes = Duplicates(only_exact=True)
57
57
  """
58
58
 
59
59
  def __init__(self, only_exact: bool = False):
60
- self.stats: StatsOutput
60
+ self.stats: HashStatsOutput
61
61
  self.only_exact = only_exact
62
62
 
63
- def _get_duplicates(self) -> dict[str, list[list[int]]]:
64
- stats_dict = self.stats.dict()
65
- if "xxhash" in stats_dict:
66
- exact_dict: dict[int, list] = {}
67
- for i, value in enumerate(stats_dict["xxhash"]):
68
- exact_dict.setdefault(value, []).append(i)
69
- exact = [sorted(v) for v in exact_dict.values() if len(v) > 1]
70
- else:
71
- exact = []
63
+ def _get_duplicates(self, stats: dict) -> dict[str, list[list[int]]]:
64
+ exact_dict: dict[int, list] = {}
65
+ for i, value in enumerate(stats["xxhash"]):
66
+ exact_dict.setdefault(value, []).append(i)
67
+ exact = [sorted(v) for v in exact_dict.values() if len(v) > 1]
72
68
 
73
- if "pchash" in stats_dict and not self.only_exact:
69
+ if not self.only_exact:
74
70
  near_dict: dict[int, list] = {}
75
- for i, value in enumerate(stats_dict["pchash"]):
71
+ for i, value in enumerate(stats["pchash"]):
76
72
  near_dict.setdefault(value, []).append(i)
77
73
  near = [sorted(v) for v in near_dict.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
78
74
  else:
@@ -84,14 +80,14 @@ class Duplicates:
84
80
  }
85
81
 
86
82
  @set_metadata("dataeval.detectors", ["only_exact"])
87
- def evaluate(self, data: Iterable[ArrayLike] | StatsOutput | Sequence[StatsOutput]) -> DuplicatesOutput:
83
+ def from_stats(self, hashes: HashStatsOutput | Sequence[HashStatsOutput]) -> DuplicatesOutput:
88
84
  """
89
85
  Returns duplicate image indices for both exact matches and near matches
90
86
 
91
87
  Parameters
92
88
  ----------
93
- data : Iterable[ArrayLike], shape - (N, C, H, W) | StatsOutput | Sequence[StatsOutput]
94
- A dataset of images in an ArrayLike format or the output(s) from an imagestats metric analysis
89
+ data : HashStatsOutput | Sequence[HashStatsOutput]
90
+ The output(s) from a hashstats analysis
95
91
 
96
92
  Returns
97
93
  -------
@@ -100,39 +96,60 @@ class Duplicates:
100
96
 
101
97
  See Also
102
98
  --------
103
- imagestats
99
+ hashstats
104
100
 
105
101
  Example
106
102
  -------
107
- >>> dups.evaluate(images)
108
- DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
109
- """ # noqa: E501
103
+ >>> exact_dupes.from_stats([hashes1, hashes2])
104
+ DuplicatesOutput(exact=[{0: [3, 20]}, {0: [16], 1: [12]}], near=[])
105
+ """
110
106
 
111
- stats, dataset_steps = combine_stats(data)
107
+ if isinstance(hashes, HashStatsOutput):
108
+ return DuplicatesOutput(**self._get_duplicates(hashes.dict()))
112
109
 
113
- if isinstance(stats, StatsOutput):
114
- if not stats.xxhash:
115
- raise ValueError("StatsOutput must include xxhash information of the images.")
116
- if not self.only_exact and not stats.pchash:
117
- raise ValueError("StatsOutput must include pchash information of the images for near matches.")
118
- self.stats = stats
119
- else:
120
- flags = ImageStat.XXHASH | (ImageStat(0) if self.only_exact else ImageStat.PCHASH)
121
- self.stats = imagestats(cast(Iterable[ArrayLike], data), flags)
110
+ if not isinstance(hashes, Sequence):
111
+ raise TypeError("Invalid stats output type; only use output from hashstats.")
122
112
 
123
- duplicates = self._get_duplicates()
113
+ combined, dataset_steps = combine_stats(hashes)
114
+ duplicates = self._get_duplicates(combined.dict())
124
115
 
125
116
  # split up results from combined dataset into individual dataset buckets
126
- if dataset_steps:
127
- dup_list: list[list[int]]
128
- for dup_type, dup_list in duplicates.items():
129
- dup_list_dict = []
130
- for idxs in dup_list:
131
- dup_dict = {}
132
- for idx in idxs:
133
- k, v = get_dataset_step_from_idx(idx, dataset_steps)
134
- dup_dict.setdefault(k, []).append(v)
135
- dup_list_dict.append(dup_dict)
136
- duplicates[dup_type] = dup_list_dict
117
+ for dup_type, dup_list in duplicates.items():
118
+ dup_list_dict = []
119
+ for idxs in dup_list:
120
+ dup_dict = {}
121
+ for idx in idxs:
122
+ k, v = get_dataset_step_from_idx(idx, dataset_steps)
123
+ dup_dict.setdefault(k, []).append(v)
124
+ dup_list_dict.append(dup_dict)
125
+ duplicates[dup_type] = dup_list_dict
126
+
127
+ return DuplicatesOutput(**duplicates)
128
+
129
+ @set_metadata("dataeval.detectors", ["only_exact"])
130
+ def evaluate(self, data: Iterable[ArrayLike]) -> DuplicatesOutput:
131
+ """
132
+ Returns duplicate image indices for both exact matches and near matches
133
+
134
+ Parameters
135
+ ----------
136
+ data : Iterable[ArrayLike], shape - (N, C, H, W) | StatsOutput | Sequence[StatsOutput]
137
+ A dataset of images in an ArrayLike format or the output(s) from a hashstats analysis
138
+
139
+ Returns
140
+ -------
141
+ DuplicatesOutput
142
+ List of groups of indices that are exact and near matches
137
143
 
144
+ See Also
145
+ --------
146
+ hashstats
147
+
148
+ Example
149
+ -------
150
+ >>> all_dupes.evaluate(images)
151
+ DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
152
+ """ # noqa: E501
153
+ self.stats = hashstats(data)
154
+ duplicates = self._get_duplicates(self.stats.dict())
138
155
  return DuplicatesOutput(**duplicates)
@@ -1,71 +1,40 @@
1
1
  from __future__ import annotations
2
2
 
3
- from typing import Sequence, cast
4
- from warnings import warn
3
+ from copy import deepcopy
4
+ from typing import Sequence, TypeVar
5
5
 
6
6
  import numpy as np
7
7
 
8
- from dataeval._internal.metrics.stats import StatsOutput
9
- from dataeval._internal.output import populate_defaults
8
+ from dataeval._internal.metrics.stats.base import BaseStatsOutput
10
9
 
10
+ TStatsOutput = TypeVar("TStatsOutput", bound=BaseStatsOutput)
11
11
 
12
- def add_stats(a: StatsOutput, b: StatsOutput) -> StatsOutput:
13
- if not isinstance(a, StatsOutput) or not isinstance(b, StatsOutput):
14
- raise TypeError(f"Cannot add object of type {type(a)} and type {type(b)}.")
15
12
 
16
- a_dict = a.dict()
17
- b_dict = b.dict()
18
- a_keys = set(a_dict)
19
- b_keys = set(b_dict)
13
+ def add_stats(a: TStatsOutput, b: TStatsOutput) -> TStatsOutput:
14
+ if type(a) is not type(b):
15
+ raise TypeError(f"Types {type(a)} and {type(b)} cannot be added.")
20
16
 
21
- missing_keys = a_keys - b_keys
22
- if missing_keys:
23
- raise ValueError(f"Required keys are missing: {missing_keys}.")
17
+ sum_dict = deepcopy(a.dict())
24
18
 
25
- extra_keys = b_keys - a_keys
26
- if extra_keys:
27
- warn(f"Extraneous keys will be dropped: {extra_keys}.")
19
+ for k in sum_dict:
20
+ if isinstance(sum_dict[k], list):
21
+ sum_dict[k].extend(b.dict()[k])
22
+ else:
23
+ sum_dict[k] = np.concatenate((sum_dict[k], b.dict()[k]))
28
24
 
29
- # perform add of multi-channel stats
30
- if "ch_idx_map" in a_dict:
31
- for k, v in a_dict.items():
32
- if k == "ch_idx_map":
33
- offset = sum([len(idxs) for idxs in v.values()])
34
- for ch_k, ch_v in b_dict[k].items():
35
- if ch_k not in v:
36
- v[ch_k] = []
37
- a_dict[k][ch_k].extend([idx + offset for idx in ch_v])
38
- else:
39
- for ch_k in b_dict[k]:
40
- if ch_k not in v:
41
- v[ch_k] = b_dict[k][ch_k]
42
- else:
43
- v[ch_k] = np.concatenate((v[ch_k], b_dict[k][ch_k]), axis=1)
44
- else:
45
- for k in a_dict:
46
- if isinstance(a_dict[k], list):
47
- a_dict[k].extend(b_dict[k])
48
- else:
49
- a_dict[k] = np.concatenate((a_dict[k], b_dict[k]))
25
+ return type(a)(**sum_dict)
50
26
 
51
- return StatsOutput(**populate_defaults(a_dict, StatsOutput))
52
-
53
-
54
- def combine_stats(stats) -> tuple[StatsOutput | None, list[int]]:
55
- dataset_steps = []
56
-
57
- if isinstance(stats, StatsOutput):
58
- return stats, dataset_steps
59
27
 
28
+ def combine_stats(stats: Sequence[TStatsOutput]) -> tuple[TStatsOutput, list[int]]:
60
29
  output = None
61
- if isinstance(stats, Sequence) and isinstance(stats[0], StatsOutput):
62
- stats = cast(Sequence[StatsOutput], stats)
63
- cur_len = 0
64
- for s in stats:
65
- output = s if output is None else add_stats(output, s)
66
- cur_len += len(s)
67
- dataset_steps.append(cur_len)
68
-
30
+ dataset_steps = []
31
+ cur_len = 0
32
+ for s in stats:
33
+ output = s if output is None else add_stats(output, s)
34
+ cur_len += len(s)
35
+ dataset_steps.append(cur_len)
36
+ if output is None:
37
+ raise TypeError("Cannot combine empty sequence of stats.")
69
38
  return output, dataset_steps
70
39
 
71
40
 
@@ -16,7 +16,7 @@ import tensorflow as tf
16
16
  from numpy.typing import ArrayLike
17
17
 
18
18
  from dataeval._internal.detectors.ood.base import OODBase, OODScore
19
- from dataeval._internal.interop import to_numpy
19
+ from dataeval._internal.interop import as_numpy
20
20
  from dataeval._internal.models.tensorflow.autoencoder import AE
21
21
  from dataeval._internal.models.tensorflow.utils import predict_batch
22
22
 
@@ -46,10 +46,10 @@ class OOD_AE(OODBase):
46
46
  ) -> None:
47
47
  if loss_fn is None:
48
48
  loss_fn = keras.losses.MeanSquaredError()
49
- super().fit(to_numpy(x_ref), threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
49
+ super().fit(as_numpy(x_ref), threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
50
50
 
51
51
  def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
52
- self._validate(X := to_numpy(X))
52
+ self._validate(X := as_numpy(X))
53
53
 
54
54
  # reconstruct instances
55
55
  X_recon = predict_batch(X, self.model, batch_size=batch_size)