dataeval 0.75.0__py3-none-any.whl → 0.76.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataeval/__init__.py +3 -3
- dataeval/detectors/drift/base.py +2 -2
- dataeval/detectors/drift/ks.py +2 -1
- dataeval/detectors/drift/mmd.py +3 -2
- dataeval/detectors/drift/uncertainty.py +2 -2
- dataeval/detectors/drift/updates.py +1 -1
- dataeval/detectors/linters/clusterer.py +3 -2
- dataeval/detectors/linters/duplicates.py +4 -4
- dataeval/detectors/linters/outliers.py +96 -3
- dataeval/detectors/ood/__init__.py +1 -1
- dataeval/detectors/ood/base.py +1 -17
- dataeval/detectors/ood/output.py +1 -1
- dataeval/interop.py +1 -1
- dataeval/metrics/__init__.py +1 -1
- dataeval/metrics/bias/__init__.py +1 -1
- dataeval/metrics/bias/balance.py +3 -3
- dataeval/metrics/bias/coverage.py +1 -1
- dataeval/metrics/bias/diversity.py +14 -10
- dataeval/metrics/bias/parity.py +7 -9
- dataeval/metrics/estimators/ber.py +4 -3
- dataeval/metrics/estimators/divergence.py +3 -3
- dataeval/metrics/estimators/uap.py +3 -3
- dataeval/metrics/stats/__init__.py +1 -1
- dataeval/metrics/stats/base.py +24 -8
- dataeval/metrics/stats/boxratiostats.py +5 -5
- dataeval/metrics/stats/datasetstats.py +39 -6
- dataeval/metrics/stats/dimensionstats.py +4 -4
- dataeval/metrics/stats/hashstats.py +2 -2
- dataeval/metrics/stats/labelstats.py +89 -6
- dataeval/metrics/stats/pixelstats.py +7 -5
- dataeval/metrics/stats/visualstats.py +6 -4
- dataeval/output.py +23 -14
- dataeval/utils/__init__.py +2 -2
- dataeval/utils/dataset/read.py +1 -1
- dataeval/utils/dataset/split.py +1 -1
- dataeval/utils/metadata.py +255 -110
- dataeval/utils/plot.py +129 -6
- dataeval/workflows/sufficiency.py +2 -2
- {dataeval-0.75.0.dist-info → dataeval-0.76.1.dist-info}/LICENSE.txt +2 -2
- {dataeval-0.75.0.dist-info → dataeval-0.76.1.dist-info}/METADATA +57 -30
- dataeval-0.76.1.dist-info/RECORD +67 -0
- dataeval-0.75.0.dist-info/RECORD +0 -67
- {dataeval-0.75.0.dist-info → dataeval-0.76.1.dist-info}/WHEEL +0 -0
dataeval/__init__.py
CHANGED
@@ -8,7 +8,7 @@ shifts that impact performance of deployed models.
|
|
8
8
|
from __future__ import annotations
|
9
9
|
|
10
10
|
__all__ = ["detectors", "log", "metrics", "utils", "workflows"]
|
11
|
-
__version__ = "0.
|
11
|
+
__version__ = "0.76.1"
|
12
12
|
|
13
13
|
import logging
|
14
14
|
|
@@ -24,10 +24,10 @@ def log(level: int = logging.DEBUG, handler: logging.Handler | None = None) -> N
|
|
24
24
|
Parameters
|
25
25
|
----------
|
26
26
|
level : int, default logging.DEBUG(10)
|
27
|
-
Set the logging level for the logger
|
27
|
+
Set the logging level for the logger.
|
28
28
|
handler : logging.Handler, optional
|
29
29
|
Sets the logging handler for the logger if provided, otherwise logger will be
|
30
|
-
provided with a StreamHandler
|
30
|
+
provided with a StreamHandler.
|
31
31
|
"""
|
32
32
|
import logging
|
33
33
|
|
dataeval/detectors/drift/base.py
CHANGED
@@ -45,7 +45,7 @@ class UpdateStrategy(ABC):
|
|
45
45
|
@dataclass(frozen=True)
|
46
46
|
class DriftBaseOutput(Output):
|
47
47
|
"""
|
48
|
-
Base output class for Drift
|
48
|
+
Base output class for Drift Detector classes
|
49
49
|
|
50
50
|
Attributes
|
51
51
|
----------
|
@@ -64,7 +64,7 @@ class DriftBaseOutput(Output):
|
|
64
64
|
@dataclass(frozen=True)
|
65
65
|
class DriftOutput(DriftBaseOutput):
|
66
66
|
"""
|
67
|
-
Output class for :class:`DriftCVM`, :class:`DriftKS`, and :class:`DriftUncertainty` drift detectors
|
67
|
+
Output class for :class:`DriftCVM`, :class:`DriftKS`, and :class:`DriftUncertainty` drift detectors.
|
68
68
|
|
69
69
|
Attributes
|
70
70
|
----------
|
dataeval/detectors/drift/ks.py
CHANGED
@@ -22,7 +22,8 @@ from dataeval.interop import to_numpy
|
|
22
22
|
|
23
23
|
class DriftKS(BaseDriftUnivariate):
|
24
24
|
"""
|
25
|
-
:term:`Drift` detector employing the Kolmogorov-Smirnov (KS)
|
25
|
+
:term:`Drift` detector employing the :term:`Kolmogorov-Smirnov (KS) \
|
26
|
+
distribution<Kolmogorov-Smirnov (K-S) test>` test.
|
26
27
|
|
27
28
|
The KS test detects changes in the maximum distance between two data
|
28
29
|
distributions with Bonferroni or :term:`False Discovery Rate (FDR)` correction
|
dataeval/detectors/drift/mmd.py
CHANGED
@@ -26,7 +26,7 @@ from dataeval.utils.torch.internal import get_device
|
|
26
26
|
@dataclass(frozen=True)
|
27
27
|
class DriftMMDOutput(DriftBaseOutput):
|
28
28
|
"""
|
29
|
-
Output class for :class:`DriftMMD` :term:`drift<Drift>` detector
|
29
|
+
Output class for :class:`DriftMMD` :term:`drift<Drift>` detector.
|
30
30
|
|
31
31
|
Attributes
|
32
32
|
----------
|
@@ -51,7 +51,8 @@ class DriftMMDOutput(DriftBaseOutput):
|
|
51
51
|
|
52
52
|
class DriftMMD(BaseDrift):
|
53
53
|
"""
|
54
|
-
:term:`Maximum Mean Discrepancy (MMD) Drift Detection` algorithm
|
54
|
+
:term:`Maximum Mean Discrepancy (MMD) Drift Detection` algorithm \
|
55
|
+
using a permutation test.
|
55
56
|
|
56
57
|
Parameters
|
57
58
|
----------
|
@@ -66,8 +66,8 @@ def classifier_uncertainty(
|
|
66
66
|
|
67
67
|
class DriftUncertainty:
|
68
68
|
"""
|
69
|
-
Test for a change in the number of instances falling into regions on which
|
70
|
-
|
69
|
+
Test for a change in the number of instances falling into regions on which \
|
70
|
+
the model is uncertain.
|
71
71
|
|
72
72
|
Performs a K-S test on prediction entropies.
|
73
73
|
|
@@ -18,7 +18,7 @@ from dataeval.utils.shared import flatten
|
|
18
18
|
@dataclass(frozen=True)
|
19
19
|
class ClustererOutput(Output):
|
20
20
|
"""
|
21
|
-
Output class for :class:`Clusterer` lint detector
|
21
|
+
Output class for :class:`Clusterer` lint detector.
|
22
22
|
|
23
23
|
Attributes
|
24
24
|
----------
|
@@ -131,7 +131,8 @@ class _ClusterMergeEntry:
|
|
131
131
|
|
132
132
|
class Clusterer:
|
133
133
|
"""
|
134
|
-
Uses hierarchical clustering to flag dataset properties of interest like
|
134
|
+
Uses hierarchical clustering to flag dataset properties of interest like outliers \
|
135
|
+
and :term:`duplicates<Duplicates>`.
|
135
136
|
|
136
137
|
Parameters
|
137
138
|
----------
|
@@ -19,7 +19,7 @@ TIndexCollection = TypeVar("TIndexCollection", DuplicateGroup, DatasetDuplicateG
|
|
19
19
|
@dataclass(frozen=True)
|
20
20
|
class DuplicatesOutput(Generic[TIndexCollection], Output):
|
21
21
|
"""
|
22
|
-
Output class for :class:`Duplicates` lint detector
|
22
|
+
Output class for :class:`Duplicates` lint detector.
|
23
23
|
|
24
24
|
Attributes
|
25
25
|
----------
|
@@ -39,8 +39,8 @@ class DuplicatesOutput(Generic[TIndexCollection], Output):
|
|
39
39
|
|
40
40
|
class Duplicates:
|
41
41
|
"""
|
42
|
-
Finds the duplicate images in a dataset using xxhash for exact
|
43
|
-
and pchash for near duplicates
|
42
|
+
Finds the duplicate images in a dataset using xxhash for exact \
|
43
|
+
:term:`duplicates<Duplicates>` and pchash for near duplicates.
|
44
44
|
|
45
45
|
Attributes
|
46
46
|
----------
|
@@ -92,7 +92,7 @@ class Duplicates:
|
|
92
92
|
|
93
93
|
Parameters
|
94
94
|
----------
|
95
|
-
|
95
|
+
hashes : HashStatsOutput | Sequence[HashStatsOutput]
|
96
96
|
The output(s) from a hashstats analysis
|
97
97
|
|
98
98
|
Returns
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
2
2
|
|
3
3
|
__all__ = []
|
4
4
|
|
5
|
+
import contextlib
|
5
6
|
from dataclasses import dataclass
|
6
7
|
from typing import Generic, Iterable, Literal, Sequence, TypeVar, Union, overload
|
7
8
|
|
@@ -12,19 +13,78 @@ from dataeval.detectors.linters.merged_stats import combine_stats, get_dataset_s
|
|
12
13
|
from dataeval.metrics.stats.base import BOX_COUNT, SOURCE_INDEX
|
13
14
|
from dataeval.metrics.stats.datasetstats import DatasetStatsOutput, datasetstats
|
14
15
|
from dataeval.metrics.stats.dimensionstats import DimensionStatsOutput
|
16
|
+
from dataeval.metrics.stats.labelstats import LabelStatsOutput
|
15
17
|
from dataeval.metrics.stats.pixelstats import PixelStatsOutput
|
16
18
|
from dataeval.metrics.stats.visualstats import VisualStatsOutput
|
17
19
|
from dataeval.output import Output, set_metadata
|
18
20
|
|
21
|
+
with contextlib.suppress(ImportError):
|
22
|
+
import pandas as pd
|
23
|
+
|
24
|
+
|
19
25
|
IndexIssueMap = dict[int, dict[str, float]]
|
20
26
|
OutlierStatsOutput = Union[DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput]
|
21
27
|
TIndexIssueMap = TypeVar("TIndexIssueMap", IndexIssueMap, list[IndexIssueMap])
|
22
28
|
|
23
29
|
|
30
|
+
def _reorganize_by_class_and_metric(result, lstats):
|
31
|
+
"""Flip result from grouping by image to grouping by class and metric"""
|
32
|
+
metrics = {}
|
33
|
+
class_wise = {label: {} for label in lstats.image_indices_per_label}
|
34
|
+
|
35
|
+
# Group metrics and calculate class-wise counts
|
36
|
+
for img, group in result.items():
|
37
|
+
for extreme in group:
|
38
|
+
metrics.setdefault(extreme, []).append(img)
|
39
|
+
for label, images in lstats.image_indices_per_label.items():
|
40
|
+
if img in images:
|
41
|
+
class_wise[label][extreme] = class_wise[label].get(extreme, 0) + 1
|
42
|
+
|
43
|
+
return metrics, class_wise
|
44
|
+
|
45
|
+
|
46
|
+
def _create_table(metrics, class_wise):
|
47
|
+
"""Create table for displaying the results"""
|
48
|
+
max_class_length = max(len(str(label)) for label in class_wise) + 2
|
49
|
+
max_total = max(len(metrics[group]) for group in metrics) + 2
|
50
|
+
|
51
|
+
table_header = " | ".join(
|
52
|
+
[f"{'Class':>{max_class_length}}"]
|
53
|
+
+ [f"{group:^{max(5, len(str(group))) + 2}}" for group in sorted(metrics.keys())]
|
54
|
+
+ [f"{'Total':<{max_total}}"]
|
55
|
+
)
|
56
|
+
table_rows = []
|
57
|
+
|
58
|
+
for class_cat, results in class_wise.items():
|
59
|
+
table_value = [f"{class_cat:>{max_class_length}}"]
|
60
|
+
total = 0
|
61
|
+
for group in sorted(metrics.keys()):
|
62
|
+
count = results.get(group, 0)
|
63
|
+
table_value.append(f"{count:^{max(5, len(str(group))) + 2}}")
|
64
|
+
total += count
|
65
|
+
table_value.append(f"{total:^{max_total}}")
|
66
|
+
table_rows.append(" | ".join(table_value))
|
67
|
+
|
68
|
+
table = [table_header] + table_rows
|
69
|
+
return table
|
70
|
+
|
71
|
+
|
72
|
+
def _create_pandas_dataframe(class_wise):
|
73
|
+
"""Create data for pandas dataframe"""
|
74
|
+
data = []
|
75
|
+
for label, metrics_dict in class_wise.items():
|
76
|
+
row = {"Class": label}
|
77
|
+
total = sum(metrics_dict.values())
|
78
|
+
row.update(metrics_dict) # Add metric counts
|
79
|
+
row["Total"] = total
|
80
|
+
data.append(row)
|
81
|
+
return data
|
82
|
+
|
83
|
+
|
24
84
|
@dataclass(frozen=True)
|
25
85
|
class OutliersOutput(Generic[TIndexIssueMap], Output):
|
26
86
|
"""
|
27
|
-
Output class for :class:`Outliers` lint detector
|
87
|
+
Output class for :class:`Outliers` lint detector.
|
28
88
|
|
29
89
|
Attributes
|
30
90
|
----------
|
@@ -45,6 +105,39 @@ class OutliersOutput(Generic[TIndexIssueMap], Output):
|
|
45
105
|
else:
|
46
106
|
return sum(len(d) for d in self.issues)
|
47
107
|
|
108
|
+
def to_table(self, labelstats: LabelStatsOutput) -> str:
|
109
|
+
if isinstance(self.issues, dict):
|
110
|
+
metrics, classwise = _reorganize_by_class_and_metric(self.issues, labelstats)
|
111
|
+
listed_table = _create_table(metrics, classwise)
|
112
|
+
table = "\n".join(listed_table)
|
113
|
+
else:
|
114
|
+
outertable = []
|
115
|
+
for d in self.issues:
|
116
|
+
metrics, classwise = _reorganize_by_class_and_metric(d, labelstats)
|
117
|
+
listed_table = _create_table(metrics, classwise)
|
118
|
+
str_table = "\n".join(listed_table)
|
119
|
+
outertable.append(str_table)
|
120
|
+
table = "\n\n".join(outertable)
|
121
|
+
return table
|
122
|
+
|
123
|
+
def to_dataframe(self, labelstats: LabelStatsOutput) -> pd.DataFrame:
|
124
|
+
import pandas as pd
|
125
|
+
|
126
|
+
if isinstance(self.issues, dict):
|
127
|
+
_, classwise = _reorganize_by_class_and_metric(self.issues, labelstats)
|
128
|
+
data = _create_pandas_dataframe(classwise)
|
129
|
+
df = pd.DataFrame(data)
|
130
|
+
else:
|
131
|
+
df_list = []
|
132
|
+
for i, d in enumerate(self.issues):
|
133
|
+
_, classwise = _reorganize_by_class_and_metric(d, labelstats)
|
134
|
+
data = _create_pandas_dataframe(classwise)
|
135
|
+
single_df = pd.DataFrame(data)
|
136
|
+
single_df["Dataset"] = i
|
137
|
+
df_list.append(single_df)
|
138
|
+
df = pd.concat(df_list)
|
139
|
+
return df
|
140
|
+
|
48
141
|
|
49
142
|
def _get_outlier_mask(
|
50
143
|
values: NDArray, method: Literal["zscore", "modzscore", "iqr"], threshold: float | None
|
@@ -71,7 +164,7 @@ def _get_outlier_mask(
|
|
71
164
|
|
72
165
|
class Outliers:
|
73
166
|
r"""
|
74
|
-
Calculates statistical
|
167
|
+
Calculates statistical outliers of a dataset using various statistical tests applied to each image.
|
75
168
|
|
76
169
|
Parameters
|
77
170
|
----------
|
@@ -164,7 +257,7 @@ class Outliers:
|
|
164
257
|
self, stats: OutlierStatsOutput | DatasetStatsOutput | Sequence[OutlierStatsOutput]
|
165
258
|
) -> OutliersOutput[IndexIssueMap] | OutliersOutput[list[IndexIssueMap]]:
|
166
259
|
"""
|
167
|
-
Returns indices of Outliers with the issues identified for each
|
260
|
+
Returns indices of Outliers with the issues identified for each.
|
168
261
|
|
169
262
|
Parameters
|
170
263
|
----------
|
@@ -1,5 +1,5 @@
|
|
1
1
|
"""
|
2
|
-
Out-of-distribution (OOD)
|
2
|
+
Out-of-distribution (OOD) detectors identify data that is different from the data used to train a particular model.
|
3
3
|
"""
|
4
4
|
|
5
5
|
__all__ = ["OODOutput", "OODScoreOutput", "OOD_AE"]
|
dataeval/detectors/ood/base.py
CHANGED
@@ -87,24 +87,8 @@ class OODBaseGMM(OODBase, OODGMMMixin[GaussianMixtureModelParams]):
|
|
87
87
|
batch_size: int,
|
88
88
|
verbose: bool,
|
89
89
|
) -> None:
|
90
|
-
|
91
|
-
trainer(
|
92
|
-
model=self.model,
|
93
|
-
x_train=to_numpy(x_ref),
|
94
|
-
y_train=None,
|
95
|
-
loss_fn=loss_fn,
|
96
|
-
optimizer=optimizer,
|
97
|
-
preprocess_fn=None,
|
98
|
-
epochs=epochs,
|
99
|
-
batch_size=batch_size,
|
100
|
-
device=self.device,
|
101
|
-
verbose=verbose,
|
102
|
-
)
|
90
|
+
super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
|
103
91
|
|
104
92
|
# Calculate the GMM parameters
|
105
93
|
_, z, gamma = cast(tuple[torch.Tensor, torch.Tensor, torch.Tensor], self.model(x_ref))
|
106
94
|
self._gmm_params = gmm_params(z, gamma)
|
107
|
-
|
108
|
-
# Infer the threshold values
|
109
|
-
self._ref_score = self.score(x_ref, batch_size)
|
110
|
-
self._threshold_perc = threshold_perc
|
dataeval/detectors/ood/output.py
CHANGED
dataeval/interop.py
CHANGED
@@ -46,7 +46,7 @@ def to_numpy(array: ArrayLike | None, copy: bool = True) -> NDArray[Any]:
|
|
46
46
|
if isinstance(array, np.ndarray):
|
47
47
|
return array.copy() if copy else array
|
48
48
|
|
49
|
-
if array.__class__.__module__.startswith("tensorflow"):
|
49
|
+
if array.__class__.__module__.startswith("tensorflow"): # pragma: no cover - removed tf from deps
|
50
50
|
tf = _try_import("tensorflow")
|
51
51
|
if tf and tf.is_tensor(array):
|
52
52
|
_logger.log(logging.INFO, "Converting Tensorflow array to NumPy array.")
|
dataeval/metrics/__init__.py
CHANGED
dataeval/metrics/bias/balance.py
CHANGED
@@ -23,8 +23,8 @@ with contextlib.suppress(ImportError):
|
|
23
23
|
@dataclass(frozen=True)
|
24
24
|
class BalanceOutput(Output):
|
25
25
|
"""
|
26
|
-
Output class for :func:`balance` bias metric
|
27
|
-
|
26
|
+
Output class for :func:`balance` :term:`bias<Bias>` metric.
|
27
|
+
|
28
28
|
Attributes
|
29
29
|
----------
|
30
30
|
balance : NDArray[np.float64]
|
@@ -123,7 +123,7 @@ def balance(
|
|
123
123
|
num_neighbors: int = 5,
|
124
124
|
) -> BalanceOutput:
|
125
125
|
"""
|
126
|
-
Mutual information (MI) between factors (class label, metadata, label/image properties)
|
126
|
+
Mutual information (MI) between factors (class label, metadata, label/image properties).
|
127
127
|
|
128
128
|
Parameters
|
129
129
|
----------
|
@@ -71,7 +71,7 @@ def _plot(images: NDArray[Any], num_images: int) -> Figure:
|
|
71
71
|
@dataclass(frozen=True)
|
72
72
|
class CoverageOutput(Output):
|
73
73
|
"""
|
74
|
-
Output class for :func:`coverage` :term:`bias<Bias>` metric
|
74
|
+
Output class for :func:`coverage` :term:`bias<Bias>` metric.
|
75
75
|
|
76
76
|
Attributes
|
77
77
|
----------
|
@@ -51,7 +51,7 @@ def _plot(labels: NDArray[Any], bar_heights: NDArray[Any]) -> Figure:
|
|
51
51
|
@dataclass(frozen=True)
|
52
52
|
class DiversityOutput(Output):
|
53
53
|
"""
|
54
|
-
Output class for :func:`diversity` :term:`bias<Bias>` metric
|
54
|
+
Output class for :func:`diversity` :term:`bias<Bias>` metric.
|
55
55
|
|
56
56
|
Attributes
|
57
57
|
----------
|
@@ -197,10 +197,12 @@ def diversity(
|
|
197
197
|
method: Literal["simpson", "shannon"] = "simpson",
|
198
198
|
) -> DiversityOutput:
|
199
199
|
"""
|
200
|
-
Compute :term:`diversity<Diversity>` and classwise diversity for
|
201
|
-
|
200
|
+
Compute :term:`diversity<Diversity>` and classwise diversity for \
|
201
|
+
discrete/categorical variables through standard histogram binning, \
|
202
|
+
for continuous variables.
|
202
203
|
|
203
|
-
|
204
|
+
The method specified defines diversity as the inverse Simpson diversity index linearly rescaled to
|
205
|
+
the unit interval, or the normalized form of the Shannon entropy.
|
204
206
|
|
205
207
|
diversity = 1 implies that samples are evenly distributed across a particular factor
|
206
208
|
diversity = 0 implies that all samples belong to one category/bin
|
@@ -209,11 +211,8 @@ def diversity(
|
|
209
211
|
----------
|
210
212
|
metadata : Metadata
|
211
213
|
Preprocessed metadata from :func:`dataeval.utils.metadata.preprocess`
|
212
|
-
|
213
|
-
|
214
|
-
----
|
215
|
-
- The expression is undefined for q=1, but it approaches the Shannon entropy in the limit.
|
216
|
-
- If there is only one category, the diversity index takes a value of 0.
|
214
|
+
method : "simpson" or "shannon", default "simpson"
|
215
|
+
The methodology used for defining diversity
|
217
216
|
|
218
217
|
Returns
|
219
218
|
-------
|
@@ -221,9 +220,14 @@ def diversity(
|
|
221
220
|
Diversity index per column of self.data or each factor in self.names and
|
222
221
|
classwise diversity [n_class x n_factor]
|
223
222
|
|
223
|
+
Note
|
224
|
+
----
|
225
|
+
- The expression is undefined for q=1, but it approaches the Shannon entropy in the limit.
|
226
|
+
- If there is only one category, the diversity index takes a value of 0.
|
227
|
+
|
224
228
|
Example
|
225
229
|
-------
|
226
|
-
Compute
|
230
|
+
Compute the diversity index of metadata and class labels
|
227
231
|
|
228
232
|
>>> div_simp = diversity(metadata, method="simpson")
|
229
233
|
>>> div_simp.diversity_index
|
dataeval/metrics/bias/parity.py
CHANGED
@@ -21,7 +21,7 @@ TData = TypeVar("TData", np.float64, NDArray[np.float64])
|
|
21
21
|
@dataclass(frozen=True)
|
22
22
|
class ParityOutput(Generic[TData], Output):
|
23
23
|
"""
|
24
|
-
Output class for :func:`parity` and :func:`label_parity` :term:`bias<Bias>` metrics
|
24
|
+
Output class for :func:`parity` and :func:`label_parity` :term:`bias<Bias>` metrics.
|
25
25
|
|
26
26
|
Attributes
|
27
27
|
----------
|
@@ -123,8 +123,8 @@ def label_parity(
|
|
123
123
|
num_classes: int | None = None,
|
124
124
|
) -> ParityOutput[np.float64]:
|
125
125
|
"""
|
126
|
-
Calculate the chi-square statistic to assess the :term:`parity<Parity>`
|
127
|
-
observed label distributions.
|
126
|
+
Calculate the chi-square statistic to assess the :term:`parity<Parity>` \
|
127
|
+
between expected and observed label distributions.
|
128
128
|
|
129
129
|
This function computes the frequency distribution of classes in both expected and observed labels, normalizes
|
130
130
|
the expected distribution to match the total number of observed labels, and then calculates the chi-square
|
@@ -208,8 +208,8 @@ def label_parity(
|
|
208
208
|
@set_metadata
|
209
209
|
def parity(metadata: Metadata) -> ParityOutput[NDArray[np.float64]]:
|
210
210
|
"""
|
211
|
-
Calculate chi-square statistics to assess the linear relationship
|
212
|
-
and class labels.
|
211
|
+
Calculate chi-square statistics to assess the linear relationship \
|
212
|
+
between multiple factors and class labels.
|
213
213
|
|
214
214
|
This function computes the chi-square statistic for each metadata factor to determine if there is
|
215
215
|
a significant relationship between the factor values and class labels. The chi-square statistic is
|
@@ -253,13 +253,11 @@ def parity(metadata: Metadata) -> ParityOutput[NDArray[np.float64]]:
|
|
253
253
|
>>> from dataeval.utils.metadata import preprocess
|
254
254
|
>>> rng = np.random.default_rng(175)
|
255
255
|
>>> labels = rng.choice([0, 1, 2], (100))
|
256
|
-
>>> metadata_dict =
|
257
|
-
... {
|
256
|
+
>>> metadata_dict = {
|
258
257
|
... "age": list(rng.choice([25, 30, 35, 45], (100))),
|
259
258
|
... "income": list(rng.choice([50000, 65000, 80000], (100))),
|
260
259
|
... "gender": list(rng.choice(["M", "F"], (100))),
|
261
|
-
...
|
262
|
-
... ]
|
260
|
+
... }
|
263
261
|
>>> continuous_factor_bincounts = {"age": 4, "income": 3}
|
264
262
|
>>> metadata = preprocess(metadata_dict, labels, continuous_factor_bincounts)
|
265
263
|
>>> parity(metadata)
|
@@ -28,7 +28,7 @@ from dataeval.utils.shared import compute_neighbors, get_classes_counts, get_met
|
|
28
28
|
@dataclass(frozen=True)
|
29
29
|
class BEROutput(Output):
|
30
30
|
"""
|
31
|
-
Output class for :func:`ber` estimator metric
|
31
|
+
Output class for :func:`ber` estimator metric.
|
32
32
|
|
33
33
|
Attributes
|
34
34
|
----------
|
@@ -44,7 +44,7 @@ class BEROutput(Output):
|
|
44
44
|
|
45
45
|
|
46
46
|
def ber_mst(images: NDArray[np.float64], labels: NDArray[np.int_], k: int = 1) -> tuple[float, float]:
|
47
|
-
"""Calculates the :term:`Bayes error rate<Bayes Error Rate (BER)>` using a minimum spanning tree
|
47
|
+
"""Calculates the :term:`Bayes error rate<Bayes Error Rate (BER)>` using a minimum spanning tree.
|
48
48
|
|
49
49
|
Parameters
|
50
50
|
----------
|
@@ -119,7 +119,8 @@ def knn_lowerbound(value: float, classes: int, k: int) -> float:
|
|
119
119
|
@set_metadata
|
120
120
|
def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN", "MST"] = "KNN") -> BEROutput:
|
121
121
|
"""
|
122
|
-
An estimator for Multi-class :term:`Bayes error rate<Bayes Error Rate (BER)>`
|
122
|
+
An estimator for Multi-class :term:`Bayes error rate<Bayes Error Rate (BER)>` \
|
123
|
+
using FR or KNN test statistic basis.
|
123
124
|
|
124
125
|
Parameters
|
125
126
|
----------
|
@@ -21,7 +21,7 @@ from dataeval.utils.shared import compute_neighbors, get_method, minimum_spannin
|
|
21
21
|
@dataclass(frozen=True)
|
22
22
|
class DivergenceOutput(Output):
|
23
23
|
"""
|
24
|
-
Output class for :func:`divergence` estimator metric
|
24
|
+
Output class for :func:`divergence` estimator metric.
|
25
25
|
|
26
26
|
Attributes
|
27
27
|
----------
|
@@ -59,7 +59,7 @@ def divergence_mst(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
|
|
59
59
|
|
60
60
|
def divergence_fnn(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
|
61
61
|
"""
|
62
|
-
Calculates the estimated label errors based on their nearest neighbors
|
62
|
+
Calculates the estimated label errors based on their nearest neighbors.
|
63
63
|
|
64
64
|
Parameters
|
65
65
|
----------
|
@@ -81,7 +81,7 @@ def divergence_fnn(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
|
|
81
81
|
@set_metadata
|
82
82
|
def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST"] = "FNN") -> DivergenceOutput:
|
83
83
|
"""
|
84
|
-
Calculates the :term
|
84
|
+
Calculates the :term:`divergence` and any errors between the datasets.
|
85
85
|
|
86
86
|
Parameters
|
87
87
|
----------
|
@@ -20,7 +20,7 @@ from dataeval.output import Output, set_metadata
|
|
20
20
|
@dataclass(frozen=True)
|
21
21
|
class UAPOutput(Output):
|
22
22
|
"""
|
23
|
-
Output class for :func:`uap` estimator metric
|
23
|
+
Output class for :func:`uap` estimator metric.
|
24
24
|
|
25
25
|
Attributes
|
26
26
|
----------
|
@@ -34,8 +34,8 @@ class UAPOutput(Output):
|
|
34
34
|
@set_metadata
|
35
35
|
def uap(labels: ArrayLike, scores: ArrayLike) -> UAPOutput:
|
36
36
|
"""
|
37
|
-
FR Test Statistic based estimate of the empirical mean precision for
|
38
|
-
|
37
|
+
FR Test Statistic based estimate of the empirical mean precision for the \
|
38
|
+
upperbound average precision.
|
39
39
|
|
40
40
|
Parameters
|
41
41
|
----------
|
dataeval/metrics/stats/base.py
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
+
from dataeval.utils.plot import histogram_plot
|
4
|
+
|
3
5
|
__all__ = []
|
4
6
|
|
5
7
|
import re
|
@@ -100,19 +102,33 @@ class BaseStatsOutput(Output):
|
|
100
102
|
for source_index in list(self.source_index) + [None]:
|
101
103
|
if source_index is None or source_index.image > cur_image:
|
102
104
|
mask.extend(cur_mask if matches(cur_max_channel + 1, channel_count) else [False for _ in cur_mask])
|
103
|
-
if source_index is None:
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
105
|
+
if source_index is not None:
|
106
|
+
cur_image = source_index.image
|
107
|
+
cur_max_channel = 0
|
108
|
+
cur_mask.clear()
|
109
|
+
if source_index is not None:
|
110
|
+
cur_mask.append(matches(source_index.channel, channel_index))
|
111
|
+
cur_max_channel = max(cur_max_channel, source_index.channel or 0)
|
110
112
|
return mask
|
111
113
|
|
112
114
|
def __len__(self) -> int:
|
113
115
|
return len(self.source_index)
|
114
116
|
|
115
117
|
|
118
|
+
def _is_plottable(k: str, v: Any, excluded_keys: Iterable[str]) -> bool:
|
119
|
+
return isinstance(v, np.ndarray) and v[v != 0].size > 0 and all(k != x for x in excluded_keys)
|
120
|
+
|
121
|
+
|
122
|
+
class HistogramPlotMixin:
|
123
|
+
_excluded_keys: Iterable[str] = []
|
124
|
+
|
125
|
+
def dict(self) -> dict[str, Any]: ...
|
126
|
+
|
127
|
+
def plot(self, log: bool) -> None:
|
128
|
+
data_dict = {k: v for k, v in self.dict().items() if _is_plottable(k, v, self._excluded_keys)}
|
129
|
+
histogram_plot(data_dict, log)
|
130
|
+
|
131
|
+
|
116
132
|
TStatsOutput = TypeVar("TStatsOutput", bound=BaseStatsOutput, covariant=True)
|
117
133
|
|
118
134
|
|
@@ -126,7 +142,7 @@ class StatsProcessor(Generic[TStatsOutput]):
|
|
126
142
|
self.raw = image
|
127
143
|
self.width: int = image.shape[-1]
|
128
144
|
self.height: int = image.shape[-2]
|
129
|
-
self.box: NDArray[
|
145
|
+
self.box: NDArray[np.int64] = np.array([0, 0, self.width, self.height]) if box is None else box.astype(np.int64)
|
130
146
|
self._per_channel = per_channel
|
131
147
|
self._image = None
|
132
148
|
self._shape = None
|