dataeval 1.2.0rc1__tar.gz → 1.2.0rc2__tar.gz

{dataeval-1.2.0rc1 → dataeval-1.2.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dataeval
-Version: 1.2.0rc1
+Version: 1.2.0rc2
 Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
 Project-URL: Homepage, https://dataeval.ai/
 Project-URL: Repository, https://github.com/aria-ml/dataeval/

{dataeval-1.2.0rc1 → dataeval-1.2.0rc2}/pyproject.toml

@@ -130,7 +130,7 @@ docs = [
   "numpy>=1.24.2",  # floor capped by the JATIC program standard; uv.lock resolves current
   "av>=12.0.0",
   "certifi>=2024.07.04",
-  "dataeval-plots[plotly]>=0.0.15",
+  "dataeval-plots[plotly]>=0.0.16",
   "ipykernel>=6.26.0",
   "ipywidgets>=8.1.1",
   "jinja2>=3.1.6",

{dataeval-1.2.0rc1 → dataeval-1.2.0rc2}/src/dataeval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.0rc1'
-__version_tuple__ = version_tuple = (1, 2, 0, 'rc1')
+__version__ = version = '1.2.0rc2'
+__version_tuple__ = version_tuple = (1, 2, 0, 'rc2')
 
 __commit_id__ = commit_id = None

{dataeval-1.2.0rc1 → dataeval-1.2.0rc2}/src/dataeval/core/_completeness.py

@@ -27,10 +27,13 @@ class CompletenessResult(TypedDict):
         space actually spanned by embeddings.
     nearest_neighbor_pairs : Sequence[tuple[int, int]]
         Sequence of tuples (i, j) representing point indices and their nearest neighbors,
-        sorted by decreasing nearest neighbor distance. Each pair appears only once.
+        sorted by decreasing nearest neighbor distance. Each pair appears only once, so two
+        points that are each other's nearest neighbor share one entry. ``j`` is never ``i``,
+        even when ``i`` has an exact duplicate.
     nearest_neighbor_distances : Sequence[float]
         Distance between each pair in ``nearest_neighbor_pairs``, aligned 1:1 and sharing
         the same decreasing order. The small-distance tail flags near-duplicate samples.
+        Each point's own nearest-neighbor distance is the smallest among the pairs it is in.
     """
 
     completeness: float
@@ -246,16 +249,12 @@ def completeness(embeddings: Array) -> CompletenessResult:
             nearest_neighbor_distances=[],
         )
 
-    # Fit nearest neighbors model (k=2 because first neighbor is always the point itself)
-    nn_model = NearestNeighbors(n_neighbors=2, algorithm="auto")
-    nn_model.fit(embeddings)
-
-    # Get distances and indices for nearest neighbors
-    distances, indices = nn_model.kneighbors(embeddings)
-
-    # Extract the actual nearest neighbor (second column, index 1)
-    nearest_neighbors = indices[:, 1]
-    nearest_distances = distances[:, 1]
+    # Each point's nearest other point. Querying the fitted points themselves (no X) makes sklearn drop each
+    # point from its own neighbors by index, which stays right for exact duplicates: there a zero-distance tie
+    # can rank the point after its copy, so "the first neighbor is the point itself" does not hold.
+    distances, indices = NearestNeighbors(n_neighbors=1).fit(embeddings).kneighbors()
+    nearest_neighbors = indices[:, 0]
+    nearest_distances = distances[:, 0]
 
     # Create pairs, deduplicating so each pair appears only once
     seen_pairs = set()

{dataeval-1.2.0rc1 → dataeval-1.2.0rc2}/src/dataeval/scope/_coverage.py

@@ -15,8 +15,8 @@ label counts cannot reveal:
   varies (via :func:`~dataeval.core.completeness`). A class can spread far yet vary along a
   single axis — all images shot from one angle, lighting or zoom aside — which dispersion's
   single radius cannot distinguish.
-* **near-duplicate fraction** — the *redundancy*, the share of a class sitting in unusually
-  tight nearest-neighbor pairs (repeated stock frames), derived from the same computation.
+* **near-duplicate fraction** — the *redundancy*, the share of a class's images whose nearest
+  neighbor is unusually close (repeated stock frames), derived from the same computation.
 
 Alongside them, the global :func:`~dataeval.core.coverage_adaptive` result flags individual
 samples in under-sampled regions of the whole space.
@@ -41,12 +41,13 @@ from pydantic import PositiveFloat, PositiveInt
 from dataeval import Metadata
 from dataeval._helpers import BareLabels, is_labels_like, reject_filtered_metadata
 from dataeval._log import get_logger
+from dataeval.core._completeness import CompletenessResult
 from dataeval.core._completeness import completeness as _completeness
 from dataeval.core._coverage import CoverageResult, coverage_adaptive, coverage_naive
 from dataeval.exceptions import ShapeMismatchError
 from dataeval.protocols import AnnotatedDataset, Array, ArrayLike, FeatureExtractor, LabelsLike
 from dataeval.types import ClassAxis, DataFrameOutput, Evaluator, EvaluatorConfig, set_metadata
-from dataeval.types._config import OpenUnitInterval
+from dataeval.types._config import OpenUnitInterval, UnitInterval
 
 _logger = get_logger(__name__)
 
@@ -61,6 +62,10 @@ _PER_CLASS_SCHEMA: dict[str, pl.DataType] = {
     "isotropy": pl.Float64(),
     "near_duplicate_fraction": pl.Float64(),
     "assessable": pl.Boolean(),
+    # Appended, so the columns released before them keep their positions.
+    "is_clustered": pl.Boolean(),
+    "is_one_dimensional": pl.Boolean(),
+    "has_near_duplicates": pl.Boolean(),
 }
 
 
@@ -69,6 +74,19 @@ def _mean_dispersion(points: NDArray[np.float64]) -> float:
     return float(np.linalg.norm(points - points.mean(axis=0), axis=1).mean())
 
 
+def _nearest_distances(result: CompletenessResult, n: int) -> NDArray[np.float64]:
+    """Each of the ``n`` points' distance to its nearest other point, from ``completeness``'s pairs.
+
+    The pairs list two mutual nearest neighbors once, so they count pairs, not points. A point's
+    nearest-neighbor distance is the smallest among the pairs it is in.
+    """
+    nearest = np.full(n, np.inf)
+    distances = np.asarray(result["nearest_neighbor_distances"], dtype=np.float64)
+    for points in np.asarray(result["nearest_neighbor_pairs"], dtype=np.intp).reshape(-1, 2).T:
+        np.minimum.at(nearest, points, distances)
+    return nearest
+
+
 class CoverageOutput(DataFrameOutput):
     """
     A dataset's per-class embedding-space coverage.
@@ -76,10 +94,47 @@ class CoverageOutput(DataFrameOutput):
     The wrapped DataFrame is the per-class breakdown — one row per class, sorted with the
     lowest-dispersion assessable classes first (the ones most worth collecting varied data
     for) and unassessable classes last — with columns ``class``, ``count``, ``uncovered``,
-    ``uncovered_fraction``, ``dispersion``, ``isotropy``, ``near_duplicate_fraction``, and
-    ``assessable``. ``isotropy`` and ``near_duplicate_fraction`` are null for classes below
-    the sample floors that make them meaningful. The sample-level coverage detail hangs off
-    it as attributes.
+    ``uncovered_fraction``, ``dispersion``, ``isotropy``, ``near_duplicate_fraction``,
+    ``assessable``, ``is_clustered``, ``is_one_dimensional``, and ``has_near_duplicates``.
+    ``isotropy`` and ``near_duplicate_fraction`` are null for classes below the sample floors
+    that make them meaningful. The sample-level coverage detail hangs off it as attributes.
+
+    The last three columns are a verdict on each variety signal, with cutoffs that are
+    parameters of :class:`Coverage`. They are independent, and a class can trip more than one:
+
+    - is_clustered: bool - True if dispersion < dispersion_factor (default 0.5): the class
+      spreads less than half as far from its centroid as the median class.
+    - is_one_dimensional: bool - True if isotropy < isotropy_factor (default 0.5) x the median
+      isotropy of the classes that report one, the class itself included. Isotropy is not
+      normalized by the evaluator, since it depends on the embedding dimension and on any
+      reduction applied first, so it is judged against its peers.
+    - has_near_duplicates: bool - True if near_duplicate_fraction > near_duplicate_threshold
+      (default 0.1).
+
+    A flag is null where it cannot judge, which is new: the flags of
+    :class:`~dataeval.bias.Balance` and :class:`~dataeval.bias.Diversity` are never null. A
+    flag is null where its signal is null, that is, for a class below ``min_class_samples`` or,
+    for ``is_one_dimensional``, below the isotropy sample floor. It is also null on every row
+    where the median it is judged against is not positive. A median spread of 0 leaves
+    ``dispersion`` in raw embedding units, and a median isotropy of 0 would flag no class at
+    all. With a single assessable class the median is the class itself, so neither
+    ``is_clustered`` nor ``is_one_dimensional`` can trip however flat the class is.
+
+    The defaults are conservative, so a flag means a class has collapsed, not that it sits a
+    little below its peers. Tighten a factor to catch a partial collapse. They are tuned for
+    deep embeddings, and two signals depend on more than the data:
+
+    - Isotropy rises with class size, so a small class reads lower beside large ones.
+      Tighten ``isotropy_factor`` only for classes of similar size.
+    - The near-duplicate share of a class with no duplicates rises as the embedding dimension
+      falls. On simulated Gaussian classes of 100 to 400 images it exceeded 0.1 for most
+      classes in 2 dimensions, for about half in 3 and for a few up to 7, and for none in 8
+      or more. Raise ``near_duplicate_threshold`` before reading ``has_near_duplicates`` on a
+      low-dimensional embedding, such as a few principal components.
+
+    .. versionadded:: 1.2
+        The ``is_clustered``, ``is_one_dimensional`` and ``has_near_duplicates`` columns,
+        after ``assessable`` so the earlier columns keep their positions.
 
     Attributes
     ----------
@@ -130,8 +185,12 @@ class Coverage(Evaluator):
       class); low means clustered even if numerous.
     * **isotropy** — shape of that spread (in how many independent directions the class
       varies); low means it varies along few axes even when dispersion is high.
-    * **near_duplicate_fraction** — redundancy (share of the class in unusually tight
-      nearest-neighbor pairs); high means repeated / near-identical samples.
+    * **near_duplicate_fraction** — redundancy (share of the class's images whose nearest
+      neighbor is unusually close); high means repeated / near-identical samples.
+
+    Each signal comes with a boolean verdict, ``is_clustered``, ``is_one_dimensional`` and
+    ``has_near_duplicates``, so the output table is the per-class gap report. See
+    :class:`CoverageOutput` for the exact rules and when a flag is null.
 
     Parameters
     ----------
@@ -156,8 +215,27 @@ class Coverage(Evaluator):
         effective-dimension estimate is degenerate when samples do not exceed dimensions).
         When None, defaults to one more than the embedding dimensionality.
     near_duplicate_factor : float, default 0.5
-        A nearest-neighbor pair counts as a near-duplicate when its distance is below this
-        fraction of the class's median nearest-neighbor distance (scale-free).
+        An image counts as a near-duplicate when its nearest neighbor in its class is no farther
+        than this fraction of a typical class's median nearest-neighbor distance (scale-free).
+        A scale for a distance, which sets ``near_duplicate_fraction``. Not to be confused with
+        ``near_duplicate_threshold``, the share of images that sets the flag.
+    dispersion_factor : float, default 0.5
+        ``is_clustered`` is True where ``dispersion < dispersion_factor``, that is, where a class
+        spreads less than this share of the median class's spread. Must be in (0, 1); a factor
+        near 1 flags about half the classes by construction.
+
+        .. versionadded:: 1.2
+    isotropy_factor : float, default 0.5
+        ``is_one_dimensional`` is True where ``isotropy < isotropy_factor * median(isotropy)``,
+        the median taken over the classes that report an isotropy. Must be in (0, 1).
+
+        .. versionadded:: 1.2
+    near_duplicate_threshold : float, default 0.1
+        ``has_near_duplicates`` is True where ``near_duplicate_fraction`` is above this share of
+        the class's images. Must be in [0, 1]: 0 flags any near-duplicate, and 1 turns the flag
+        off. A share, where ``near_duplicate_factor`` is a distance scale.
+
+        .. versionadded:: 1.2
     config : Coverage.Config or None, default None
         Optional configuration object; parameters passed directly to ``__init__`` override
         its values.
@@ -185,10 +263,25 @@ class Coverage(Evaluator):
     high-dimensional extractor most classes fall below that floor and report ``isotropy``
     as null rather than zero. Reduce the embedding dimensionality (for example with PCA) if
     you need the signal on classes of a few hundred samples. ``near_duplicate_fraction`` is the
-    share of within-class nearest-neighbor pairs closer than ``near_duplicate_factor`` x the
-    class median, surfacing repeated / near-identical samples that inflate counts without
-    adding variety. Embeddings are auto-rescaled to the unit interval for the coverage
-    computation. This evaluator assumes one embedding per label (image classification). For
+    share of a class's images whose nearest other image in the class is within
+    ``near_duplicate_factor`` x the *median* across assessable classes of each class's median
+    nearest-neighbor distance, surfacing repeated / near-identical samples that inflate counts
+    without adding variety. Both images of a near-identical pair count, and an exact copy counts
+    the same as one that differs by float noise. Like ``dispersion``, the scale comes from the
+    peers rather than the class itself, so a tight clump that is most of its class still shows;
+    the cost is that a class whose images sit that close together by nature reads high too.
+    With a single assessable class there are no peers, and a clump of more than half of it goes
+    unflagged. Embeddings are auto-rescaled to the unit interval for the coverage
+    computation.
+
+    The flag defaults are conservative and tuned for deep embeddings: at 0.5 a flag means a
+    class has collapsed. A flag is null where its signal is null, and on every row where the
+    median it is judged against is not positive. On an embedding of fewer than about 8
+    dimensions, healthy classes can have a near-duplicate share above the 0.1 default, so raise
+    ``near_duplicate_threshold`` there. Isotropy rises with class size, so tighten
+    ``isotropy_factor`` only for classes of similar size.
+
+    This evaluator assumes one embedding per label (image classification). For
     object detection, wrap the dataset with :class:`~dataeval.data.DetectionCrops` to present
     its boxes as an image-classification dataset (one crop per detection, aligned 1:1 with the
     labels) and evaluate that, or supply detection-level embeddings you have computed yourself.
@@ -244,8 +337,17 @@ class Coverage(Evaluator):
         isotropy_min_samples : int or None, default None
             Minimum samples for a class's isotropy to be reported (None: dims + 1).
         near_duplicate_factor : float, default 0.5
-            Fraction of the class median nearest-neighbor distance below which a pair is a
-            near-duplicate.
+            Fraction of a typical class's median nearest-neighbor distance within which an
+            image's nearest neighbor makes it a near-duplicate (a distance scale, which sets
+            ``near_duplicate_fraction``).
+        dispersion_factor : float, default 0.5
+            ``is_clustered`` is True where ``dispersion`` is below this, in (0, 1).
+        isotropy_factor : float, default 0.5
+            ``is_one_dimensional`` is True where ``isotropy`` is below this share of the median
+            isotropy, in (0, 1).
+        near_duplicate_threshold : float, default 0.1
+            ``has_near_duplicates`` is True where ``near_duplicate_fraction`` is above this
+            share of images, in [0, 1].
         """
 
         extractor: FeatureExtractor | None = None
@@ -256,6 +358,9 @@ class Coverage(Evaluator):
         min_class_samples: PositiveInt = 20
         isotropy_min_samples: PositiveInt | None = None
         near_duplicate_factor: PositiveFloat = 0.5
+        dispersion_factor: OpenUnitInterval = 0.5
+        isotropy_factor: OpenUnitInterval = 0.5
+        near_duplicate_threshold: UnitInterval = 0.1
 
     # Set by apply_config from Config.
     extractor: FeatureExtractor | None
@@ -266,6 +371,9 @@ class Coverage(Evaluator):
     min_class_samples: int
     isotropy_min_samples: int | None
     near_duplicate_factor: float
+    dispersion_factor: float
+    isotropy_factor: float
+    near_duplicate_threshold: float
     config: Config
 
     def __init__(
@@ -279,6 +387,9 @@ class Coverage(Evaluator):
         min_class_samples: int | None = None,
         isotropy_min_samples: int | None = None,
         near_duplicate_factor: float | None = None,
+        dispersion_factor: float | None = None,
+        isotropy_factor: float | None = None,
+        near_duplicate_threshold: float | None = None,
         config: Config | None = None,
     ) -> None:
         super().__init__(locals())
@@ -340,20 +451,6 @@ class Coverage(Evaluator):
         """Minimum class size for a non-degenerate isotropy estimate (samples must exceed dims)."""
         return self.isotropy_min_samples if self.isotropy_min_samples is not None else dim + 1
 
-    def _near_duplicate_fraction(self, distances: Any) -> float | None:
-        """Share of nearest-neighbor pairs closer than ``near_duplicate_factor`` x the class median.
-
-        Relative-to-median so it is scale-free. A zero median means at least half the pairs are
-        exact duplicates, so the exact-duplicate share is reported instead.
-        """
-        dists = np.asarray(distances, dtype=np.float64)
-        if dists.size == 0:
-            return None
-        median = float(np.median(dists))
-        if median == 0.0:
-            return float(np.mean(dists == 0.0))
-        return float(np.mean(dists < self.near_duplicate_factor * median))
-
     def _per_class(
         self,
         embeddings: NDArray[np.float64],
@@ -376,7 +473,8 @@ class Coverage(Evaluator):
         # Normalize by the median assessable class, so a typical class scores ~1 regardless of
         # how far apart the classes sit in the embedding space (the global spread would conflate
         # within-class variety with between-class separation). Avoid /0.
-        reference = (float(np.median(list(raw_spread.values()))) if raw_spread else 1.0) or 1.0
+        median_spread = float(np.median(list(raw_spread.values()))) if raw_spread else 0.0
+        reference = median_spread or 1.0
 
         # One completeness pass per assessable class yields both directional variety (isotropy)
         # and the nearest-neighbor distances that expose near-duplicates.
@@ -388,27 +486,46 @@ class Coverage(Evaluator):
         # don't exceed dimensions, so it needs more samples than dispersion (default: more than dims).
         isotropy_floor = max(self.min_class_samples, self._isotropy_floor(embeddings.shape[1]))
         isotropy = {cls: float(r["isotropy"]) for cls, r in results.items() if counts[cls] >= isotropy_floor}
-        near_duplicate = {
-            cls: self._near_duplicate_fraction(r["nearest_neighbor_distances"]) for cls, r in results.items()
-        }
+        # Isotropy is not normalized, so its flag compares each class with the median class instead.
+        median_isotropy = float(np.median(list(isotropy.values()))) if isotropy else 0.0
+        # Each image's distance to its nearest other image in its class; a class of one has none.
+        nearest = {cls: _nearest_distances(r, counts[cls]) for cls, r in results.items() if counts[cls] > 1}
+        # Like dispersion, measure against a typical (median) class's nearest-neighbor spacing rather than
+        # the class's own, which a tight clump covering most of the class would set. `<=` keeps exact
+        # duplicates when that spacing is itself zero.
+        spacing = float(np.median([np.median(d) for d in nearest.values()])) if nearest else 0.0
+        near_duplicate = {cls: float(np.mean(d <= self.near_duplicate_factor * spacing)) for cls, d in nearest.items()}
 
         rows: list[dict[str, Any]] = []
         for cls, mask in masks.items():
             count = counts[cls]
             uncovered = int(is_uncovered[mask].sum())
             assessable = cls in raw_spread
+            dispersion = raw_spread[cls] / reference if assessable else None
+            class_isotropy = isotropy.get(cls)
+            duplicates = near_duplicate.get(cls)
             rows.append({
                 "class": index2label.get(cls, str(cls)),
                 "count": count,
                 "uncovered": uncovered,
                 "uncovered_fraction": uncovered / count if count else 0.0,
                 # Spread relative to a typical class; < 1 == clustered / near-duplicate.
-                "dispersion": raw_spread[cls] / reference if assessable else None,
+                "dispersion": dispersion,
                 # In how many independent directions the class varies; null when too few samples.
-                "isotropy": isotropy.get(cls),
-                # Share of the class sitting in near-duplicate nearest-neighbor pairs.
-                "near_duplicate_fraction": near_duplicate.get(cls),
+                "isotropy": class_isotropy,
+                # Share of the class's images whose nearest neighbor is a near-duplicate.
+                "near_duplicate_fraction": duplicates,
                 "assessable": assessable,
+                # A flag is null where its signal is, or where the median it is judged against is not positive.
+                "is_clustered": (
+                    dispersion < self.dispersion_factor if dispersion is not None and median_spread > 0 else None
+                ),
+                "is_one_dimensional": (
+                    class_isotropy < self.isotropy_factor * median_isotropy
+                    if class_isotropy is not None and median_isotropy > 0
+                    else None
+                ),
+                "has_near_duplicates": duplicates > self.near_duplicate_threshold if duplicates is not None else None,
             })
         # Lowest-dispersion assessable classes first (most worth broadening); unassessable last.
         return sorted(rows, key=lambda row: (not row["assessable"], row["dispersion"] or 0.0, row["class"]))
@@ -474,6 +591,23 @@ class Coverage(Evaluator):
         │ plane  ┆ 27    ┆ 0         ┆ 1.194517   │
         │ person ┆ 20    ┆ 1         ┆ 1.488841   │
         └────────┴───────┴───────────┴────────────┘
+
+        The flags give the verdict. ``car`` spreads less than half as far as the median class.
+        ``is_one_dimensional`` is null because no class has more crops than the embedding has
+        dimensions, so none reports an isotropy:
+
+        >>> result.data().select("class", "is_clustered", "is_one_dimensional", "has_near_duplicates")
+        shape: (4, 4)
+        ┌────────┬──────────────┬────────────────────┬─────────────────────┐
+        │ class  ┆ is_clustered ┆ is_one_dimensional ┆ has_near_duplicates │
+        │ ---    ┆ ---          ┆ ---                ┆ ---                 │
+        │ str    ┆ bool         ┆ bool               ┆ bool                │
+        ╞════════╪══════════════╪════════════════════╪═════════════════════╡
+        │ car    ┆ true         ┆ null               ┆ false               │
+        │ boat   ┆ false        ┆ null               ┆ false               │
+        │ plane  ┆ false        ┆ null               ┆ false               │
+        │ person ┆ false        ┆ null               ┆ false               │
+        └────────┴──────────────┴────────────────────┴─────────────────────┘
         """
         reject_filtered_metadata(dataset, "Coverage")
 

{dataeval-1.2.0rc1 → dataeval-1.2.0rc2}/src/dataeval/shift/_drift/_base.py

@@ -102,6 +102,9 @@ class DriftOutput(DictOutput, Generic[TDetails]):
         Populated when the detector's ``extractor`` is a
         :class:`~dataeval.protocols.NamedFeatureExtractor`, such as
         :class:`~dataeval.Metadata`; empty otherwise.
+    baseline : Sequence[float]
+        Metric value of each reference chunk, from which the thresholds are
+        derived. Populated in chunked mode and empty otherwise.
     """
 
     drifted: bool
@@ -110,6 +113,11 @@ class DriftOutput(DictOutput, Generic[TDetails]):
     metric_name: str
     details: TDetails
     feature_names: Sequence[str] = ()
+    baseline: Sequence[float] = ()
+
+    @property
+    def plot_type(self) -> Literal["drift"]:
+        return "drift"
 
 
 @dataclass(frozen=True, repr=False)
@@ -550,7 +558,7 @@ class ChunkedDrift(Generic[TDetails]):
         *reference_data : Any
             Reference dataset(s), forwarded verbatim to the underlying detector's
             ``fit()``. Most detectors take a single reference; detectors with a
-            calibrated baseline (e.g. :class:`DriftWasserstein`) take both a
+            validation baseline (e.g. :class:`DriftWasserstein`) take both a
             training and a validation reference. When the underlying detector has an
             ``extractor`` configured, each reference may be any input the extractor
             accepts (e.g. a full MAITE dataset or raw images); otherwise each must be
@@ -700,6 +708,7 @@ class ChunkedDrift(Generic[TDetails]):
             distance=distance,
             metric_name=self._detector._metric_name,
             feature_names=self._detector._feature_names,
+            baseline=tuple(float(v) for v in self._baseline_values),
             details=df,
         )