dataeval 1.2.0rc0__tar.gz → 1.2.0rc2__tar.gz

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dataeval
-Version: 1.2.0rc0
+Version: 1.2.0rc2
 Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
 Project-URL: Homepage, https://dataeval.ai/
 Project-URL: Repository, https://github.com/aria-ml/dataeval/

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/pyproject.toml

@@ -128,8 +128,9 @@ type = [
 ]
 docs = [
   "numpy>=1.24.2",  # floor capped by the JATIC program standard; uv.lock resolves current
+  "av>=12.0.0",
   "certifi>=2024.07.04",
-  "dataeval-plots[plotly]>=0.0.15",
+  "dataeval-plots[plotly]>=0.0.16",
   "ipykernel>=6.26.0",
   "ipywidgets>=8.1.1",
   "jinja2>=3.1.6",

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/src/dataeval/_embeddings.py

@@ -19,9 +19,8 @@ from dataeval.extractors import FlattenExtractor
 from dataeval.protocols import (
     AnnotatedDataset,
     Array,
-    ArrayLike,
-    Dataset,
     FeatureExtractor,
+    ImageDataset,
     ProgressCallback,
 )
 from dataeval.utils.data import requires_maite_dataset, unwrap_image
@@ -121,7 +120,7 @@ class Embeddings(Array, FeatureExtractor):
     def __init__(
         self,
         # Technically more permissive than ImageClassificationDataset or ObjectDetectionDataset
-        dataset: Dataset[tuple[ArrayLike, Any, Any]] | Dataset[ArrayLike] | None = None,
+        dataset: ImageDataset | None = None,
         extractor: FeatureExtractor | None = None,
         batch_size: int | None = None,
         path: Path | str | None = None,
@@ -184,7 +183,7 @@ class Embeddings(Array, FeatureExtractor):
         return self._dataset is not None
 
     @requires_maite_dataset("dataset", expected="image_only")
-    def bind(self, dataset: Dataset[tuple[ArrayLike, Any, Any]] | Dataset[ArrayLike]) -> Self:
+    def bind(self, dataset: ImageDataset) -> Self:
         """Bind this instance to a dataset.
 
         Attaches a dataset to this Embeddings instance for embedding computation.
@@ -414,7 +413,7 @@ class Embeddings(Array, FeatureExtractor):
             # In-memory only
             self._embeddings = np.empty(full_shape, dtype=dtype)
 
-    def new(self, dataset: Dataset[tuple[ArrayLike, Any, Any]] | Dataset[ArrayLike]) -> Self:
+    def new(self, dataset: ImageDataset) -> Self:
         """
         Create new Embeddings instance with a different dataset.
 

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/src/dataeval/_metadata/_metadata.py

@@ -93,8 +93,8 @@ from dataeval.core._bin import (
 from dataeval.core._compute_stats import FactorResult, StatsResult
 from dataeval.exceptions import NotFittedError, ShapeMismatchError
 from dataeval.protocols import (
-    AnnotatedDataset,
     Array,
+    Dataset,
     DatumMetadata,
     FeatureExtractor,
     ProgressCallback,
@@ -549,7 +549,7 @@ class Metadata(Array, FeatureExtractor):
 
     Parameters
     ----------
-    dataset : AnnotatedDataset or None, default None
+    dataset : Dataset or None, default None
         Dataset that provides original targets and metadata for processing. When None,
         creates an unbound instance that can be used as a reusable feature extractor.
         Use :meth:`bind` to attach a dataset later, or pass data directly to :meth:`__call__`.
@@ -650,7 +650,7 @@ class Metadata(Array, FeatureExtractor):
     @requires_maite_dataset("dataset", expected="any_target")
     def __init__(
         self,
-        dataset: AnnotatedDataset[tuple[Any, Any, DatumMetadata]] | None = None,
+        dataset: Dataset[tuple[Any, Any, DatumMetadata]] | None = None,
         *,
         task: TaskOverride | None = None,
         continuous_factor_bins: Mapping[str, int | Sequence[float]] | None = None,
@@ -999,7 +999,7 @@ class Metadata(Array, FeatureExtractor):
     def load(
         cls,
         path: Path | str,
-        dataset: AnnotatedDataset[tuple[Any, Any, DatumMetadata]] | None = None,
+        dataset: Dataset[tuple[Any, Any, DatumMetadata]] | None = None,
         *,
         continuous_factor_bins: Mapping[str, int | Sequence[float]] | None = None,
         encoding: str | Path | Mapping[str, FactorEncoding] | None = None,
@@ -1271,7 +1271,7 @@ class Metadata(Array, FeatureExtractor):
         return self._dataset is not None
 
     @property
-    def dataset(self) -> AnnotatedDataset[tuple[Any, Any, DatumMetadata]] | None:
+    def dataset(self) -> Dataset[tuple[Any, Any, DatumMetadata]] | None:
         """The dataset these factors describe, or None when none is bound.
 
         Metadata references its dataset; it does not contain it or serve its images.
@@ -1283,7 +1283,7 @@ class Metadata(Array, FeatureExtractor):
 
         Returns
         -------
-        AnnotatedDataset or None
+        Dataset or None
             The bound dataset, the same object that was passed in.
 
         Example
@@ -1301,7 +1301,7 @@ class Metadata(Array, FeatureExtractor):
         return self._dataset is not None or self._is_structured
 
     @requires_maite_dataset("dataset", expected="any_target")
-    def bind(self, dataset: AnnotatedDataset[tuple[Any, Any, DatumMetadata]]) -> Self:
+    def bind(self, dataset: Dataset[tuple[Any, Any, DatumMetadata]]) -> Self:
         """Bind this instance to a dataset.
 
         Attaches a dataset to this Metadata instance for metadata extraction.
@@ -1309,7 +1309,7 @@ class Metadata(Array, FeatureExtractor):
 
         Parameters
         ----------
-        dataset : AnnotatedDataset
+        dataset : Dataset
             Dataset to bind for metadata extraction.
 
         Returns
@@ -1501,7 +1501,7 @@ class Metadata(Array, FeatureExtractor):
             return data[index]
         raise TypeError("Index must be an int, str, or slice.")
 
-    def new(self, dataset: AnnotatedDataset[tuple[Any, Any, DatumMetadata]]) -> Self:
+    def new(self, dataset: Dataset[tuple[Any, Any, DatumMetadata]]) -> Self:
         """Create new Metadata instance with a different dataset.
 
         Generate a new Metadata object using the same configuration
@@ -1509,7 +1509,7 @@ class Metadata(Array, FeatureExtractor):
 
         Parameters
         ----------
-        dataset : AnnotatedDataset
+        dataset : Dataset
             Dataset that provides metadata for the new Metadata instance.
 
         Returns
@@ -1672,7 +1672,7 @@ class Metadata(Array, FeatureExtractor):
         data = derived.factor_data
         return data[:, [columns[name] for name in names]]
 
-    def _fit(self, data: AnnotatedDataset[tuple[Any, Any, DatumMetadata]]) -> None:
+    def _fit(self, data: Dataset[tuple[Any, Any, DatumMetadata]]) -> None:
         """Bind to ``data`` and freeze the encoding derived from it.
 
         Notes
@@ -5146,7 +5146,9 @@ class Metadata(Array, FeatureExtractor):
         data = hold_back_invalid_known(data)
 
         unique_labels = np.unique(data.class_labels) if len(data.class_labels) else np.array([], dtype=np.intp)
-        self._index2label = build_index2label(self._dataset.metadata.get("index2label", None), unique_labels)
+        self._index2label = build_index2label(
+            (getattr(self._dataset, "metadata", None) or {}).get("index2label", None), unique_labels
+        )
         self._adopt(structurer, data)
         # From the bound dataset rather than the `_structuring_source` replacement: the replacement holds
         # the datums of a view, and it is the view's own chain that says which items share a root.

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/src/dataeval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.0rc0'
-__version_tuple__ = version_tuple = (1, 2, 0, 'rc0')
+__version__ = version = '1.2.0rc2'
+__version_tuple__ = version_tuple = (1, 2, 0, 'rc2')
 
 __commit_id__ = commit_id = None

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/src/dataeval/core/_completeness.py

@@ -27,10 +27,13 @@ class CompletenessResult(TypedDict):
         space actually spanned by embeddings.
     nearest_neighbor_pairs : Sequence[tuple[int, int]]
         Sequence of tuples (i, j) representing point indices and their nearest neighbors,
-        sorted by decreasing nearest neighbor distance. Each pair appears only once.
+        sorted by decreasing nearest neighbor distance. Each pair appears only once, so two
+        points that are each other's nearest neighbor share one entry. ``j`` is never ``i``,
+        even when ``i`` has an exact duplicate.
     nearest_neighbor_distances : Sequence[float]
         Distance between each pair in ``nearest_neighbor_pairs``, aligned 1:1 and sharing
         the same decreasing order. The small-distance tail flags near-duplicate samples.
+        Each point's own nearest-neighbor distance is the smallest among the pairs it is in.
     """
 
     completeness: float
@@ -246,16 +249,12 @@ def completeness(embeddings: Array) -> CompletenessResult:
             nearest_neighbor_distances=[],
         )
 
-    # Fit nearest neighbors model (k=2 because first neighbor is always the point itself)
-    nn_model = NearestNeighbors(n_neighbors=2, algorithm="auto")
-    nn_model.fit(embeddings)
-
-    # Get distances and indices for nearest neighbors
-    distances, indices = nn_model.kneighbors(embeddings)
-
-    # Extract the actual nearest neighbor (second column, index 1)
-    nearest_neighbors = indices[:, 1]
-    nearest_distances = distances[:, 1]
+    # Each point's nearest other point. Querying the fitted points themselves (no X) makes sklearn drop each
+    # point from its own neighbors by index, which stays right for exact duplicates: there a zero-distance tie
+    # can rank the point after its copy, so "the first neighbor is the point itself" does not hold.
+    distances, indices = NearestNeighbors(n_neighbors=1).fit(embeddings).kneighbors()
+    nearest_neighbors = indices[:, 0]
+    nearest_distances = distances[:, 0]
 
     # Create pairs, deduplicating so each pair appears only once
     seen_pairs = set()

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/src/dataeval/core/_compute_stats.py

@@ -10,7 +10,7 @@ from typing_extensions import TypedDict, TypeVar
 from dataeval._experimental import deprecated
 from dataeval.core._calculators._base import Calculator
 from dataeval.flags import ImageStats
-from dataeval.protocols import ArrayLike, Dataset, ProgressCallback
+from dataeval.protocols import ArrayLike, ImageDataset, ProgressCallback
 from dataeval.types import Aggregator, SourceIndex, StatsMap
 from dataeval.utils.preprocessing import (
     BoxLike,
@@ -166,7 +166,7 @@ def _produced_stat_names(calculators: Sequence[tuple[type[Calculator[Any]], Flag
 
 @deprecated(since="1.2", removal="2.0", alternative="image_stats")
 def compute_stats(
-    data: Iterable[ArrayLike] | Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    data: Iterable[ArrayLike] | ImageDataset,
     *,
     boxes: Iterable[Iterable[BoxLike] | None] | None = None,
     stats: StatsRequest = ImageStats.ALL,

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/src/dataeval/core/_image_stats.py

@@ -28,7 +28,14 @@ from dataeval.core._compute_stats import (
 )
 from dataeval.data import unzip_dataset
 from dataeval.flags import ImageStats
-from dataeval.protocols import ArrayLike, Dataset, ObjectDetectionTarget, ProgressCallback, _is_protocol_instance
+from dataeval.protocols import (
+    ArrayLike,
+    Dataset,
+    ImageDataset,
+    ObjectDetectionTarget,
+    ProgressCallback,
+    _is_protocol_instance,
+)
 from dataeval.types import FactorLevel, SourceIndex
 from dataeval.utils._internal import PoolWrapper
 from dataeval.utils.preprocessing import (
@@ -825,7 +832,7 @@ def _aggregate_batch(
 
 
 def image_stats(  # noqa: C901
-    data: Iterable[ArrayLike] | Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    data: Iterable[ArrayLike] | ImageDataset,
     *,
     boxes: Iterable[Iterable[BoxLike] | None] | None = None,
     stats: StatsRequest = ImageStats.ALL,
@@ -845,7 +852,7 @@ def image_stats(  # noqa: C901
 
     Parameters
     ----------
-    data : Iterable[ArrayLike] | Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]]
+    data : Iterable[ArrayLike] | ImageDataset
         An iterable of images or a Dataset to compute statistics on.
     boxes : Iterable[Iterable[BoxLike] | None] | None
         Optional bounding boxes for each image. If None, defers to the data provided.

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/src/dataeval/data/_frames.py

@@ -23,24 +23,26 @@ from dataeval.data._selectors import (
     _StageTrace,
     sequence_infos,
 )
-from dataeval.data._video_proxies import checked_index, inherited_metadata
+from dataeval.data._video_proxies import as_tracking, checked_index, inherited_metadata
 from dataeval.flags import ImageStats
 from dataeval.protocols import (
     AnnotatedDataset,
     ArrayLike,
+    Dataset,
     DatasetMetadata,
     DatumMetadata,
-    MultiobjectTrackingDataset,
+    MultiobjectTrackingDatum,
     MultiobjectTrackingTarget,
     ObjectDetectionDatum,
     ObjectDetectionTarget,
     SingleFrameObjectTrackingTarget,
+    VideoDataset,
 )
 from dataeval.types import FactorLevel, SourceIndex
 from dataeval.types._target import instance_arrays
 from dataeval.types._track import frame_number
 from dataeval.utils._array import as_numpy
-from dataeval.utils.data import requires_maite_dataset
+from dataeval.utils.data import validate_dataset
 from dataeval.utils.preprocessing import BoundingBox, normalize_image_shape
 
 _logger = get_logger(__name__)
@@ -202,9 +204,10 @@ class SequenceFrames(AnnotatedDataset[ObjectDetectionDatum]):
 
     Parameters
     ----------
-    dataset : MultiobjectTrackingDataset
+    dataset : VideoDataset
         The source dataset. Each datum is a MAITE ``(VideoStream, MultiobjectTrackingTarget,
-        metadata)`` 3-tuple.
+        metadata)`` 3-tuple. The dataset's own ``metadata`` attribute, when it has one, supplies the ``id`` and
+        ``index2label`` of the view; without it the view is named ``dataset-frames`` and has no class names.
     selector : FrameSelector or None, default None
         Which frames take part, and what to record about each. None keeps every frame
         (:class:`~dataeval.data.AllFrames`). See :class:`~dataeval.data.FrameSelector`.
@@ -232,6 +235,7 @@ class SequenceFrames(AnnotatedDataset[ObjectDetectionDatum]):
     --------
     :class:`~dataeval.data.FrameSelector` : The frame-selection extension point
     :class:`~dataeval.data.DetectionCrops` : The same reframing one level down
+    :doc:`/notebooks/h2_select_frames` : Thin each video to a budget of frames and read what a kept frame stands for
 
     Notes
     -----
@@ -279,10 +283,10 @@ class SequenceFrames(AnnotatedDataset[ObjectDetectionDatum]):
     factor through the ordinary path. ``sequence`` as a factor is what
     ``split_on=["sequence"]`` needs to keep every frame of one video on one side of a split. The
     split's fractions then count videos rather than frames: over five videos, ``test_frac=0.2``
-    aims at one whole video, however many frames it has. With more than 20 distinct values
-    :class:`~dataeval.Metadata` bins ``sequence`` automatically, so a group is then a run of
-    consecutive videos (a video is still never split) and the fractions count those runs. For one
-    group per video, pass ``Metadata(frames, factor_levels={"sequence": range(n_videos)})``.
+    aims at one whole video, however many frames it has. The split groups by the recorded
+    ``sequence`` values, so every video is a group of its own however many there are, even where
+    there are too many for :class:`~dataeval.Metadata` to score one at a time and it bins
+    ``sequence`` for the bias evaluators.
 
     Frames of one video are not independent samples, so :class:`~dataeval.bias.Parity` clusters
     them by their source video, and :class:`~dataeval.bias.Balance` counts a video as one
@@ -317,12 +321,18 @@ class SequenceFrames(AnnotatedDataset[ObjectDetectionDatum]):
     >>> stats = image_stats(frames, stats=ImageStats.HASH)  # doctest: +SKIP
     """
 
-    @requires_maite_dataset("dataset", expected="multiobject_tracking")
     def __init__(
         self,
-        dataset: MultiobjectTrackingDataset,
+        dataset: VideoDataset,
         selector: FrameSelector | None = None,
     ) -> None:
+        # Videos with no targets become a tracking dataset of frames with no detections, so everything below
+        # reads one kind of dataset. The check is the one the signature would have made.
+        dataset = as_tracking(dataset)
+        if dataset is not None:
+            validate_dataset(
+                dataset, expected="multiobject_tracking", arg_name="dataset", caller="SequenceFrames.__init__"
+            )
         self._dataset = dataset
         self._selector = AllFrames() if selector is None else selector
         self._reads_pixels = bool(self._selector.needs & FrameInput.PIXELS)
@@ -381,11 +391,12 @@ class SequenceFrames(AnnotatedDataset[ObjectDetectionDatum]):
     # ------------------------------------------------------------------
 
     @property
-    def source(self) -> MultiobjectTrackingDataset:
+    def source(self) -> Dataset[MultiobjectTrackingDatum]:
         """The tracking dataset this wraps -- one link up the chain.
 
         Named to match :attr:`~dataeval.data.View.source`, so a mixed wrapping chain is walkable
-        through one public attribute and an invalidating operation inside it stays visible.
+        through one public attribute and an invalidating operation inside it stays visible. For a dataset of
+        videos with no targets this is the tracking dataset that gives every frame none, not the dataset passed in.
         """
         return self._dataset
 

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/src/dataeval/data/_locate.py

@@ -15,6 +15,7 @@ from numpy.typing import NDArray
 
 from dataeval.data._crops import CropPolicy, FillType, RegionType, SquareType
 from dataeval.data._tracks import build_tracks
+from dataeval.data._video_proxies import as_tracking
 from dataeval.data._view import View
 from dataeval.protocols import (
     DatumMetadata,
@@ -519,6 +520,7 @@ class SourceLocator:
     """
 
     def __init__(self, dataset: Any, *, source: Any | None = None) -> None:
+        dataset = as_tracking(dataset)
         self._dataset = dataset
         self._view = dataset if isinstance(dataset, View) else None
         # A view's root, not its `source`: `resolve_indices` steps one link, and a chain of

{dataeval-1.2.0rc0 → dataeval-1.2.0rc2}/src/dataeval/data/_segments.py

@@ -25,9 +25,9 @@ from dataeval.data._video_proxies import (
 from dataeval.flags import ImageStats
 from dataeval.protocols import (
     AnnotatedDataset,
+    Dataset,
     DatasetMetadata,
     DatumMetadata,
-    MultiobjectTrackingDataset,
     MultiobjectTrackingDatum,
 )
 from dataeval.utils.data import requires_maite_dataset
@@ -79,7 +79,7 @@ def _plan_costs(plan: NDArray[np.intp], info: SequenceInfo) -> tuple[int, int]:
 
 
 def _plan_dataset(
-    dataset: MultiobjectTrackingDataset, planner: SegmentPlanner
+    dataset: Dataset[MultiobjectTrackingDatum], planner: SegmentPlanner
 ) -> tuple[list[SequenceInfo], list[NDArray[np.intp]], set[int], int, int]:
     """Compute segment plans for each video in the dataset without decoding frames."""
     sequences: list[SequenceInfo] = []
@@ -120,7 +120,7 @@ class VideoSegments(AnnotatedDataset[MultiobjectTrackingDatum]):
 
     Parameters
     ----------
-    dataset : MultiobjectTrackingDataset
+    dataset : Dataset of MultiobjectTrackingDatum
         The source tracking dataset.
     planner : SegmentPlanner
         Planner determining where to cut each video. See :class:`~dataeval.data.Window`
@@ -181,7 +181,7 @@ class VideoSegments(AnnotatedDataset[MultiobjectTrackingDatum]):
     @requires_maite_dataset("dataset", expected="multiobject_tracking")
     def __init__(
         self,
-        dataset: MultiobjectTrackingDataset,
+        dataset: Dataset[MultiobjectTrackingDatum],
         planner: SegmentPlanner,
         *,
         timestamps: TimestampPolicy = "source",
@@ -240,7 +240,7 @@ class VideoSegments(AnnotatedDataset[MultiobjectTrackingDatum]):
     # ------------------------------------------------------------------
 
     @property
-    def source(self) -> MultiobjectTrackingDataset:
+    def source(self) -> Dataset[MultiobjectTrackingDatum]:
         """Underlying tracking dataset wrapped by this view."""
         return self._dataset