valor-lite 0.36.0__tar.gz → 0.36.2__tar.gz

{valor_lite-0.36.0 → valor_lite-0.36.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: valor-lite
-Version: 0.36.0
+Version: 0.36.2
 Summary: Evaluate machine learning models.
 Project-URL: homepage, https://www.striveworks.com
 Requires-Python: >=3.10

{valor_lite-0.36.0 → valor_lite-0.36.2}/valor_lite/classification/__init__.py

@@ -3,7 +3,7 @@ from .computation import (
     compute_confusion_matrix,
     compute_precision_recall_rocauc,
 )
-from .manager import DataLoader, Evaluator
+from .manager import DataLoader, Evaluator, Filter, Metadata
 from .metric import Metric, MetricType
 
 __all__ = [
@@ -14,4 +14,6 @@ __all__ = [
     "DataLoader",
     "Evaluator",
     "Metric",
+    "Metadata",
+    "Filter",
 ]

{valor_lite-0.36.0 → valor_lite-0.36.2}/valor_lite/classification/manager.py

@@ -1,4 +1,3 @@
-import warnings
 from dataclasses import asdict, dataclass
 
 import numpy as np
@@ -17,6 +16,7 @@ from valor_lite.classification.utilities import (
     unpack_confusion_matrix_into_metric_list,
     unpack_precision_recall_rocauc_into_metric_lists,
 )
+from valor_lite.exceptions import EmptyEvaluatorException, EmptyFilterException
 
 """
 Usage
@@ -85,6 +85,18 @@ class Filter:
     valid_label_indices: NDArray[np.int32] | None
     metadata: Metadata
 
+    def __post_init__(self):
+        # validate datum mask
+        if not self.datum_mask.any():
+            raise EmptyFilterException("filter removes all datums")
+
+        # validate label indices
+        if (
+            self.valid_label_indices is not None
+            and self.valid_label_indices.size == 0
+        ):
+            raise EmptyFilterException("filter removes all labels")
+
 
 class Evaluator:
     """
@@ -155,7 +167,6 @@ class Evaluator:
         datum_mask = np.ones(n_pairs, dtype=np.bool_)
         if datum_ids is not None:
             if not datum_ids:
-                warnings.warn("no valid filtered pairs")
                 return Filter(
                     datum_mask=np.zeros_like(datum_mask),
                     valid_label_indices=None,
@@ -173,7 +184,6 @@ class Evaluator:
         valid_label_indices = None
         if labels is not None:
             if not labels:
-                warnings.warn("no valid filtered pairs")
                 return Filter(
                     datum_mask=datum_mask,
                     valid_label_indices=np.array([], dtype=np.int32),
@@ -224,21 +234,6 @@ class Evaluator:
         NDArray[int32]
             The filtered label metadata.
         """
-        empty_datum_mask = not filter_.datum_mask.any()
-        empty_label_mask = (
-            filter_.valid_label_indices.size == 0
-            if filter_.valid_label_indices is not None
-            else False
-        )
-        if empty_datum_mask or empty_label_mask:
-            if empty_datum_mask:
-                warnings.warn("filter removes all datums")
-            if empty_label_mask:
-                warnings.warn("filter removes all labels")
-            return (
-                np.array([], dtype=np.float64),
-                np.zeros((self.metadata.number_of_labels, 2), dtype=np.int32),
-            )
         return filter_cache(
             detailed_pairs=self._detailed_pairs,
             datum_mask=filter_.datum_mask,
@@ -502,9 +497,7 @@ class Evaluator:
             A ready-to-use evaluator object.
         """
         if self._detailed_pairs.size == 0:
-            self._label_metadata = np.array([], dtype=np.int32)
-            warnings.warn("evaluator is empty")
-            return self
+            raise EmptyEvaluatorException()
 
         self._label_metadata = compute_label_metadata(
             ids=self._detailed_pairs[:, :3].astype(np.int32),

valor_lite-0.36.2/valor_lite/exceptions.py

@@ -0,0 +1,15 @@
+class EmptyEvaluatorException(Exception):
+    def __init__(self):
+        super().__init__(
+            "evaluator cannot be finalized as it contains no data"
+        )
+
+
+class EmptyFilterException(Exception):
+    def __init__(self, message: str):
+        super().__init__(message)
+
+
+class InternalCacheException(Exception):
+    def __init__(self, message: str):
+        super().__init__(message)

{valor_lite-0.36.0 → valor_lite-0.36.2}/valor_lite/object_detection/__init__.py

@@ -1,5 +1,5 @@
 from .annotation import Bitmask, BoundingBox, Detection, Polygon
-from .manager import DataLoader, Evaluator, Filter
+from .manager import DataLoader, Evaluator, Filter, Metadata
 from .metric import Metric, MetricType
 
 __all__ = [
@@ -12,4 +12,5 @@ __all__ = [
     "DataLoader",
     "Evaluator",
     "Filter",
+    "Metadata",
 ]

{valor_lite-0.36.0 → valor_lite-0.36.2}/valor_lite/object_detection/computation.py

@@ -1,4 +1,3 @@
-import warnings
 from enum import IntFlag, auto
 
 import numpy as np
@@ -280,14 +279,6 @@ def filter_cache(
     )
     detailed_pairs = detailed_pairs[~mask_null_pairs]
 
-    if detailed_pairs.size == 0:
-        warnings.warn("no valid filtered pairs")
-        return (
-            np.array([], dtype=np.float64),
-            np.array([], dtype=np.float64),
-            np.zeros((n_labels, 2), dtype=np.int32),
-        )
-
     # sorts by score, iou with ground truth id as a tie-breaker
     indices = np.lexsort(
         (
@@ -441,15 +432,6 @@ def compute_precion_recall(
     counts = np.zeros((n_ious, n_scores, n_labels, 6), dtype=np.float64)
     pr_curve = np.zeros((n_ious, n_labels, 101, 2))
 
-    if ranked_pairs.size == 0:
-        warnings.warn("no valid ranked pairs")
-        return (
-            (average_precision, mAP),
-            (average_recall, mAR),
-            counts,
-            pr_curve,
-        )
-
     # start computation
     ids = ranked_pairs[:, :5].astype(np.int32)
     gt_ids = ids[:, 1]

{valor_lite-0.36.0 → valor_lite-0.36.2}/valor_lite/object_detection/manager.py

@@ -5,6 +5,11 @@ import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
 
+from valor_lite.exceptions import (
+    EmptyEvaluatorException,
+    EmptyFilterException,
+    InternalCacheException,
+)
 from valor_lite.object_detection.annotation import (
     Bitmask,
     BoundingBox,
@@ -94,6 +99,21 @@ class Filter:
     mask_predictions: NDArray[np.bool_]
     metadata: Metadata
 
+    def __post_init__(self):
+        # validate datums mask
+        if not self.mask_datums.any():
+            raise EmptyFilterException("filter removes all datums")
+
+        # validate annotation masks
+        no_gts = self.mask_groundtruths.all()
+        no_pds = self.mask_predictions.all()
+        if no_gts and no_pds:
+            raise EmptyFilterException("filter removes all annotations")
+        elif no_gts:
+            warnings.warn("filter removes all ground truths")
+        elif no_pds:
+            warnings.warn("filter removes all predictions")
+
 
 class Evaluator:
     """
@@ -151,123 +171,407 @@ class Evaluator:
         """
         return self._metadata
 
-    def _add_datum(self, datum_id: str) -> int:
+    def create_filter(
+        self,
+        datum_ids: list[str] | None = None,
+        groundtruth_ids: list[str] | None = None,
+        prediction_ids: list[str] | None = None,
+        labels: list[str] | None = None,
+    ) -> Filter:
         """
-        Helper function for adding a datum to the cache.
+        Creates a filter object.
 
         Parameters
         ----------
-        datum_id : str
-            The datum identifier.
-
-        Returns
-        -------
-        int
-            The datum index.
+        datum_uids : list[str], optional
+            An optional list of string uids representing datums to keep.
+        groundtruth_ids : list[str], optional
+            An optional list of string uids representing ground truth annotations to keep.
+        prediction_ids : list[str], optional
+            An optional list of string uids representing prediction annotations to keep.
+        labels : list[str], optional
+            An optional list of labels to keep.
         """
-        if datum_id not in self.datum_id_to_index:
-            if len(self.datum_id_to_index) != len(self.index_to_datum_id):
-                raise RuntimeError("datum cache size mismatch")
-            idx = len(self.datum_id_to_index)
-            self.datum_id_to_index[datum_id] = idx
-            self.index_to_datum_id.append(datum_id)
-        return self.datum_id_to_index[datum_id]
+        mask_datums = np.ones(self._detailed_pairs.shape[0], dtype=np.bool_)
 
-    def _add_groundtruth(self, annotation_id: str) -> int:
+        # filter datums
+        if datum_ids is not None:
+            if not datum_ids:
+                raise EmptyFilterException("filter removes all datums")
+            valid_datum_indices = np.array(
+                [self.datum_id_to_index[uid] for uid in datum_ids],
+                dtype=np.int32,
+            )
+            mask_datums = np.isin(
+                self._detailed_pairs[:, 0], valid_datum_indices
+            )
+
+        filtered_detailed_pairs = self._detailed_pairs[mask_datums]
+        n_pairs = self._detailed_pairs[mask_datums].shape[0]
+        mask_groundtruths = np.zeros(n_pairs, dtype=np.bool_)
+        mask_predictions = np.zeros_like(mask_groundtruths)
+
+        # filter by ground truth annotation ids
+        if groundtruth_ids is not None:
+            valid_groundtruth_indices = np.array(
+                [self.groundtruth_id_to_index[uid] for uid in groundtruth_ids],
+                dtype=np.int32,
+            )
+            mask_groundtruths[
+                ~np.isin(
+                    filtered_detailed_pairs[:, 1],
+                    valid_groundtruth_indices,
+                )
+            ] = True
+
+        # filter by prediction annotation ids
+        if prediction_ids is not None:
+            valid_prediction_indices = np.array(
+                [self.prediction_id_to_index[uid] for uid in prediction_ids],
+                dtype=np.int32,
+            )
+            mask_predictions[
+                ~np.isin(
+                    filtered_detailed_pairs[:, 2],
+                    valid_prediction_indices,
+                )
+            ] = True
+
+        # filter by labels
+        if labels is not None:
+            if not labels:
+                raise EmptyFilterException("filter removes all labels")
+            valid_label_indices = np.array(
+                [self.label_to_index[label] for label in labels] + [-1]
+            )
+            mask_groundtruths[
+                ~np.isin(filtered_detailed_pairs[:, 3], valid_label_indices)
+            ] = True
+            mask_predictions[
+                ~np.isin(filtered_detailed_pairs[:, 4], valid_label_indices)
+            ] = True
+
+        filtered_detailed_pairs, _, _ = filter_cache(
+            self._detailed_pairs,
+            mask_datums=mask_datums,
+            mask_ground_truths=mask_groundtruths,
+            mask_predictions=mask_predictions,
+            n_labels=len(self.index_to_label),
+        )
+
+        number_of_datums = (
+            len(datum_ids)
+            if datum_ids
+            else np.unique(filtered_detailed_pairs[:, 0]).size
+        )
+
+        return Filter(
+            mask_datums=mask_datums,
+            mask_groundtruths=mask_groundtruths,
+            mask_predictions=mask_predictions,
+            metadata=Metadata.create(
+                detailed_pairs=filtered_detailed_pairs,
+                number_of_datums=number_of_datums,
+                number_of_labels=len(self.index_to_label),
+            ),
+        )
+
+    def filter(
+        self, filter_: Filter
+    ) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.int32],]:
         """
-        Helper function for adding a ground truth annotation identifier to the cache.
+        Performs filtering over the internal cache.
 
         Parameters
         ----------
-        annotation_id : str
-            The ground truth annotation identifier.
+        filter_ : Filter
+            The filter parameterization.
 
         Returns
         -------
-        int
-            The ground truth annotation index.
+        NDArray[float64]
+            Filtered detailed pairs.
+        NDArray[float64]
+            Filtered ranked pairs.
+        NDArray[int32]
+            Label metadata.
         """
-        if annotation_id not in self.groundtruth_id_to_index:
-            if len(self.groundtruth_id_to_index) != len(
-                self.index_to_groundtruth_id
-            ):
-                raise RuntimeError("ground truth cache size mismatch")
-            idx = len(self.groundtruth_id_to_index)
-            self.groundtruth_id_to_index[annotation_id] = idx
-            self.index_to_groundtruth_id.append(annotation_id)
-        return self.groundtruth_id_to_index[annotation_id]
+        return filter_cache(
+            detailed_pairs=self._detailed_pairs,
+            mask_datums=filter_.mask_datums,
+            mask_ground_truths=filter_.mask_groundtruths,
+            mask_predictions=filter_.mask_predictions,
+            n_labels=len(self.index_to_label),
+        )
 
-    def _add_prediction(self, annotation_id: str) -> int:
+    def compute_precision_recall(
+        self,
+        iou_thresholds: list[float],
+        score_thresholds: list[float],
+        filter_: Filter | None = None,
+    ) -> dict[MetricType, list[Metric]]:
         """
-        Helper function for adding a prediction annotation identifier to the cache.
+        Computes all metrics except for ConfusionMatrix
 
         Parameters
         ----------
-        annotation_id : str
-            The prediction annotation identifier.
+        iou_thresholds : list[float]
+            A list of IOU thresholds to compute metrics over.
+        score_thresholds : list[float]
+            A list of score thresholds to compute metrics over.
+        filter_ : Filter, optional
+            A collection of filter parameters and masks.
 
         Returns
         -------
-        int
-            The prediction annotation index.
+        dict[MetricType, list]
+            A dictionary mapping MetricType enumerations to lists of computed metrics.
         """
-        if annotation_id not in self.prediction_id_to_index:
-            if len(self.prediction_id_to_index) != len(
-                self.index_to_prediction_id
-            ):
-                raise RuntimeError("prediction cache size mismatch")
-            idx = len(self.prediction_id_to_index)
-            self.prediction_id_to_index[annotation_id] = idx
-            self.index_to_prediction_id.append(annotation_id)
-        return self.prediction_id_to_index[annotation_id]
+        if not iou_thresholds:
+            raise ValueError("At least one IOU threshold must be passed.")
+        elif not score_thresholds:
+            raise ValueError("At least one score threshold must be passed.")
 
-    def _add_label(self, label: str) -> int:
+        if filter_ is not None:
+            _, ranked_pairs, label_metadata = self.filter(filter_=filter_)
+        else:
+            ranked_pairs = self._ranked_pairs
+            label_metadata = self._label_metadata
+
+        results = compute_precion_recall(
+            ranked_pairs=ranked_pairs,
+            label_metadata=label_metadata,
+            iou_thresholds=np.array(iou_thresholds),
+            score_thresholds=np.array(score_thresholds),
+        )
+        return unpack_precision_recall_into_metric_lists(
+            results=results,
+            label_metadata=label_metadata,
+            iou_thresholds=iou_thresholds,
+            score_thresholds=score_thresholds,
+            index_to_label=self.index_to_label,
+        )
+
+    def compute_confusion_matrix(
+        self,
+        iou_thresholds: list[float],
+        score_thresholds: list[float],
+        filter_: Filter | None = None,
+    ) -> list[Metric]:
         """
-        Helper function for adding a label to the cache.
+        Computes confusion matrices at various thresholds.
 
         Parameters
         ----------
-        label : str
-            The label associated with the annotation.
+        iou_thresholds : list[float]
+            A list of IOU thresholds to compute metrics over.
+        score_thresholds : list[float]
+            A list of score thresholds to compute metrics over.
+        filter_ : Filter, optional
+            A collection of filter parameters and masks.
 
         Returns
         -------
-        int
-            Label index.
+        list[Metric]
+            List of confusion matrices per threshold pair.
         """
-        label_id = len(self.index_to_label)
-        if label not in self.label_to_index:
-            if len(self.label_to_index) != len(self.index_to_label):
-                raise RuntimeError("label cache size mismatch")
-            self.label_to_index[label] = label_id
-            self.index_to_label.append(label)
-            label_id += 1
-        return self.label_to_index[label]
+        if not iou_thresholds:
+            raise ValueError("At least one IOU threshold must be passed.")
+        elif not score_thresholds:
+            raise ValueError("At least one score threshold must be passed.")
 
-    def _add_data(
+        if filter_ is not None:
+            detailed_pairs, _, _ = self.filter(filter_=filter_)
+        else:
+            detailed_pairs = self._detailed_pairs
+
+        if detailed_pairs.size == 0:
+            return []
+
+        results = compute_confusion_matrix(
+            detailed_pairs=detailed_pairs,
+            iou_thresholds=np.array(iou_thresholds),
+            score_thresholds=np.array(score_thresholds),
+        )
+        return unpack_confusion_matrix_into_metric_list(
+            results=results,
+            detailed_pairs=detailed_pairs,
+            iou_thresholds=iou_thresholds,
+            score_thresholds=score_thresholds,
+            index_to_datum_id=self.index_to_datum_id,
+            index_to_groundtruth_id=self.index_to_groundtruth_id,
+            index_to_prediction_id=self.index_to_prediction_id,
+            index_to_label=self.index_to_label,
+        )
+
+    def evaluate(
         self,
-        detections: list[Detection],
-        detection_ious: list[NDArray[np.float64]],
-        show_progress: bool = False,
-    ):
+        iou_thresholds: list[float] = [0.1, 0.5, 0.75],
+        score_thresholds: list[float] = [0.5],
+        filter_: Filter | None = None,
+    ) -> dict[MetricType, list[Metric]]:
         """
-        Adds detections to the cache.
+        Computes all available metrics.
 
         Parameters
         ----------
-        detections : list[Detection]
-            A list of Detection objects.
-        detection_ious : list[NDArray[np.float64]]
-            A list of arrays containing IOUs per detection.
-        show_progress : bool, default=False
-            Toggle for tqdm progress bar.
-        """
-        disable_tqdm = not show_progress
-        for detection, ious in tqdm(
-            zip(detections, detection_ious), disable=disable_tqdm
-        ):
-            # cache labels and annotation pairs
-            pairs = []
+        iou_thresholds : list[float], default=[0.1, 0.5, 0.75]
+            A list of IOU thresholds to compute metrics over.
+        score_thresholds : list[float], default=[0.5]
+            A list of score thresholds to compute metrics over.
+        filter_ : Filter, optional
+            A collection of filter parameters and masks.
+
+        Returns
+        -------
+        dict[MetricType, list[Metric]]
+            Lists of metrics organized by metric type.
+        """
+        metrics = self.compute_precision_recall(
+            iou_thresholds=iou_thresholds,
+            score_thresholds=score_thresholds,
+            filter_=filter_,
+        )
+        metrics[MetricType.ConfusionMatrix] = self.compute_confusion_matrix(
+            iou_thresholds=iou_thresholds,
+            score_thresholds=score_thresholds,
+            filter_=filter_,
+        )
+        return metrics
+
+
+class DataLoader:
+    """
+    Object Detection DataLoader
+    """
+
+    def __init__(self):
+        self._evaluator = Evaluator()
+        self.pairs: list[NDArray[np.float64]] = list()
+
+    def _add_datum(self, datum_id: str) -> int:
+        """
+        Helper function for adding a datum to the cache.
+
+        Parameters
+        ----------
+        datum_id : str
+            The datum identifier.
+
+        Returns
+        -------
+        int
+            The datum index.
+        """
+        if datum_id not in self._evaluator.datum_id_to_index:
+            if len(self._evaluator.datum_id_to_index) != len(
+                self._evaluator.index_to_datum_id
+            ):
+                raise InternalCacheException("datum cache size mismatch")
+            idx = len(self._evaluator.datum_id_to_index)
+            self._evaluator.datum_id_to_index[datum_id] = idx
+            self._evaluator.index_to_datum_id.append(datum_id)
+        return self._evaluator.datum_id_to_index[datum_id]
+
+    def _add_groundtruth(self, annotation_id: str) -> int:
+        """
+        Helper function for adding a ground truth annotation identifier to the cache.
+
+        Parameters
+        ----------
+        annotation_id : str
+            The ground truth annotation identifier.
+
+        Returns
+        -------
+        int
+            The ground truth annotation index.
+        """
+        if annotation_id not in self._evaluator.groundtruth_id_to_index:
+            if len(self._evaluator.groundtruth_id_to_index) != len(
+                self._evaluator.index_to_groundtruth_id
+            ):
+                raise InternalCacheException(
+                    "ground truth cache size mismatch"
+                )
+            idx = len(self._evaluator.groundtruth_id_to_index)
+            self._evaluator.groundtruth_id_to_index[annotation_id] = idx
+            self._evaluator.index_to_groundtruth_id.append(annotation_id)
+        return self._evaluator.groundtruth_id_to_index[annotation_id]
+
+    def _add_prediction(self, annotation_id: str) -> int:
+        """
+        Helper function for adding a prediction annotation identifier to the cache.
+
+        Parameters
+        ----------
+        annotation_id : str
+            The prediction annotation identifier.
+
+        Returns
+        -------
+        int
+            The prediction annotation index.
+        """
+        if annotation_id not in self._evaluator.prediction_id_to_index:
+            if len(self._evaluator.prediction_id_to_index) != len(
+                self._evaluator.index_to_prediction_id
+            ):
+                raise InternalCacheException("prediction cache size mismatch")
+            idx = len(self._evaluator.prediction_id_to_index)
+            self._evaluator.prediction_id_to_index[annotation_id] = idx
+            self._evaluator.index_to_prediction_id.append(annotation_id)
+        return self._evaluator.prediction_id_to_index[annotation_id]
+
+    def _add_label(self, label: str) -> int:
+        """
+        Helper function for adding a label to the cache.
+
+        Parameters
+        ----------
+        label : str
+            The label associated with the annotation.
+
+        Returns
+        -------
+        int
+            Label index.
+        """
+        label_id = len(self._evaluator.index_to_label)
+        if label not in self._evaluator.label_to_index:
+            if len(self._evaluator.label_to_index) != len(
+                self._evaluator.index_to_label
+            ):
+                raise InternalCacheException("label cache size mismatch")
+            self._evaluator.label_to_index[label] = label_id
+            self._evaluator.index_to_label.append(label)
+            label_id += 1
+        return self._evaluator.label_to_index[label]
+
+    def _add_data(
+        self,
+        detections: list[Detection],
+        detection_ious: list[NDArray[np.float64]],
+        show_progress: bool = False,
+    ):
+        """
+        Adds detections to the cache.
+
+        Parameters
+        ----------
+        detections : list[Detection]
+            A list of Detection objects.
+        detection_ious : list[NDArray[np.float64]]
+            A list of arrays containing IOUs per detection.
+        show_progress : bool, default=False
+            Toggle for tqdm progress bar.
+        """
+        disable_tqdm = not show_progress
+        for detection, ious in tqdm(
+            zip(detections, detection_ious), disable=disable_tqdm
+        ):
+            # cache labels and annotation pairs
+            pairs = []
             datum_idx = self._add_datum(detection.uid)
             if detection.groundtruths:
                 for gidx, gann in enumerate(detection.groundtruths):
@@ -352,12 +656,7 @@ class Evaluator:
 
             data = np.array(pairs)
             if data.size > 0:
-                # reset filtered cache if it exists
-                if self._temp_cache is None:
-                    raise RuntimeError(
-                        "cannot add data as evaluator has already been finalized"
-                    )
-                self._temp_cache.append(data)
+                self.pairs.append(data)
 
     def add_bounding_boxes(
         self,
@@ -459,7 +758,7 @@ class Evaluator:
             show_progress=show_progress,
         )
 
-    def finalize(self):
+    def finalize(self) -> Evaluator:
         """
         Performs data finalization and some preprocessing steps.
 
@@ -468,347 +767,37 @@ class Evaluator:
         Evaluator
             A ready-to-use evaluator object.
         """
-        n_labels = len(self.index_to_label)
-        n_datums = len(self.index_to_datum_id)
-        if self._temp_cache is None:
-            warnings.warn("evaluator is already finalized or in a bad state")
-            return self
-        elif not self._temp_cache:
-            self._detailed_pairs = np.array([], dtype=np.float64)
-            self._ranked_pairs = np.array([], dtype=np.float64)
-            self._label_metadata = np.zeros((n_labels, 2), dtype=np.int32)
-            self._metadata = Metadata()
-            warnings.warn("no valid pairs")
-            return self
-        else:
-            self._detailed_pairs = np.concatenate(self._temp_cache, axis=0)
-            self._temp_cache = None
+        if not self.pairs:
+            raise EmptyEvaluatorException()
+
+        n_labels = len(self._evaluator.index_to_label)
+        n_datums = len(self._evaluator.index_to_datum_id)
+
+        self._evaluator._detailed_pairs = np.concatenate(self.pairs, axis=0)
+        if self._evaluator._detailed_pairs.size == 0:
+            raise EmptyEvaluatorException()
 
         # order pairs by descending score, iou
         indices = np.lexsort(
             (
-                -self._detailed_pairs[:, 5],  # iou
-                -self._detailed_pairs[:, 6],  # score
+                -self._evaluator._detailed_pairs[:, 5],  # iou
+                -self._evaluator._detailed_pairs[:, 6],  # score
             )
         )
-        self._detailed_pairs = self._detailed_pairs[indices]
-        self._label_metadata = compute_label_metadata(
-            ids=self._detailed_pairs[:, :5].astype(np.int32),
+        self._evaluator._detailed_pairs = self._evaluator._detailed_pairs[
+            indices
+        ]
+        self._evaluator._label_metadata = compute_label_metadata(
+            ids=self._evaluator._detailed_pairs[:, :5].astype(np.int32),
             n_labels=n_labels,
         )
-        self._ranked_pairs = rank_pairs(
-            detailed_pairs=self._detailed_pairs,
-            label_metadata=self._label_metadata,
+        self._evaluator._ranked_pairs = rank_pairs(
+            detailed_pairs=self._evaluator._detailed_pairs,
+            label_metadata=self._evaluator._label_metadata,
         )
-        self._metadata = Metadata.create(
-            detailed_pairs=self._detailed_pairs,
+        self._evaluator._metadata = Metadata.create(
+            detailed_pairs=self._evaluator._detailed_pairs,
             number_of_datums=n_datums,
             number_of_labels=n_labels,
         )
-        return self
-
-    def create_filter(
-        self,
-        datum_ids: list[str] | None = None,
-        groundtruth_ids: list[str] | None = None,
-        prediction_ids: list[str] | None = None,
-        labels: list[str] | None = None,
-    ) -> Filter:
-        """
-        Creates a filter object.
-
-        Parameters
-        ----------
-        datum_uids : list[str], optional
-            An optional list of string uids representing datums to keep.
-        groundtruth_ids : list[str], optional
-            An optional list of string uids representing ground truth annotations to keep.
-        prediction_ids : list[str], optional
-            An optional list of string uids representing prediction annotations to keep.
-        labels : list[str], optional
-            An optional list of labels to keep.
-        """
-        mask_datums = np.ones(self._detailed_pairs.shape[0], dtype=np.bool_)
-
-        # filter datums
-        if datum_ids is not None:
-            if not datum_ids:
-                warnings.warn("creating a filter that removes all datums")
-                return Filter(
-                    mask_datums=np.zeros_like(mask_datums),
-                    mask_groundtruths=np.array([], dtype=np.bool_),
-                    mask_predictions=np.array([], dtype=np.bool_),
-                    metadata=Metadata(),
-                )
-            valid_datum_indices = np.array(
-                [self.datum_id_to_index[uid] for uid in datum_ids],
-                dtype=np.int32,
-            )
-            mask_datums = np.isin(
-                self._detailed_pairs[:, 0], valid_datum_indices
-            )
-
-        filtered_detailed_pairs = self._detailed_pairs[mask_datums]
-        n_pairs = self._detailed_pairs[mask_datums].shape[0]
-        mask_groundtruths = np.zeros(n_pairs, dtype=np.bool_)
-        mask_predictions = np.zeros_like(mask_groundtruths)
-
-        # filter by ground truth annotation ids
-        if groundtruth_ids is not None:
-            if not groundtruth_ids:
-                warnings.warn(
-                    "creating a filter that removes all ground truths"
-                )
-            valid_groundtruth_indices = np.array(
-                [self.groundtruth_id_to_index[uid] for uid in groundtruth_ids],
-                dtype=np.int32,
-            )
-            mask_groundtruths[
-                ~np.isin(
-                    filtered_detailed_pairs[:, 1],
-                    valid_groundtruth_indices,
-                )
-            ] = True
-
-        # filter by prediction annotation ids
-        if prediction_ids is not None:
-            if not prediction_ids:
-                warnings.warn("creating a filter that removes all predictions")
-            valid_prediction_indices = np.array(
-                [self.prediction_id_to_index[uid] for uid in prediction_ids],
-                dtype=np.int32,
-            )
-            mask_predictions[
-                ~np.isin(
-                    filtered_detailed_pairs[:, 2],
-                    valid_prediction_indices,
-                )
-            ] = True
-
-        # filter by labels
-        if labels is not None:
-            if not labels:
-                warnings.warn("creating a filter that removes all labels")
-                return Filter(
-                    mask_datums=mask_datums,
-                    mask_groundtruths=np.ones_like(mask_datums),
-                    mask_predictions=np.ones_like(mask_datums),
-                    metadata=Metadata(),
-                )
-            valid_label_indices = np.array(
-                [self.label_to_index[label] for label in labels] + [-1]
-            )
-            mask_groundtruths[
-                ~np.isin(filtered_detailed_pairs[:, 3], valid_label_indices)
-            ] = True
-            mask_predictions[
-                ~np.isin(filtered_detailed_pairs[:, 4], valid_label_indices)
-            ] = True
-
-        filtered_detailed_pairs, _, _ = filter_cache(
-            self._detailed_pairs,
-            mask_datums=mask_datums,
-            mask_ground_truths=mask_groundtruths,
-            mask_predictions=mask_predictions,
-            n_labels=len(self.index_to_label),
-        )
-
-        number_of_datums = (
-            len(datum_ids)
-            if datum_ids
-            else np.unique(filtered_detailed_pairs[:, 0]).size
-        )
-
-        return Filter(
-            mask_datums=mask_datums,
-            mask_groundtruths=mask_groundtruths,
-            mask_predictions=mask_predictions,
-            metadata=Metadata.create(
-                detailed_pairs=filtered_detailed_pairs,
-                number_of_datums=number_of_datums,
-                number_of_labels=len(self.index_to_label),
-            ),
-        )
-
-    def filter(
-        self, filter_: Filter
-    ) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.int32],]:
-        """
-        Performs filtering over the internal cache.
-
-        Parameters
-        ----------
-        filter_ : Filter
-            The filter parameterization.
-
-        Returns
-        -------
-        NDArray[float64]
-            Filtered detailed pairs.
-        NDArray[float64]
-            Filtered ranked pairs.
-        NDArray[int32]
-            Label metadata.
-        """
-        if not filter_.mask_datums.any():
-            warnings.warn("filter removed all datums")
-            return (
-                np.array([], dtype=np.float64),
-                np.array([], dtype=np.float64),
-                np.zeros((self.metadata.number_of_labels, 2), dtype=np.int32),
-            )
-        if filter_.mask_groundtruths.all():
-            warnings.warn("filter removed all ground truths")
-        if filter_.mask_predictions.all():
-            warnings.warn("filter removed all predictions")
-        return filter_cache(
-            detailed_pairs=self._detailed_pairs,
-            mask_datums=filter_.mask_datums,
-            mask_ground_truths=filter_.mask_groundtruths,
-            mask_predictions=filter_.mask_predictions,
-            n_labels=len(self.index_to_label),
-        )
-
-    def compute_precision_recall(
-        self,
-        iou_thresholds: list[float],
-        score_thresholds: list[float],
-        filter_: Filter | None = None,
-    ) -> dict[MetricType, list[Metric]]:
-        """
-        Computes all metrics except for ConfusionMatrix
-
-        Parameters
-        ----------
-        iou_thresholds : list[float]
-            A list of IOU thresholds to compute metrics over.
-        score_thresholds : list[float]
-            A list of score thresholds to compute metrics over.
-        filter_ : Filter, optional
-            A collection of filter parameters and masks.
-
-        Returns
-        -------
-        dict[MetricType, list]
-            A dictionary mapping MetricType enumerations to lists of computed metrics.
-        """
-        if not iou_thresholds:
-            raise ValueError("At least one IOU threshold must be passed.")
-        elif not score_thresholds:
-            raise ValueError("At least one score threshold must be passed.")
-
-        if filter_ is not None:
-            _, ranked_pairs, label_metadata = self.filter(filter_=filter_)
-        else:
-            ranked_pairs = self._ranked_pairs
-            label_metadata = self._label_metadata
-
-        results = compute_precion_recall(
-            ranked_pairs=ranked_pairs,
-            label_metadata=label_metadata,
-            iou_thresholds=np.array(iou_thresholds),
-            score_thresholds=np.array(score_thresholds),
-        )
-        return unpack_precision_recall_into_metric_lists(
-            results=results,
-            label_metadata=label_metadata,
-            iou_thresholds=iou_thresholds,
-            score_thresholds=score_thresholds,
-            index_to_label=self.index_to_label,
-        )
-
-    def compute_confusion_matrix(
-        self,
-        iou_thresholds: list[float],
-        score_thresholds: list[float],
-        filter_: Filter | None = None,
-    ) -> list[Metric]:
-        """
-        Computes confusion matrices at various thresholds.
-
-        Parameters
-        ----------
-        iou_thresholds : list[float]
-            A list of IOU thresholds to compute metrics over.
-        score_thresholds : list[float]
-            A list of score thresholds to compute metrics over.
-        filter_ : Filter, optional
-            A collection of filter parameters and masks.
-
-        Returns
-        -------
-        list[Metric]
-            List of confusion matrices per threshold pair.
-        """
-        if not iou_thresholds:
-            raise ValueError("At least one IOU threshold must be passed.")
-        elif not score_thresholds:
-            raise ValueError("At least one score threshold must be passed.")
-
-        if filter_ is not None:
-            detailed_pairs, _, _ = self.filter(filter_=filter_)
-        else:
-            detailed_pairs = self._detailed_pairs
-
-        if detailed_pairs.size == 0:
-            warnings.warn("attempted to compute over an empty set")
-            return []
-
-        results = compute_confusion_matrix(
-            detailed_pairs=detailed_pairs,
-            iou_thresholds=np.array(iou_thresholds),
-            score_thresholds=np.array(score_thresholds),
-        )
-        return unpack_confusion_matrix_into_metric_list(
-            results=results,
-            detailed_pairs=detailed_pairs,
-            iou_thresholds=iou_thresholds,
-            score_thresholds=score_thresholds,
-            index_to_datum_id=self.index_to_datum_id,
-            index_to_groundtruth_id=self.index_to_groundtruth_id,
-            index_to_prediction_id=self.index_to_prediction_id,
-            index_to_label=self.index_to_label,
-        )
-
-    def evaluate(
-        self,
-        iou_thresholds: list[float] = [0.1, 0.5, 0.75],
-        score_thresholds: list[float] = [0.5],
-        filter_: Filter | None = None,
-    ) -> dict[MetricType, list[Metric]]:
-        """
-        Computes all available metrics.
-
-        Parameters
-        ----------
-        iou_thresholds : list[float], default=[0.1, 0.5, 0.75]
-            A list of IOU thresholds to compute metrics over.
-        score_thresholds : list[float], default=[0.5]
-            A list of score thresholds to compute metrics over.
-        filter_ : Filter, optional
-            A collection of filter parameters and masks.
-
-        Returns
-        -------
-        dict[MetricType, list[Metric]]
-            Lists of metrics organized by metric type.
-        """
-        metrics = self.compute_precision_recall(
-            iou_thresholds=iou_thresholds,
-            score_thresholds=score_thresholds,
-            filter_=filter_,
-        )
-        metrics[MetricType.ConfusionMatrix] = self.compute_confusion_matrix(
-            iou_thresholds=iou_thresholds,
-            score_thresholds=score_thresholds,
-            filter_=filter_,
-        )
-        return metrics
-
-
-class DataLoader(Evaluator):
-    """
-    Used for backwards compatibility as the Evaluator now handles ingestion.
-    """
-
-    pass
+        return self._evaluator

{valor_lite-0.36.0 → valor_lite-0.36.2}/valor_lite/semantic_segmentation/__init__.py

@@ -1,5 +1,5 @@
 from .annotation import Bitmask, Segmentation
-from .manager import DataLoader, Evaluator, Filter
+from .manager import DataLoader, Evaluator, Filter, Metadata
 from .metric import Metric, MetricType
 
 __all__ = [
@@ -10,4 +10,5 @@ __all__ = [
     "Metric",
     "MetricType",
     "Filter",
+    "Metadata",
 ]

{valor_lite-0.36.0 → valor_lite-0.36.2}/valor_lite/semantic_segmentation/manager.py

@@ -1,10 +1,10 @@
-import warnings
 from dataclasses import asdict, dataclass
 
 import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
 
+from valor_lite.exceptions import EmptyEvaluatorException, EmptyFilterException
 from valor_lite.semantic_segmentation.annotation import Segmentation
 from valor_lite.semantic_segmentation.computation import (
     compute_intermediate_confusion_matrices,
@@ -71,6 +71,15 @@ class Filter:
     label_mask: NDArray[np.bool_]
     metadata: Metadata
 
+    def __post_init__(self):
+        # validate datum mask
+        if not self.datum_mask.any():
+            raise EmptyFilterException("filter removes all datums")
+
+        # validate label mask
+        if self.label_mask.all():
+            raise EmptyFilterException("filter removes all labels")
+
 
 class Evaluator:
     """
@@ -140,10 +149,9 @@ class Evaluator:
         label_mask = np.zeros(
             self.metadata.number_of_labels + 1, dtype=np.bool_
         )
+
         if datum_ids is not None:
             if not datum_ids:
-                filtered_confusion_matrices = np.array([], dtype=np.int64)
-                warnings.warn("datum filter results in empty data array")
                 return Filter(
                     datum_mask=np.zeros_like(datum_mask),
                     label_mask=label_mask,
@@ -159,10 +167,9 @@ class Evaluator:
                 == datum_id_array.reshape(1, -1)
             ).any(axis=1)
             datum_mask[~mask_valid_datums] = False
+
         if labels is not None:
             if not labels:
-                filtered_confusion_matrices = np.array([], dtype=np.int64)
-                warnings.warn("label filter results in empty data array")
                 return Filter(
                     datum_mask=datum_mask,
                     label_mask=np.ones_like(label_mask),
@@ -211,18 +218,6 @@ class Evaluator:
         NDArray[int64]
             Filtered label metadata
         """
-        empty_datum_mask = not filter_.datum_mask.any()
-        empty_label_mask = filter_.label_mask.all()
-        if empty_datum_mask or empty_label_mask:
-            if empty_datum_mask:
-                warnings.warn("filter does not allow any datum")
-            if empty_label_mask:
-                warnings.warn("filter removes all labels")
-            return (
-                np.array([], dtype=np.int64),
-                np.zeros((self.metadata.number_of_labels, 2), dtype=np.int64),
-            )
-
         return filter_cache(
             confusion_matrices=self._confusion_matrices.copy(),
             datum_mask=filter_.datum_mask,
@@ -408,7 +403,7 @@ class DataLoader:
         """
 
         if len(self.matrices) == 0:
-            raise ValueError("No data available to create evaluator.")
+            raise EmptyEvaluatorException()
 
         n_labels = len(self._evaluator.index_to_label)
         n_datums = len(self._evaluator.index_to_datum_id)

{valor_lite-0.36.0 → valor_lite-0.36.2}/valor_lite.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: valor-lite
-Version: 0.36.0
+Version: 0.36.2
 Summary: Evaluate machine learning models.
 Project-URL: homepage, https://www.striveworks.com
 Requires-Python: >=3.10

{valor_lite-0.36.0 → valor_lite-0.36.2}/valor_lite.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ README.md
 pyproject.toml
 valor_lite/LICENSE
 valor_lite/__init__.py
+valor_lite/exceptions.py
 valor_lite/profiling.py
 valor_lite/schemas.py
 valor_lite.egg-info/PKG-INFO