valor-lite 0.33.1__py3-none-any.whl → 0.33.3__py3-none-any.whl

valor_lite/detection/annotation.py

@@ -36,8 +36,20 @@ class Bitmask:
                 "If scores are defined, there must be a 1:1 pairing with labels."
             )
 
-    def to_box(self) -> BoundingBox:
-        raise NotImplementedError
+    def to_box(self) -> BoundingBox | None:
+
+        if not self.mask.any():
+            return None
+
+        rows, cols = np.nonzero(self.mask)
+        return BoundingBox(
+            xmin=cols.min(),
+            xmax=cols.max(),
+            ymin=rows.min(),
+            ymax=rows.max(),
+            labels=self.labels,
+            scores=self.scores,
+        )
 
 
 @dataclass

valor_lite/detection/computation.py

@@ -1,16 +1,38 @@
 import numpy as np
 from numpy.typing import NDArray
 
-# datum id  0
-# gt        1
-# pd        2
-# iou       3
-# gt label  4
-# pd label  5
-# score     6
-
 
 def compute_iou(data: NDArray[np.floating]) -> NDArray[np.floating]:
+    """
+    Computes intersection-over-union (IoU) for axis-aligned bounding boxes.
+
+    Takes data with shape (N, 8):
+
+    Index 0 - xmin for Box 1
+    Index 1 - xmax for Box 1
+    Index 2 - ymin for Box 1
+    Index 3 - ymax for Box 1
+    Index 4 - xmin for Box 2
+    Index 5 - xmax for Box 2
+    Index 6 - ymin for Box 2
+    Index 7 - ymax for Box 2
+
+    Returns data with shape (N, 1):
+
+    Index 0 - IoU
+
+    Parameters
+    ----------
+    data : NDArray[np.floating]
+        A sorted array of classification pairs.
+    label_metadata : NDArray[np.int32]
+        An array containing metadata related to labels.
+
+    Returns
+    -------
+    NDArray[np.floating]
+        Compute IoU's.
+    """
 
     xmin1, xmax1, ymin1, ymax1 = (
         data[:, 0],
@@ -48,9 +70,9 @@ def compute_iou(data: NDArray[np.floating]) -> NDArray[np.floating]:
 
 
 def _compute_ranked_pairs_for_datum(
-    data: np.ndarray,
-    label_metadata: np.ndarray,
-) -> np.ndarray:
+    data: NDArray[np.floating],
+    label_metadata: NDArray[np.int32],
+) -> NDArray[np.floating]:
     """
     Computes ranked pairs for a datum.
     """
@@ -91,25 +113,51 @@ def _compute_ranked_pairs_for_datum(
 
 def compute_ranked_pairs(
     data: list[NDArray[np.floating]],
-    label_metadata: NDArray[np.integer],
+    label_metadata: NDArray[np.int32],
 ) -> NDArray[np.floating]:
-    pairs = np.concatenate(
-        [
-            _compute_ranked_pairs_for_datum(
-                datum,
-                label_metadata=label_metadata,
-            )
-            for datum in data
-        ],
-        axis=0,
-    )
+    """
+    Performs pair ranking on input data.
+
+    Takes data with shape (N, 7):
+
+    Index 0 - Datum Index
+    Index 1 - GroundTruth Index
+    Index 2 - Prediction Index
+    Index 3 - IoU
+    Index 4 - GroundTruth Label Index
+    Index 5 - Prediction Label Index
+    Index 6 - Score
+
+    Returns data with shape (N - M, 7)
+
+    Parameters
+    ----------
+    data : NDArray[np.floating]
+        A sorted array of classification pairs.
+    label_metadata : NDArray[np.int32]
+        An array containing metadata related to labels.
+
+    Returns
+    -------
+    NDArray[np.floating]
+        A filtered array containing only ranked pairs.
+    """
+
+    ranked_pairs_by_datum = [
+        _compute_ranked_pairs_for_datum(
+            data=datum,
+            label_metadata=label_metadata,
+        )
+        for datum in data
+    ]
+    ranked_pairs = np.concatenate(ranked_pairs_by_datum, axis=0)
     indices = np.lexsort(
         (
-            -pairs[:, 3],  # iou
-            -pairs[:, 6],  # score
+            -ranked_pairs[:, 3],  # iou
+            -ranked_pairs[:, 6],  # score
         )
     )
-    return pairs[indices]
+    return ranked_pairs[indices]
 
 
 def compute_metrics(
@@ -136,6 +184,27 @@ def compute_metrics(
     """
     Computes Object Detection metrics.
 
+    Takes data with shape (N, 7):
+
+    Index 0 - Datum Index
+    Index 1 - GroundTruth Index
+    Index 2 - Prediction Index
+    Index 3 - IoU
+    Index 4 - GroundTruth Label Index
+    Index 5 - Prediction Label Index
+    Index 6 - Score
+
+    Parameters
+    ----------
+    data : NDArray[np.floating]
+        A sorted array of classification pairs.
+    label_metadata : NDArray[np.int32]
+        An array containing metadata related to labels.
+    iou_thresholds : NDArray[np.floating]
+        A 1-D array containing IoU thresholds.
+    score_thresholds : NDArray[np.floating]
+        A 1-D array containing score thresholds.
+
     Returns
     -------
     tuple[NDArray, NDArray, NDArray NDArray]
@@ -155,7 +224,7 @@ def compute_metrics(
 
     average_precision = np.zeros((n_ious, n_labels))
     average_recall = np.zeros((n_scores, n_labels))
-    precision_recall = np.zeros((n_ious, n_scores, n_labels, 7))
+    counts = np.zeros((n_ious, n_scores, n_labels, 7))
 
     pd_labels = data[:, 5].astype(int)
     unique_pd_labels = np.unique(pd_labels)
@@ -245,7 +314,7 @@ def compute_metrics(
                 out=accuracy,
             )
 
-            precision_recall[iou_idx][score_idx] = np.concatenate(
+            counts[iou_idx][score_idx] = np.concatenate(
                 (
                     tp_count[:, np.newaxis],
                     fp_count[:, np.newaxis],
@@ -353,42 +422,77 @@ def compute_metrics(
     return (
         ap_results,
         ar_results,
-        precision_recall,
+        counts,
         pr_curve,
     )
 
 
 def compute_detailed_counts(
-    data: np.ndarray,
-    label_metadata: np.ndarray,
-    iou_thresholds: np.ndarray,
-    score_thresholds: np.ndarray,
+    data: NDArray[np.floating],
+    label_metadata: NDArray[np.int32],
+    iou_thresholds: NDArray[np.floating],
+    score_thresholds: NDArray[np.floating],
     n_samples: int,
-) -> np.ndarray:
-
+) -> NDArray[np.int32]:
     """
-    0  label
-    1  tp
-    ...
-    2  fp - 1
-    3  fp - 2
-    4  fn - misclassification
-    5  fn - hallucination
-    6  tn
+    Compute detailed counts.
+
+    Takes data with shape (N, 7):
+
+    Index 0 - Datum Index
+    Index 1 - GroundTruth Index
+    Index 2 - Prediction Index
+    Index 3 - IoU
+    Index 4 - GroundTruth Label Index
+    Index 5 - Prediction Label Index
+    Index 6 - Score
+
+    Outputs an array with shape (N_IoUs, N_Score, N_Labels, 5 * n_samples + 5):
+
+    Index 0 - True Positive Count
+    ... Datum ID Examples
+    Index 2 * n_samples + 1 - False Positive Misclassification Count
+    ... Datum ID Examples
+    Index 4 * n_samples + 2 - False Positive Hallucination Count
+    ... Datum ID Examples
+    Index 6 * n_samples + 3 - False Negative Misclassification Count
+    ... Datum ID Examples
+    Index 8 * n_samples + 4 - False Negative Missing Prediction Count
+    ... Datum ID Examples
+
+    Parameters
+    ----------
+    data : NDArray[np.floating]
+        A sorted array of classification pairs.
+    label_metadata : NDArray[np.int32]
+        An array containing metadata related to labels.
+    iou_thresholds : NDArray[np.floating]
+        A 1-D array containing IoU thresholds.
+    score_thresholds : NDArray[np.floating]
+        A 1-D array containing score thresholds.
+    n_samples : int
+        The number of examples to return per count.
+
+    Returns
+    -------
+    NDArray[np.int32]
+        The detailed counts with optional examples.
     """
 
     n_labels = label_metadata.shape[0]
     n_ious = iou_thresholds.shape[0]
     n_scores = score_thresholds.shape[0]
-    n_metrics = 5 * (n_samples + 1)
+    n_metrics = 5 * (2 * n_samples + 1)
 
     tp_idx = 0
-    fp_misclf_idx = tp_idx + n_samples + 1
-    fp_halluc_idx = fp_misclf_idx + n_samples + 1
-    fn_misclf_idx = fp_halluc_idx + n_samples + 1
-    fn_misprd_idx = fn_misclf_idx + n_samples + 1
+    fp_misclf_idx = 2 * n_samples + 1
+    fp_halluc_idx = 4 * n_samples + 2
+    fn_misclf_idx = 6 * n_samples + 3
+    fn_misprd_idx = 8 * n_samples + 4
 
-    detailed_pr_curve = np.ones((n_ious, n_scores, n_labels, n_metrics)) * -1.0
+    detailed_pr_curve = -1 * np.ones(
+        (n_ious, n_scores, n_labels, n_metrics), dtype=np.int32
+    )
 
     mask_gt_exists = data[:, 1] > -0.5
     mask_pd_exists = data[:, 2] > -0.5
@@ -406,13 +510,14 @@ def compute_detailed_counts(
         mask_iou_threshold = data[:, 3] >= iou_thresholds[iou_idx]
         mask_iou = mask_iou_nonzero & mask_iou_threshold
 
-        groundtruths_with_pairs = np.unique(groundtruths[mask_iou], axis=0)
-        mask_groundtruths_with_passing_ious = (
-            groundtruths.reshape(-1, 1, 2)
-            == groundtruths_with_pairs.reshape(1, -1, 2)
-        ).all(axis=2)
+        groundtruths_passing_ious = np.unique(groundtruths[mask_iou], axis=0)
         mask_groundtruths_with_passing_ious = (
-            mask_groundtruths_with_passing_ious.any(axis=1)
+            (
+                groundtruths.reshape(-1, 1, 2)
+                == groundtruths_passing_ious.reshape(1, -1, 2)
+            )
+            .all(axis=2)
+            .any(axis=1)
         )
         mask_groundtruths_without_passing_ious = (
             ~mask_groundtruths_with_passing_ious & mask_gt_exists
@@ -422,11 +527,12 @@ def compute_detailed_counts(
             predictions[mask_iou], axis=0
         )
         mask_predictions_with_passing_ious = (
-            predictions.reshape(-1, 1, 2)
-            == predictions_with_passing_ious.reshape(1, -1, 2)
-        ).all(axis=2)
-        mask_predictions_with_passing_ious = (
-            mask_predictions_with_passing_ious.any(axis=1)
+            (
+                predictions.reshape(-1, 1, 2)
+                == predictions_with_passing_ious.reshape(1, -1, 2)
+            )
+            .all(axis=2)
+            .any(axis=1)
         )
         mask_predictions_without_passing_ious = (
             ~mask_predictions_with_passing_ious & mask_pd_exists
@@ -440,11 +546,12 @@ def compute_detailed_counts(
                 groundtruths[mask_iou & mask_score], axis=0
             )
             mask_groundtruths_with_passing_score = (
-                groundtruths.reshape(-1, 1, 2)
-                == groundtruths_with_passing_score.reshape(1, -1, 2)
-            ).all(axis=2)
-            mask_groundtruths_with_passing_score = (
-                mask_groundtruths_with_passing_score.any(axis=1)
+                (
+                    groundtruths.reshape(-1, 1, 2)
+                    == groundtruths_with_passing_score.reshape(1, -1, 2)
+                )
+                .all(axis=2)
+                .any(axis=1)
             )
             mask_groundtruths_without_passing_score = (
                 ~mask_groundtruths_with_passing_score & mask_gt_exists
@@ -466,12 +573,30 @@ def compute_detailed_counts(
                 | mask_groundtruths_without_passing_score
             )
 
-            tp = np.unique(data[mask_tp][:, [0, 2, 5]], axis=0)
+            tp_pds = np.unique(data[mask_tp][:, [0, 2, 5]], axis=0)
+            tp_gts = np.unique(data[mask_tp][:, [0, 1, 4]], axis=0)
             fp_misclf = np.unique(data[mask_fp_misclf][:, [0, 2, 5]], axis=0)
             fp_halluc = np.unique(data[mask_fp_halluc][:, [0, 2, 5]], axis=0)
             fn_misclf = np.unique(data[mask_fn_misclf][:, [0, 1, 4]], axis=0)
             fn_misprd = np.unique(data[mask_fn_misprd][:, [0, 1, 4]], axis=0)
 
+            mask_fp_misclf_is_tp = (
+                (fp_misclf.reshape(-1, 1, 3) == tp_pds.reshape(1, -1, 3))
+                .all(axis=2)
+                .any(axis=1)
+            )
+            mask_fn_misclf_is_tp = (
+                (fn_misclf.reshape(-1, 1, 3) == tp_gts.reshape(1, -1, 3))
+                .all(axis=2)
+                .any(axis=1)
+            )
+
+            tp = tp_pds
+            fp_misclf = fp_misclf[~mask_fp_misclf_is_tp]
+            fp_halluc = fp_halluc
+            fn_misclf = fn_misclf[~mask_fn_misclf_is_tp]
+            fn_misprd = fn_misprd
+
             tp_count = np.bincount(tp[:, 2].astype(int), minlength=n_labels)
             fp_misclf_count = np.bincount(
                 fp_misclf[:, 2].astype(int), minlength=n_labels
@@ -502,21 +627,41 @@ def compute_detailed_counts(
 
             if n_samples > 0:
                 for label_idx in range(n_labels):
-                    tp_examples = tp[tp[:, 2].astype(int) == label_idx][
-                        :n_samples, 0
-                    ]
-                    fp_misclf_examples = fp_misclf[
-                        fp_misclf[:, 2].astype(int) == label_idx
-                    ][:n_samples, 0]
-                    fp_halluc_examples = fp_halluc[
-                        fp_halluc[:, 2].astype(int) == label_idx
-                    ][:n_samples, 0]
-                    fn_misclf_examples = fn_misclf[
-                        fn_misclf[:, 2].astype(int) == label_idx
-                    ][:n_samples, 0]
-                    fn_misprd_examples = fn_misprd[
-                        fn_misprd[:, 2].astype(int) == label_idx
-                    ][:n_samples, 0]
+                    tp_examples = (
+                        tp[tp[:, 2].astype(int) == label_idx][
+                            :n_samples, [0, 1]
+                        ]
+                        .astype(int)
+                        .flatten()
+                    )
+                    fp_misclf_examples = (
+                        fp_misclf[fp_misclf[:, 2].astype(int) == label_idx][
+                            :n_samples, [0, 1]
+                        ]
+                        .astype(int)
+                        .flatten()
+                    )
+                    fp_halluc_examples = (
+                        fp_halluc[fp_halluc[:, 2].astype(int) == label_idx][
+                            :n_samples, [0, 1]
+                        ]
+                        .astype(int)
+                        .flatten()
+                    )
+                    fn_misclf_examples = (
+                        fn_misclf[fn_misclf[:, 2].astype(int) == label_idx][
+                            :n_samples, [0, 1]
+                        ]
+                        .astype(int)
+                        .flatten()
+                    )
+                    fn_misprd_examples = (
+                        fn_misprd[fn_misprd[:, 2].astype(int) == label_idx][
+                            :n_samples, [0, 1]
+                        ]
+                        .astype(int)
+                        .flatten()
+                    )
 
                     detailed_pr_curve[
                         iou_idx,

valor_lite/detection/manager.py

@@ -58,6 +58,10 @@ class Filter:
 
 
 class Evaluator:
+    """
+    Object Detection Evaluator
+    """
+
     def __init__(self):
 
         # metadata
@@ -70,6 +74,10 @@ class Evaluator:
         self.uid_to_index: dict[str, int] = dict()
         self.index_to_uid: dict[int, str] = dict()
 
+        # annotation reference
+        self.groundtruth_examples: dict[int, NDArray[np.float16]] = dict()
+        self.prediction_examples: dict[int, NDArray[np.float16]] = dict()
+
         # label reference
         self.label_to_index: dict[tuple[str, str], int] = dict()
         self.index_to_label: dict[int, tuple[str, str]] = dict()
@@ -80,13 +88,16 @@ class Evaluator:
         self.label_index_to_label_key_index: dict[int, int] = dict()
 
         # computation caches
-        self._detailed_pairs = np.array([])
-        self._ranked_pairs = np.array([])
-        self._label_metadata = np.array([])
-        self._label_metadata_per_datum = np.array([])
+        self._detailed_pairs: NDArray[np.floating] = np.array([])
+        self._ranked_pairs: NDArray[np.floating] = np.array([])
+        self._label_metadata: NDArray[np.int32] = np.array([])
+        self._label_metadata_per_datum: NDArray[np.int32] = np.array([])
 
     @property
     def ignored_prediction_labels(self) -> list[tuple[str, str]]:
+        """
+        Prediction labels that are not present in the ground truth set.
+        """
         glabels = set(np.where(self._label_metadata[:, 0] > 0)[0])
         plabels = set(np.where(self._label_metadata[:, 1] > 0)[0])
         return [
@@ -95,6 +106,9 @@ class Evaluator:
 
     @property
     def missing_prediction_labels(self) -> list[tuple[str, str]]:
+        """
+        Ground truth labels that are not present in the prediction set.
+        """
         glabels = set(np.where(self._label_metadata[:, 0] > 0)[0])
         plabels = set(np.where(self._label_metadata[:, 1] > 0)[0])
         return [
@@ -103,6 +117,9 @@ class Evaluator:
 
     @property
     def metadata(self) -> dict:
+        """
+        Evaluation metadata.
+        """
         return {
             "n_datums": self.n_datums,
             "n_groundtruths": self.n_groundtruths,
@@ -211,21 +228,32 @@ class Evaluator:
 
     def evaluate(
         self,
+        metrics_to_return: list[MetricType] = MetricType.base_metrics(),
         iou_thresholds: list[float] = [0.5, 0.75, 0.9],
         score_thresholds: list[float] = [0.5],
+        number_of_examples: int = 0,
         filter_: Filter | None = None,
     ) -> dict[MetricType, list]:
         """
-        Runs evaluation over cached data.
+        Performs an evaluation and returns metrics.
 
         Parameters
         ----------
+        metrics_to_return : list[MetricType]
+            A list of metrics to return in the results.
         iou_thresholds : list[float]
-            A list of iou thresholds to compute over.
+            A list of IoU thresholds to compute metrics over.
         score_thresholds : list[float]
-            A list of score thresholds to compute over.
-        filter_mask : NDArray[bool], optional
-            A boolean mask that filters the cached data.
+            A list of score thresholds to compute metrics over.
+        number_of_examples : int, default=0
+            Number of annotation examples to return in DetailedCounts.
+        filter_ : Filter, optional
+            An optional filter object.
+
+        Returns
+        -------
+        dict[MetricType, list]
+            A dictionary mapping MetricType enumerations to lists of computed metrics.
         """
 
         data = self._ranked_pairs
@@ -360,6 +388,10 @@ class Evaluator:
         for label_idx, label in self.index_to_label.items():
             for score_idx, score_threshold in enumerate(score_thresholds):
                 for iou_idx, iou_threshold in enumerate(iou_thresholds):
+
+                    if label_metadata[label_idx, 0] == 0:
+                        continue
+
                     row = precision_recall[iou_idx][score_idx][label_idx]
                     kwargs = {
                         "label": label,
@@ -374,6 +406,7 @@ class Evaluator:
                             **kwargs,
                         )
                     )
+
                     metrics[MetricType.Precision].append(
                         Precision(
                             value=row[3],
@@ -399,16 +432,27 @@ class Evaluator:
                         )
                     )
 
+        if MetricType.DetailedCounts in metrics_to_return:
+            metrics[MetricType.DetailedCounts] = self._compute_detailed_counts(
+                iou_thresholds=iou_thresholds,
+                score_thresholds=score_thresholds,
+                n_samples=number_of_examples,
+            )
+
+        for metric in set(metrics.keys()):
+            if metric not in metrics_to_return:
+                del metrics[metric]
+
         return metrics
 
-    def compute_detailed_counts(
+    def _compute_detailed_counts(
         self,
         iou_thresholds: list[float] = [0.5],
         score_thresholds: list[float] = [
             score / 10.0 for score in range(1, 11)
         ],
         n_samples: int = 0,
-    ) -> list[list[DetailedCounts]]:
+    ) -> list[DetailedCounts]:
         """
         Computes detailed counting metrics.
 
@@ -439,106 +483,139 @@ class Evaluator:
         )
 
         tp_idx = 0
-        fp_misclf_idx = tp_idx + n_samples + 1
-        fp_halluc_idx = fp_misclf_idx + n_samples + 1
-        fn_misclf_idx = fp_halluc_idx + n_samples + 1
-        fn_misprd_idx = fn_misclf_idx + n_samples + 1
+        fp_misclf_idx = 2 * n_samples + 1
+        fp_halluc_idx = 4 * n_samples + 2
+        fn_misclf_idx = 6 * n_samples + 3
+        fn_misprd_idx = 8 * n_samples + 4
+
+        def _unpack_examples(
+            iou_idx: int,
+            label_idx: int,
+            type_idx: int,
+            example_source: dict[int, NDArray[np.float16]],
+        ) -> list[list[tuple[str, tuple[float, float, float, float]]]]:
+            """
+            Unpacks metric examples from computation.
+            """
+            type_idx += 1
+
+            results = list()
+            for score_idx in range(n_scores):
+                examples = list()
+                for example_idx in range(n_samples):
+                    datum_idx = metrics[
+                        iou_idx,
+                        score_idx,
+                        label_idx,
+                        type_idx + example_idx * 2,
+                    ]
+                    annotation_idx = metrics[
+                        iou_idx,
+                        score_idx,
+                        label_idx,
+                        type_idx + example_idx * 2 + 1,
+                    ]
+                    if datum_idx >= 0:
+                        examples.append(
+                            (
+                                self.index_to_uid[datum_idx],
+                                tuple(
+                                    example_source[datum_idx][
+                                        annotation_idx
+                                    ].tolist()
+                                ),
+                            )
+                        )
+                results.append(examples)
+
+            return results
 
         n_ious, n_scores, n_labels, _ = metrics.shape
         return [
-            [
-                DetailedCounts(
-                    iou_threshold=iou_thresholds[iou_idx],
-                    label=self.index_to_label[label_idx],
-                    score_thresholds=score_thresholds,
-                    tp=metrics[iou_idx, :, label_idx, tp_idx]
-                    .astype(int)
-                    .tolist(),
-                    tp_examples=[
-                        [
-                            self.index_to_uid[int(datum_idx)]
-                            for datum_idx in metrics[iou_idx][score_idx][
-                                label_idx
-                            ][tp_idx + 1 : fp_misclf_idx]
-                            if int(datum_idx) >= 0
-                        ]
-                        for score_idx in range(n_scores)
-                    ],
-                    fp_misclassification=metrics[
-                        iou_idx, :, label_idx, fp_misclf_idx
-                    ]
-                    .astype(int)
-                    .tolist(),
-                    fp_misclassification_examples=[
-                        [
-                            self.index_to_uid[int(datum_idx)]
-                            for datum_idx in metrics[iou_idx][score_idx][
-                                label_idx
-                            ][fp_misclf_idx + 1 : fp_halluc_idx]
-                            if int(datum_idx) >= 0
-                        ]
-                        for score_idx in range(n_scores)
-                    ],
-                    fp_hallucination=metrics[
-                        iou_idx, :, label_idx, fp_halluc_idx
-                    ]
-                    .astype(int)
-                    .tolist(),
-                    fp_hallucination_examples=[
-                        [
-                            self.index_to_uid[int(datum_idx)]
-                            for datum_idx in metrics[iou_idx][score_idx][
-                                label_idx
-                            ][fp_halluc_idx + 1 : fn_misclf_idx]
-                            if int(datum_idx) >= 0
-                        ]
-                        for score_idx in range(n_scores)
-                    ],
-                    fn_misclassification=metrics[
-                        iou_idx, :, label_idx, fn_misclf_idx
-                    ]
-                    .astype(int)
-                    .tolist(),
-                    fn_misclassification_examples=[
-                        [
-                            self.index_to_uid[int(datum_idx)]
-                            for datum_idx in metrics[iou_idx][score_idx][
-                                label_idx
-                            ][fn_misclf_idx + 1 : fn_misprd_idx]
-                            if int(datum_idx) >= 0
-                        ]
-                        for score_idx in range(n_scores)
-                    ],
-                    fn_missing_prediction=metrics[
-                        iou_idx, :, label_idx, fn_misprd_idx
-                    ]
-                    .astype(int)
-                    .tolist(),
-                    fn_missing_prediction_examples=[
-                        [
-                            self.index_to_uid[int(datum_idx)]
-                            for datum_idx in metrics[iou_idx][score_idx][
-                                label_idx
-                            ][fn_misprd_idx + 1 :]
-                            if int(datum_idx) >= 0
-                        ]
-                        for score_idx in range(n_scores)
-                    ],
-                )
-                for iou_idx in range(n_ious)
-            ]
+            DetailedCounts(
+                iou_threshold=iou_thresholds[iou_idx],
+                label=self.index_to_label[label_idx],
+                score_thresholds=score_thresholds,
+                tp=metrics[iou_idx, :, label_idx, tp_idx].astype(int).tolist(),
+                fp_misclassification=metrics[
+                    iou_idx, :, label_idx, fp_misclf_idx
+                ]
+                .astype(int)
+                .tolist(),
+                fp_hallucination=metrics[iou_idx, :, label_idx, fp_halluc_idx]
+                .astype(int)
+                .tolist(),
+                fn_misclassification=metrics[
+                    iou_idx, :, label_idx, fn_misclf_idx
+                ]
+                .astype(int)
+                .tolist(),
+                fn_missing_prediction=metrics[
+                    iou_idx, :, label_idx, fn_misprd_idx
+                ]
+                .astype(int)
+                .tolist(),
+                tp_examples=_unpack_examples(
+                    iou_idx=iou_idx,
+                    label_idx=label_idx,
+                    type_idx=tp_idx,
+                    example_source=self.prediction_examples,
+                ),
+                fp_misclassification_examples=_unpack_examples(
+                    iou_idx=iou_idx,
+                    label_idx=label_idx,
+                    type_idx=fp_misclf_idx,
+                    example_source=self.prediction_examples,
+                ),
+                fp_hallucination_examples=_unpack_examples(
+                    iou_idx=iou_idx,
+                    label_idx=label_idx,
+                    type_idx=fp_halluc_idx,
+                    example_source=self.prediction_examples,
+                ),
+                fn_misclassification_examples=_unpack_examples(
+                    iou_idx=iou_idx,
+                    label_idx=label_idx,
+                    type_idx=fn_misclf_idx,
+                    example_source=self.groundtruth_examples,
+                ),
+                fn_missing_prediction_examples=_unpack_examples(
+                    iou_idx=iou_idx,
+                    label_idx=label_idx,
+                    type_idx=fn_misprd_idx,
+                    example_source=self.groundtruth_examples,
+                ),
+            )
             for label_idx in range(n_labels)
+            for iou_idx in range(n_ious)
         ]
 
 
 class DataLoader:
+    """
+    Object Detection DataLoader
+    """
+
     def __init__(self):
         self._evaluator = Evaluator()
-        self.pairs = list()
+        self.pairs: list[NDArray[np.floating]] = list()
         self.groundtruth_count = defaultdict(lambda: defaultdict(int))
         self.prediction_count = defaultdict(lambda: defaultdict(int))
 
     def _add_datum(self, uid: str) -> int:
+        """
+        Helper function for adding a datum to the cache.
+
+        Parameters
+        ----------
+        uid : str
+            The datum uid.
+
+        Returns
+        -------
+        int
+            The datum index.
+        """
         if uid not in self._evaluator.uid_to_index:
             index = len(self._evaluator.uid_to_index)
             self._evaluator.uid_to_index[uid] = index
@@ -546,6 +623,22 @@ class DataLoader:
         return self._evaluator.uid_to_index[uid]
 
     def _add_label(self, label: tuple[str, str]) -> tuple[int, int]:
+        """
+        Helper function for adding a label to the cache.
+
+        Parameters
+        ----------
+        label : tuple[str, str]
+            The label as a tuple in format (key, value).
+
+        Returns
+        -------
+        int
+            Label index.
+        int
+            Label key index.
+        """
+
         label_id = len(self._evaluator.index_to_label)
         label_key_id = len(self._evaluator.index_to_label_key)
         if label not in self._evaluator.label_to_index:
@@ -568,11 +661,140 @@ class DataLoader:
             self._evaluator.label_key_to_index[label[0]],
         )
 
+    def _add_data(
+        self,
+        uid_index: int,
+        keyed_groundtruths: dict,
+        keyed_predictions: dict,
+    ):
+        gt_keys = set(keyed_groundtruths.keys())
+        pd_keys = set(keyed_predictions.keys())
+        joint_keys = gt_keys.intersection(pd_keys)
+        gt_unique_keys = gt_keys - pd_keys
+        pd_unique_keys = pd_keys - gt_keys
+
+        pairs = list()
+        for key in joint_keys:
+            n_predictions = len(keyed_predictions[key])
+            n_groundtruths = len(keyed_groundtruths[key])
+            boxes = np.array(
+                [
+                    np.array([*gextrema, *pextrema])
+                    for _, _, _, pextrema in keyed_predictions[key]
+                    for _, _, gextrema in keyed_groundtruths[key]
+                ]
+            )
+            ious = compute_iou(boxes)
+            mask_nonzero_iou = (ious > 1e-9).reshape(
+                (n_predictions, n_groundtruths)
+            )
+            mask_ious_halluc = ~(mask_nonzero_iou.any(axis=1))
+            mask_ious_misprd = ~(mask_nonzero_iou.any(axis=0))
+
+            pairs.extend(
+                [
+                    np.array(
+                        [
+                            float(uid_index),
+                            float(gidx),
+                            float(pidx),
+                            ious[pidx * len(keyed_groundtruths[key]) + gidx],
+                            float(glabel),
+                            float(plabel),
+                            float(score),
+                        ]
+                    )
+                    for pidx, plabel, score, _ in keyed_predictions[key]
+                    for gidx, glabel, _ in keyed_groundtruths[key]
+                    if ious[pidx * len(keyed_groundtruths[key]) + gidx] > 1e-9
+                ]
+            )
+            pairs.extend(
+                [
+                    np.array(
+                        [
+                            float(uid_index),
+                            -1.0,
+                            float(pidx),
+                            0.0,
+                            -1.0,
+                            float(plabel),
+                            float(score),
+                        ]
+                    )
+                    for pidx, plabel, score, _ in keyed_predictions[key]
+                    if mask_ious_halluc[pidx]
+                ]
+            )
+            pairs.extend(
+                [
+                    np.array(
+                        [
+                            float(uid_index),
+                            float(gidx),
+                            -1.0,
+                            0.0,
+                            float(glabel),
+                            -1.0,
+                            -1.0,
+                        ]
+                    )
+                    for gidx, glabel, _ in keyed_groundtruths[key]
+                    if mask_ious_misprd[gidx]
+                ]
+            )
+        for key in gt_unique_keys:
+            pairs.extend(
+                [
+                    np.array(
+                        [
+                            float(uid_index),
+                            float(gidx),
+                            -1.0,
+                            0.0,
+                            float(glabel),
+                            -1.0,
+                            -1.0,
+                        ]
+                    )
+                    for gidx, glabel, _ in keyed_groundtruths[key]
+                ]
+            )
+        for key in pd_unique_keys:
+            pairs.extend(
+                [
+                    np.array(
+                        [
+                            float(uid_index),
+                            -1.0,
+                            float(pidx),
+                            0.0,
+                            -1.0,
+                            float(plabel),
+                            float(score),
+                        ]
+                    )
+                    for pidx, plabel, score, _ in keyed_predictions[key]
+                ]
+            )
+
+        self.pairs.append(np.array(pairs))
+
     def add_data(
         self,
         detections: list[Detection],
         show_progress: bool = False,
     ):
+        """
+        Adds detections to the cache.
+
+        Parameters
+        ----------
+        detections : list[Detection]
+            A list of Detection objects.
+        show_progress : bool, default=False
+            Toggle for tqdm progress bar.
+        """
         disable_tqdm = not show_progress
         for detection in tqdm(detections, disable=disable_tqdm):
 
@@ -584,10 +806,21 @@ class DataLoader:
             # update datum uid index
             uid_index = self._add_datum(uid=detection.uid)
 
+            # initialize bounding box examples
+            self._evaluator.groundtruth_examples[uid_index] = np.zeros(
+                (len(detection.groundtruths), 4), dtype=np.float16
+            )
+            self._evaluator.prediction_examples[uid_index] = np.zeros(
+                (len(detection.predictions), 4), dtype=np.float16
+            )
+
             # cache labels and annotations
             keyed_groundtruths = defaultdict(list)
             keyed_predictions = defaultdict(list)
             for gidx, gann in enumerate(detection.groundtruths):
+                self._evaluator.groundtruth_examples[uid_index][
+                    gidx
+                ] = np.array(gann.extrema)
                 for glabel in gann.labels:
                     label_idx, label_key_idx = self._add_label(glabel)
                     self.groundtruth_count[label_idx][uid_index] += 1
@@ -599,6 +832,9 @@ class DataLoader:
                         )
                     )
             for pidx, pann in enumerate(detection.predictions):
+                self._evaluator.prediction_examples[uid_index][
+                    pidx
+                ] = np.array(pann.extrema)
                 for plabel, pscore in zip(pann.labels, pann.scores):
                     label_idx, label_key_idx = self._add_label(plabel)
                     self.prediction_count[label_idx][uid_index] += 1
@@ -611,83 +847,28 @@ class DataLoader:
                         )
                     )
 
-            gt_keys = set(keyed_groundtruths.keys())
-            pd_keys = set(keyed_predictions.keys())
-            joint_keys = gt_keys.intersection(pd_keys)
-            gt_unique_keys = gt_keys - pd_keys
-            pd_unique_keys = pd_keys - gt_keys
-
-            pairs = list()
-            for key in joint_keys:
-                boxes = np.array(
-                    [
-                        np.array([*gextrema, *pextrema])
-                        for _, _, _, pextrema in keyed_predictions[key]
-                        for _, _, gextrema in keyed_groundtruths[key]
-                    ]
-                )
-                ious = compute_iou(boxes)
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                float(gidx),
-                                float(pidx),
-                                ious[
-                                    pidx * len(keyed_groundtruths[key]) + gidx
-                                ],
-                                float(glabel),
-                                float(plabel),
-                                float(score),
-                            ]
-                        )
-                        for pidx, plabel, score, _ in keyed_predictions[key]
-                        for gidx, glabel, _ in keyed_groundtruths[key]
-                    ]
-                )
-            for key in gt_unique_keys:
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                float(gidx),
-                                -1.0,
-                                0.0,
-                                float(glabel),
-                                -1.0,
-                                -1.0,
-                            ]
-                        )
-                        for gidx, glabel, _ in keyed_groundtruths[key]
-                    ]
-                )
-            for key in pd_unique_keys:
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                -1.0,
-                                float(pidx),
-                                0.0,
-                                -1.0,
-                                float(plabel),
-                                float(score),
-                            ]
-                        )
-                        for pidx, plabel, score, _ in keyed_predictions[key]
-                    ]
-                )
-
-            self.pairs.append(np.array(pairs))
+            self._add_data(
+                uid_index=uid_index,
+                keyed_groundtruths=keyed_groundtruths,
+                keyed_predictions=keyed_predictions,
+            )
 
     def add_data_from_valor_dict(
         self,
         detections: list[tuple[dict, dict]],
         show_progress: bool = False,
     ):
+        """
+        Adds Valor-format detections to the cache.
+
+        Parameters
+        ----------
+        detections : list[tuple[dict, dict]]
+            A list of groundtruth, prediction pairs in Valor-format dictionaries.
+        show_progress : bool, default=False
+            Toggle for tqdm progress bar.
+        """
+
         def _get_bbox_extrema(
             data: list[list[list[float]]],
         ) -> tuple[float, float, float, float]:
@@ -706,10 +887,21 @@ class DataLoader:
             # update datum uid index
             uid_index = self._add_datum(uid=groundtruth["datum"]["uid"])
 
+            # initialize bounding box examples
+            self._evaluator.groundtruth_examples[uid_index] = np.zeros(
+                (len(groundtruth["annotations"]), 4), dtype=np.float16
+            )
+            self._evaluator.prediction_examples[uid_index] = np.zeros(
+                (len(prediction["annotations"]), 4), dtype=np.float16
+            )
+
             # cache labels and annotations
             keyed_groundtruths = defaultdict(list)
             keyed_predictions = defaultdict(list)
             for gidx, gann in enumerate(groundtruth["annotations"]):
+                self._evaluator.groundtruth_examples[uid_index][
+                    gidx
+                ] = np.array(_get_bbox_extrema(gann["bounding_box"]))
                 for valor_label in gann["labels"]:
                     glabel = (valor_label["key"], valor_label["value"])
                     label_idx, label_key_idx = self._add_label(glabel)
@@ -722,6 +914,9 @@ class DataLoader:
                         )
                     )
             for pidx, pann in enumerate(prediction["annotations"]):
+                self._evaluator.prediction_examples[uid_index][
+                    pidx
+                ] = np.array(_get_bbox_extrema(pann["bounding_box"]))
                 for valor_label in pann["labels"]:
                     plabel = (valor_label["key"], valor_label["value"])
                     pscore = valor_label["score"]
@@ -736,79 +931,21 @@ class DataLoader:
                         )
                     )
 
-            gt_keys = set(keyed_groundtruths.keys())
-            pd_keys = set(keyed_predictions.keys())
-            joint_keys = gt_keys.intersection(pd_keys)
-            gt_unique_keys = gt_keys - pd_keys
-            pd_unique_keys = pd_keys - gt_keys
-
-            pairs = list()
-            for key in joint_keys:
-                boxes = np.array(
-                    [
-                        np.array([*gextrema, *pextrema])
-                        for _, _, _, pextrema in keyed_predictions[key]
-                        for _, _, gextrema in keyed_groundtruths[key]
-                    ]
-                )
-                ious = compute_iou(boxes)
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                float(gidx),
-                                float(pidx),
-                                ious[
-                                    pidx * len(keyed_groundtruths[key]) + gidx
-                                ],
-                                float(glabel),
-                                float(plabel),
-                                float(score),
-                            ]
-                        )
-                        for pidx, plabel, score, _ in keyed_predictions[key]
-                        for gidx, glabel, _ in keyed_groundtruths[key]
-                    ]
-                )
-            for key in gt_unique_keys:
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                float(gidx),
-                                -1.0,
-                                0.0,
-                                float(glabel),
-                                -1.0,
-                                -1.0,
-                            ]
-                        )
-                        for gidx, glabel, _ in keyed_groundtruths[key]
-                    ]
-                )
-            for key in pd_unique_keys:
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                -1.0,
-                                float(pidx),
-                                0.0,
-                                -1.0,
-                                float(plabel),
-                                float(score),
-                            ]
-                        )
-                        for pidx, plabel, score, _ in keyed_predictions[key]
-                    ]
-                )
-
-            self.pairs.append(np.array(pairs))
+            self._add_data(
+                uid_index=uid_index,
+                keyed_groundtruths=keyed_groundtruths,
+                keyed_predictions=keyed_predictions,
+            )
 
     def finalize(self) -> Evaluator:
+        """
+        Performs data finalization and some preprocessing steps.
+
+        Returns
+        -------
+        Evaluator
+            A ready-to-use evaluator object.
+        """
 
         self.pairs = [pair for pair in self.pairs if pair.size > 0]
         if len(self.pairs) == 0:

valor_lite/detection/metric.py

@@ -21,6 +21,25 @@ class MetricType(str, Enum):
     PrecisionRecallCurve = "PrecisionRecallCurve"
     DetailedCounts = "DetailedCounts"
 
+    @classmethod
+    def base_metrics(cls):
+        return [
+            cls.Counts,
+            cls.Accuracy,
+            cls.Precision,
+            cls.Recall,
+            cls.F1,
+            cls.AP,
+            cls.AR,
+            cls.mAP,
+            cls.mAR,
+            cls.APAveragedOverIOUs,
+            cls.mAPAveragedOverIOUs,
+            cls.ARAveragedOverScores,
+            cls.mARAveragedOverScores,
+            cls.PrecisionRecallCurve,
+        ]
+
 
 @dataclass
 class Counts:
@@ -316,11 +335,19 @@ class DetailedCounts:
     fp_hallucination: list[int]
     fn_misclassification: list[int]
     fn_missing_prediction: list[int]
-    tp_examples: list[list[str]]
-    fp_misclassification_examples: list[list[str]]
-    fp_hallucination_examples: list[list[str]]
-    fn_misclassification_examples: list[list[str]]
-    fn_missing_prediction_examples: list[list[str]]
+    tp_examples: list[list[tuple[str, tuple[float, float, float, float]]]]
+    fp_misclassification_examples: list[
+        list[tuple[str, tuple[float, float, float, float]]]
+    ]
+    fp_hallucination_examples: list[
+        list[tuple[str, tuple[float, float, float, float]]]
+    ]
+    fn_misclassification_examples: list[
+        list[tuple[str, tuple[float, float, float, float]]]
+    ]
+    fn_missing_prediction_examples: list[
+        list[tuple[str, tuple[float, float, float, float]]]
+    ]
     score_thresholds: list[float]
     iou_threshold: float
     label: tuple[str, str]
@@ -335,13 +362,11 @@ class DetailedCounts:
                 "fp_hallucination": self.fp_hallucination,
                 "fn_misclassification": self.fn_misclassification,
                 "fn_missing_prediction": self.fn_missing_prediction,
-                "tn": None,
                 "tp_examples": self.tp_examples,
                 "fp_misclassification_examples": self.fp_misclassification_examples,
                 "fp_hallucination_examples": self.fp_hallucination_examples,
                 "fn_misclassification_examples": self.fn_misclassification_examples,
                 "fn_missing_prediction_examples": self.fn_missing_prediction_examples,
-                "tn_examples": None,
             },
             parameters={
                 "score_thresholds": self.score_thresholds,

{valor_lite-0.33.1.dist-info → valor_lite-0.33.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.1
+Version: 0.33.3
 Summary: Compute valor metrics directly in your client.
 License: MIT License
         

valor_lite-0.33.3.dist-info/RECORD

@@ -0,0 +1,12 @@
+valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
+valor_lite/detection/__init__.py,sha256=WHLHwHoKzXTBjkjC6E1_lhqB7gRWkiGWVWPqkKn-yK8,997
+valor_lite/detection/annotation.py,sha256=c45pZD1Pp2vf5GeyW_6Kl9JCx5FoaaktCaaa4q3QDUo,1758
+valor_lite/detection/computation.py,sha256=7PttK0VuOWlhRN92wpLVrGzB7RAdfdZyT3b1aTm_WaI,23214
+valor_lite/detection/manager.py,sha256=ziVnukGs-WrkyBEBBO3LVSv4LTbaWFaWqLWarVosj2c,35807
+valor_lite/detection/metric.py,sha256=DLqpODJZOG7SCqt7TCgR4am68PQORRCIQW_SXiTb1IA,9473
+valor_lite-0.33.3.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite-0.33.3.dist-info/METADATA,sha256=FBpd-wMWv-m37EK5BfFuiVmnJXg4GNzCaJrTDHv4-gE,1842
+valor_lite-0.33.3.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+valor_lite-0.33.3.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.33.3.dist-info/RECORD,,

valor_lite-0.33.1.dist-info/RECORD

@@ -1,12 +0,0 @@
-valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
-valor_lite/detection/__init__.py,sha256=WHLHwHoKzXTBjkjC6E1_lhqB7gRWkiGWVWPqkKn-yK8,997
-valor_lite/detection/annotation.py,sha256=ON9iVa33pxysUmZVTCb0wNz-eFX6MDOqDhGDz-ouymc,1466
-valor_lite/detection/computation.py,sha256=2FnVw6_dcAOvwCqpU9bIkeD7gPqDzfW48WSARnvKeOg,18873
-valor_lite/detection/manager.py,sha256=HyODoIkmj92Kfspnpojp1pUY7noAw3FuCgQ36r6vMa4,32356
-valor_lite/detection/metric.py,sha256=hHqClS7c71ztoUnfoaW3T7RmGYaVNU1SlM6vUs1P08I,8809
-valor_lite-0.33.1.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
-valor_lite-0.33.1.dist-info/METADATA,sha256=W36vWkCaas8e0H5RqfGwwlh5FritdeNO7bBj8r-lf6s,1842
-valor_lite-0.33.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-valor_lite-0.33.1.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
-valor_lite-0.33.1.dist-info/RECORD,,