valor-lite 0.33.2__py3-none-any.whl → 0.33.3__py3-none-any.whl

valor_lite/detection/annotation.py

@@ -36,8 +36,20 @@ class Bitmask:
                 "If scores are defined, there must be a 1:1 pairing with labels."
             )
 
-    def to_box(self) -> BoundingBox:
-        raise NotImplementedError
+    def to_box(self) -> BoundingBox | None:
+
+        if not self.mask.any():
+            return None
+
+        rows, cols = np.nonzero(self.mask)
+        return BoundingBox(
+            xmin=cols.min(),
+            xmax=cols.max(),
+            ymin=rows.min(),
+            ymax=rows.max(),
+            labels=self.labels,
+            scores=self.scores,
+        )
 
 
 @dataclass

valor_lite/detection/computation.py

@@ -70,9 +70,9 @@ def compute_iou(data: NDArray[np.floating]) -> NDArray[np.floating]:
 
 
 def _compute_ranked_pairs_for_datum(
-    data: np.ndarray,
-    label_metadata: np.ndarray,
-) -> np.ndarray:
+    data: NDArray[np.floating],
+    label_metadata: NDArray[np.int32],
+) -> NDArray[np.floating]:
     """
     Computes ranked pairs for a datum.
     """
@@ -113,7 +113,7 @@ def _compute_ranked_pairs_for_datum(
 
 def compute_ranked_pairs(
     data: list[NDArray[np.floating]],
-    label_metadata: NDArray[np.integer],
+    label_metadata: NDArray[np.int32],
 ) -> NDArray[np.floating]:
     """
     Performs pair ranking on input data.
@@ -142,23 +142,22 @@ def compute_ranked_pairs(
     NDArray[np.floating]
         A filtered array containing only ranked pairs.
     """
-    pairs = np.concatenate(
-        [
-            _compute_ranked_pairs_for_datum(
-                datum,
-                label_metadata=label_metadata,
-            )
-            for datum in data
-        ],
-        axis=0,
-    )
+
+    ranked_pairs_by_datum = [
+        _compute_ranked_pairs_for_datum(
+            data=datum,
+            label_metadata=label_metadata,
+        )
+        for datum in data
+    ]
+    ranked_pairs = np.concatenate(ranked_pairs_by_datum, axis=0)
     indices = np.lexsort(
         (
-            -pairs[:, 3],  # iou
-            -pairs[:, 6],  # score
+            -ranked_pairs[:, 3],  # iou
+            -ranked_pairs[:, 6],  # score
         )
     )
-    return pairs[indices]
+    return ranked_pairs[indices]
 
 
 def compute_metrics(
@@ -429,12 +428,12 @@ def compute_metrics(
 
 
 def compute_detailed_counts(
-    data: np.ndarray,
-    label_metadata: np.ndarray,
-    iou_thresholds: np.ndarray,
-    score_thresholds: np.ndarray,
+    data: NDArray[np.floating],
+    label_metadata: NDArray[np.int32],
+    iou_thresholds: NDArray[np.floating],
+    score_thresholds: NDArray[np.floating],
     n_samples: int,
-) -> np.ndarray:
+) -> NDArray[np.int32]:
     """
     Compute detailed counts.
 
@@ -452,13 +451,13 @@ def compute_detailed_counts(
 
     Index 0 - True Positive Count
     ... Datum ID Examples
-    Index n_samples + 1 - False Positive Misclassification Count
+    Index 2 * n_samples + 1 - False Positive Misclassification Count
     ... Datum ID Examples
-    Index 2 * n_samples + 2 - False Positive Hallucination Count
+    Index 4 * n_samples + 2 - False Positive Hallucination Count
     ... Datum ID Examples
-    Index 3 * n_samples + 3 - False Negative Misclassification Count
+    Index 6 * n_samples + 3 - False Negative Misclassification Count
     ... Datum ID Examples
-    Index 4 * n_samples + 4 - False Negative Missing Prediction Count
+    Index 8 * n_samples + 4 - False Negative Missing Prediction Count
     ... Datum ID Examples
 
     Parameters
@@ -476,22 +475,24 @@ def compute_detailed_counts(
 
     Returns
     -------
-    NDArray[np.floating]
+    NDArray[np.int32]
         The detailed counts with optional examples.
     """
 
     n_labels = label_metadata.shape[0]
     n_ious = iou_thresholds.shape[0]
     n_scores = score_thresholds.shape[0]
-    n_metrics = 5 * (n_samples + 1)
+    n_metrics = 5 * (2 * n_samples + 1)
 
     tp_idx = 0
-    fp_misclf_idx = tp_idx + n_samples + 1
-    fp_halluc_idx = fp_misclf_idx + n_samples + 1
-    fn_misclf_idx = fp_halluc_idx + n_samples + 1
-    fn_misprd_idx = fn_misclf_idx + n_samples + 1
+    fp_misclf_idx = 2 * n_samples + 1
+    fp_halluc_idx = 4 * n_samples + 2
+    fn_misclf_idx = 6 * n_samples + 3
+    fn_misprd_idx = 8 * n_samples + 4
 
-    detailed_pr_curve = np.ones((n_ious, n_scores, n_labels, n_metrics)) * -1.0
+    detailed_pr_curve = -1 * np.ones(
+        (n_ious, n_scores, n_labels, n_metrics), dtype=np.int32
+    )
 
     mask_gt_exists = data[:, 1] > -0.5
     mask_pd_exists = data[:, 2] > -0.5
@@ -509,13 +510,14 @@ def compute_detailed_counts(
         mask_iou_threshold = data[:, 3] >= iou_thresholds[iou_idx]
         mask_iou = mask_iou_nonzero & mask_iou_threshold
 
-        groundtruths_with_pairs = np.unique(groundtruths[mask_iou], axis=0)
-        mask_groundtruths_with_passing_ious = (
-            groundtruths.reshape(-1, 1, 2)
-            == groundtruths_with_pairs.reshape(1, -1, 2)
-        ).all(axis=2)
+        groundtruths_passing_ious = np.unique(groundtruths[mask_iou], axis=0)
         mask_groundtruths_with_passing_ious = (
-            mask_groundtruths_with_passing_ious.any(axis=1)
+            (
+                groundtruths.reshape(-1, 1, 2)
+                == groundtruths_passing_ious.reshape(1, -1, 2)
+            )
+            .all(axis=2)
+            .any(axis=1)
         )
         mask_groundtruths_without_passing_ious = (
             ~mask_groundtruths_with_passing_ious & mask_gt_exists
@@ -525,11 +527,12 @@ def compute_detailed_counts(
             predictions[mask_iou], axis=0
         )
         mask_predictions_with_passing_ious = (
-            predictions.reshape(-1, 1, 2)
-            == predictions_with_passing_ious.reshape(1, -1, 2)
-        ).all(axis=2)
-        mask_predictions_with_passing_ious = (
-            mask_predictions_with_passing_ious.any(axis=1)
+            (
+                predictions.reshape(-1, 1, 2)
+                == predictions_with_passing_ious.reshape(1, -1, 2)
+            )
+            .all(axis=2)
+            .any(axis=1)
         )
         mask_predictions_without_passing_ious = (
             ~mask_predictions_with_passing_ious & mask_pd_exists
@@ -543,11 +546,12 @@ def compute_detailed_counts(
                 groundtruths[mask_iou & mask_score], axis=0
             )
             mask_groundtruths_with_passing_score = (
-                groundtruths.reshape(-1, 1, 2)
-                == groundtruths_with_passing_score.reshape(1, -1, 2)
-            ).all(axis=2)
-            mask_groundtruths_with_passing_score = (
-                mask_groundtruths_with_passing_score.any(axis=1)
+                (
+                    groundtruths.reshape(-1, 1, 2)
+                    == groundtruths_with_passing_score.reshape(1, -1, 2)
+                )
+                .all(axis=2)
+                .any(axis=1)
             )
             mask_groundtruths_without_passing_score = (
                 ~mask_groundtruths_with_passing_score & mask_gt_exists
@@ -623,21 +627,41 @@ def compute_detailed_counts(
 
             if n_samples > 0:
                 for label_idx in range(n_labels):
-                    tp_examples = tp[tp[:, 2].astype(int) == label_idx][
-                        :n_samples, 0
-                    ]
-                    fp_misclf_examples = fp_misclf[
-                        fp_misclf[:, 2].astype(int) == label_idx
-                    ][:n_samples, 0]
-                    fp_halluc_examples = fp_halluc[
-                        fp_halluc[:, 2].astype(int) == label_idx
-                    ][:n_samples, 0]
-                    fn_misclf_examples = fn_misclf[
-                        fn_misclf[:, 2].astype(int) == label_idx
-                    ][:n_samples, 0]
-                    fn_misprd_examples = fn_misprd[
-                        fn_misprd[:, 2].astype(int) == label_idx
-                    ][:n_samples, 0]
+                    tp_examples = (
+                        tp[tp[:, 2].astype(int) == label_idx][
+                            :n_samples, [0, 1]
+                        ]
+                        .astype(int)
+                        .flatten()
+                    )
+                    fp_misclf_examples = (
+                        fp_misclf[fp_misclf[:, 2].astype(int) == label_idx][
+                            :n_samples, [0, 1]
+                        ]
+                        .astype(int)
+                        .flatten()
+                    )
+                    fp_halluc_examples = (
+                        fp_halluc[fp_halluc[:, 2].astype(int) == label_idx][
+                            :n_samples, [0, 1]
+                        ]
+                        .astype(int)
+                        .flatten()
+                    )
+                    fn_misclf_examples = (
+                        fn_misclf[fn_misclf[:, 2].astype(int) == label_idx][
+                            :n_samples, [0, 1]
+                        ]
+                        .astype(int)
+                        .flatten()
+                    )
+                    fn_misprd_examples = (
+                        fn_misprd[fn_misprd[:, 2].astype(int) == label_idx][
+                            :n_samples, [0, 1]
+                        ]
+                        .astype(int)
+                        .flatten()
+                    )
 
                     detailed_pr_curve[
                         iou_idx,

valor_lite/detection/manager.py

@@ -74,6 +74,10 @@ class Evaluator:
         self.uid_to_index: dict[str, int] = dict()
         self.index_to_uid: dict[int, str] = dict()
 
+        # annotation reference
+        self.groundtruth_examples: dict[int, NDArray[np.float16]] = dict()
+        self.prediction_examples: dict[int, NDArray[np.float16]] = dict()
+
         # label reference
         self.label_to_index: dict[tuple[str, str], int] = dict()
         self.index_to_label: dict[int, tuple[str, str]] = dict()
@@ -84,10 +88,10 @@ class Evaluator:
         self.label_index_to_label_key_index: dict[int, int] = dict()
 
         # computation caches
-        self._detailed_pairs = np.array([])
-        self._ranked_pairs = np.array([])
-        self._label_metadata = np.array([])
-        self._label_metadata_per_datum = np.array([])
+        self._detailed_pairs: NDArray[np.floating] = np.array([])
+        self._ranked_pairs: NDArray[np.floating] = np.array([])
+        self._label_metadata: NDArray[np.int32] = np.array([])
+        self._label_metadata_per_datum: NDArray[np.int32] = np.array([])
 
     @property
     def ignored_prediction_labels(self) -> list[tuple[str, str]]:
@@ -224,8 +228,10 @@ class Evaluator:
 
     def evaluate(
         self,
+        metrics_to_return: list[MetricType] = MetricType.base_metrics(),
         iou_thresholds: list[float] = [0.5, 0.75, 0.9],
         score_thresholds: list[float] = [0.5],
+        number_of_examples: int = 0,
         filter_: Filter | None = None,
     ) -> dict[MetricType, list]:
         """
@@ -233,10 +239,14 @@ class Evaluator:
 
         Parameters
         ----------
+        metrics_to_return : list[MetricType]
+            A list of metrics to return in the results.
         iou_thresholds : list[float]
             A list of IoU thresholds to compute metrics over.
         score_thresholds : list[float]
             A list of score thresholds to compute metrics over.
+        number_of_examples : int, default=0
+            Number of annotation examples to return in DetailedCounts.
         filter_ : Filter, optional
             An optional filter object.
 
@@ -422,16 +432,27 @@ class Evaluator:
                         )
                     )
 
+        if MetricType.DetailedCounts in metrics_to_return:
+            metrics[MetricType.DetailedCounts] = self._compute_detailed_counts(
+                iou_thresholds=iou_thresholds,
+                score_thresholds=score_thresholds,
+                n_samples=number_of_examples,
+            )
+
+        for metric in set(metrics.keys()):
+            if metric not in metrics_to_return:
+                del metrics[metric]
+
         return metrics
 
-    def compute_detailed_counts(
+    def _compute_detailed_counts(
         self,
         iou_thresholds: list[float] = [0.5],
         score_thresholds: list[float] = [
             score / 10.0 for score in range(1, 11)
         ],
         n_samples: int = 0,
-    ) -> list[list[DetailedCounts]]:
+    ) -> list[DetailedCounts]:
         """
         Computes detailed counting metrics.
 
@@ -462,95 +483,111 @@ class Evaluator:
         )
 
         tp_idx = 0
-        fp_misclf_idx = tp_idx + n_samples + 1
-        fp_halluc_idx = fp_misclf_idx + n_samples + 1
-        fn_misclf_idx = fp_halluc_idx + n_samples + 1
-        fn_misprd_idx = fn_misclf_idx + n_samples + 1
+        fp_misclf_idx = 2 * n_samples + 1
+        fp_halluc_idx = 4 * n_samples + 2
+        fn_misclf_idx = 6 * n_samples + 3
+        fn_misprd_idx = 8 * n_samples + 4
+
+        def _unpack_examples(
+            iou_idx: int,
+            label_idx: int,
+            type_idx: int,
+            example_source: dict[int, NDArray[np.float16]],
+        ) -> list[list[tuple[str, tuple[float, float, float, float]]]]:
+            """
+            Unpacks metric examples from computation.
+            """
+            type_idx += 1
+
+            results = list()
+            for score_idx in range(n_scores):
+                examples = list()
+                for example_idx in range(n_samples):
+                    datum_idx = metrics[
+                        iou_idx,
+                        score_idx,
+                        label_idx,
+                        type_idx + example_idx * 2,
+                    ]
+                    annotation_idx = metrics[
+                        iou_idx,
+                        score_idx,
+                        label_idx,
+                        type_idx + example_idx * 2 + 1,
+                    ]
+                    if datum_idx >= 0:
+                        examples.append(
+                            (
+                                self.index_to_uid[datum_idx],
+                                tuple(
+                                    example_source[datum_idx][
+                                        annotation_idx
+                                    ].tolist()
+                                ),
+                            )
+                        )
+                results.append(examples)
+
+            return results
 
         n_ious, n_scores, n_labels, _ = metrics.shape
         return [
-            [
-                DetailedCounts(
-                    iou_threshold=iou_thresholds[iou_idx],
-                    label=self.index_to_label[label_idx],
-                    score_thresholds=score_thresholds,
-                    tp=metrics[iou_idx, :, label_idx, tp_idx]
-                    .astype(int)
-                    .tolist(),
-                    tp_examples=[
-                        [
-                            self.index_to_uid[int(datum_idx)]
-                            for datum_idx in metrics[iou_idx][score_idx][
-                                label_idx
-                            ][tp_idx + 1 : fp_misclf_idx]
-                            if int(datum_idx) >= 0
-                        ]
-                        for score_idx in range(n_scores)
-                    ],
-                    fp_misclassification=metrics[
-                        iou_idx, :, label_idx, fp_misclf_idx
-                    ]
-                    .astype(int)
-                    .tolist(),
-                    fp_misclassification_examples=[
-                        [
-                            self.index_to_uid[int(datum_idx)]
-                            for datum_idx in metrics[iou_idx][score_idx][
-                                label_idx
-                            ][fp_misclf_idx + 1 : fp_halluc_idx]
-                            if int(datum_idx) >= 0
-                        ]
-                        for score_idx in range(n_scores)
-                    ],
-                    fp_hallucination=metrics[
-                        iou_idx, :, label_idx, fp_halluc_idx
-                    ]
-                    .astype(int)
-                    .tolist(),
-                    fp_hallucination_examples=[
-                        [
-                            self.index_to_uid[int(datum_idx)]
-                            for datum_idx in metrics[iou_idx][score_idx][
-                                label_idx
-                            ][fp_halluc_idx + 1 : fn_misclf_idx]
-                            if int(datum_idx) >= 0
-                        ]
-                        for score_idx in range(n_scores)
-                    ],
-                    fn_misclassification=metrics[
-                        iou_idx, :, label_idx, fn_misclf_idx
-                    ]
-                    .astype(int)
-                    .tolist(),
-                    fn_misclassification_examples=[
-                        [
-                            self.index_to_uid[int(datum_idx)]
-                            for datum_idx in metrics[iou_idx][score_idx][
-                                label_idx
-                            ][fn_misclf_idx + 1 : fn_misprd_idx]
-                            if int(datum_idx) >= 0
-                        ]
-                        for score_idx in range(n_scores)
-                    ],
-                    fn_missing_prediction=metrics[
-                        iou_idx, :, label_idx, fn_misprd_idx
-                    ]
-                    .astype(int)
-                    .tolist(),
-                    fn_missing_prediction_examples=[
-                        [
-                            self.index_to_uid[int(datum_idx)]
-                            for datum_idx in metrics[iou_idx][score_idx][
-                                label_idx
-                            ][fn_misprd_idx + 1 :]
-                            if int(datum_idx) >= 0
-                        ]
-                        for score_idx in range(n_scores)
-                    ],
-                )
-                for iou_idx in range(n_ious)
-            ]
+            DetailedCounts(
+                iou_threshold=iou_thresholds[iou_idx],
+                label=self.index_to_label[label_idx],
+                score_thresholds=score_thresholds,
+                tp=metrics[iou_idx, :, label_idx, tp_idx].astype(int).tolist(),
+                fp_misclassification=metrics[
+                    iou_idx, :, label_idx, fp_misclf_idx
+                ]
+                .astype(int)
+                .tolist(),
+                fp_hallucination=metrics[iou_idx, :, label_idx, fp_halluc_idx]
+                .astype(int)
+                .tolist(),
+                fn_misclassification=metrics[
+                    iou_idx, :, label_idx, fn_misclf_idx
+                ]
+                .astype(int)
+                .tolist(),
+                fn_missing_prediction=metrics[
+                    iou_idx, :, label_idx, fn_misprd_idx
+                ]
+                .astype(int)
+                .tolist(),
+                tp_examples=_unpack_examples(
+                    iou_idx=iou_idx,
+                    label_idx=label_idx,
+                    type_idx=tp_idx,
+                    example_source=self.prediction_examples,
+                ),
+                fp_misclassification_examples=_unpack_examples(
+                    iou_idx=iou_idx,
+                    label_idx=label_idx,
+                    type_idx=fp_misclf_idx,
+                    example_source=self.prediction_examples,
+                ),
+                fp_hallucination_examples=_unpack_examples(
+                    iou_idx=iou_idx,
+                    label_idx=label_idx,
+                    type_idx=fp_halluc_idx,
+                    example_source=self.prediction_examples,
+                ),
+                fn_misclassification_examples=_unpack_examples(
+                    iou_idx=iou_idx,
+                    label_idx=label_idx,
+                    type_idx=fn_misclf_idx,
+                    example_source=self.groundtruth_examples,
+                ),
+                fn_missing_prediction_examples=_unpack_examples(
+                    iou_idx=iou_idx,
+                    label_idx=label_idx,
+                    type_idx=fn_misprd_idx,
+                    example_source=self.groundtruth_examples,
+                ),
+            )
             for label_idx in range(n_labels)
+            for iou_idx in range(n_ious)
         ]
 
 
@@ -561,7 +598,7 @@ class DataLoader:
 
     def __init__(self):
         self._evaluator = Evaluator()
-        self.pairs = list()
+        self.pairs: list[NDArray[np.floating]] = list()
         self.groundtruth_count = defaultdict(lambda: defaultdict(int))
         self.prediction_count = defaultdict(lambda: defaultdict(int))
 
@@ -624,6 +661,125 @@ class DataLoader:
             self._evaluator.label_key_to_index[label[0]],
         )
 
+    def _add_data(
+        self,
+        uid_index: int,
+        keyed_groundtruths: dict,
+        keyed_predictions: dict,
+    ):
+        gt_keys = set(keyed_groundtruths.keys())
+        pd_keys = set(keyed_predictions.keys())
+        joint_keys = gt_keys.intersection(pd_keys)
+        gt_unique_keys = gt_keys - pd_keys
+        pd_unique_keys = pd_keys - gt_keys
+
+        pairs = list()
+        for key in joint_keys:
+            n_predictions = len(keyed_predictions[key])
+            n_groundtruths = len(keyed_groundtruths[key])
+            boxes = np.array(
+                [
+                    np.array([*gextrema, *pextrema])
+                    for _, _, _, pextrema in keyed_predictions[key]
+                    for _, _, gextrema in keyed_groundtruths[key]
+                ]
+            )
+            ious = compute_iou(boxes)
+            mask_nonzero_iou = (ious > 1e-9).reshape(
+                (n_predictions, n_groundtruths)
+            )
+            mask_ious_halluc = ~(mask_nonzero_iou.any(axis=1))
+            mask_ious_misprd = ~(mask_nonzero_iou.any(axis=0))
+
+            pairs.extend(
+                [
+                    np.array(
+                        [
+                            float(uid_index),
+                            float(gidx),
+                            float(pidx),
+                            ious[pidx * len(keyed_groundtruths[key]) + gidx],
+                            float(glabel),
+                            float(plabel),
+                            float(score),
+                        ]
+                    )
+                    for pidx, plabel, score, _ in keyed_predictions[key]
+                    for gidx, glabel, _ in keyed_groundtruths[key]
+                    if ious[pidx * len(keyed_groundtruths[key]) + gidx] > 1e-9
+                ]
+            )
+            pairs.extend(
+                [
+                    np.array(
+                        [
+                            float(uid_index),
+                            -1.0,
+                            float(pidx),
+                            0.0,
+                            -1.0,
+                            float(plabel),
+                            float(score),
+                        ]
+                    )
+                    for pidx, plabel, score, _ in keyed_predictions[key]
+                    if mask_ious_halluc[pidx]
+                ]
+            )
+            pairs.extend(
+                [
+                    np.array(
+                        [
+                            float(uid_index),
+                            float(gidx),
+                            -1.0,
+                            0.0,
+                            float(glabel),
+                            -1.0,
+                            -1.0,
+                        ]
+                    )
+                    for gidx, glabel, _ in keyed_groundtruths[key]
+                    if mask_ious_misprd[gidx]
+                ]
+            )
+        for key in gt_unique_keys:
+            pairs.extend(
+                [
+                    np.array(
+                        [
+                            float(uid_index),
+                            float(gidx),
+                            -1.0,
+                            0.0,
+                            float(glabel),
+                            -1.0,
+                            -1.0,
+                        ]
+                    )
+                    for gidx, glabel, _ in keyed_groundtruths[key]
+                ]
+            )
+        for key in pd_unique_keys:
+            pairs.extend(
+                [
+                    np.array(
+                        [
+                            float(uid_index),
+                            -1.0,
+                            float(pidx),
+                            0.0,
+                            -1.0,
+                            float(plabel),
+                            float(score),
+                        ]
+                    )
+                    for pidx, plabel, score, _ in keyed_predictions[key]
+                ]
+            )
+
+        self.pairs.append(np.array(pairs))
+
     def add_data(
         self,
         detections: list[Detection],
@@ -650,10 +806,21 @@ class DataLoader:
             # update datum uid index
             uid_index = self._add_datum(uid=detection.uid)
 
+            # initialize bounding box examples
+            self._evaluator.groundtruth_examples[uid_index] = np.zeros(
+                (len(detection.groundtruths), 4), dtype=np.float16
+            )
+            self._evaluator.prediction_examples[uid_index] = np.zeros(
+                (len(detection.predictions), 4), dtype=np.float16
+            )
+
             # cache labels and annotations
             keyed_groundtruths = defaultdict(list)
             keyed_predictions = defaultdict(list)
             for gidx, gann in enumerate(detection.groundtruths):
+                self._evaluator.groundtruth_examples[uid_index][
+                    gidx
+                ] = np.array(gann.extrema)
                 for glabel in gann.labels:
                     label_idx, label_key_idx = self._add_label(glabel)
                     self.groundtruth_count[label_idx][uid_index] += 1
@@ -665,6 +832,9 @@ class DataLoader:
                         )
                     )
             for pidx, pann in enumerate(detection.predictions):
+                self._evaluator.prediction_examples[uid_index][
+                    pidx
+                ] = np.array(pann.extrema)
                 for plabel, pscore in zip(pann.labels, pann.scores):
                     label_idx, label_key_idx = self._add_label(plabel)
                     self.prediction_count[label_idx][uid_index] += 1
@@ -677,77 +847,11 @@ class DataLoader:
                         )
                     )
 
-            gt_keys = set(keyed_groundtruths.keys())
-            pd_keys = set(keyed_predictions.keys())
-            joint_keys = gt_keys.intersection(pd_keys)
-            gt_unique_keys = gt_keys - pd_keys
-            pd_unique_keys = pd_keys - gt_keys
-
-            pairs = list()
-            for key in joint_keys:
-                boxes = np.array(
-                    [
-                        np.array([*gextrema, *pextrema])
-                        for _, _, _, pextrema in keyed_predictions[key]
-                        for _, _, gextrema in keyed_groundtruths[key]
-                    ]
-                )
-                ious = compute_iou(boxes)
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                float(gidx),
-                                float(pidx),
-                                ious[
-                                    pidx * len(keyed_groundtruths[key]) + gidx
-                                ],
-                                float(glabel),
-                                float(plabel),
-                                float(score),
-                            ]
-                        )
-                        for pidx, plabel, score, _ in keyed_predictions[key]
-                        for gidx, glabel, _ in keyed_groundtruths[key]
-                    ]
-                )
-            for key in gt_unique_keys:
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                float(gidx),
-                                -1.0,
-                                0.0,
-                                float(glabel),
-                                -1.0,
-                                -1.0,
-                            ]
-                        )
-                        for gidx, glabel, _ in keyed_groundtruths[key]
-                    ]
-                )
-            for key in pd_unique_keys:
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                -1.0,
-                                float(pidx),
-                                0.0,
-                                -1.0,
-                                float(plabel),
-                                float(score),
-                            ]
-                        )
-                        for pidx, plabel, score, _ in keyed_predictions[key]
-                    ]
-                )
-
-            self.pairs.append(np.array(pairs))
+            self._add_data(
+                uid_index=uid_index,
+                keyed_groundtruths=keyed_groundtruths,
+                keyed_predictions=keyed_predictions,
+            )
 
     def add_data_from_valor_dict(
         self,
@@ -783,10 +887,21 @@ class DataLoader:
             # update datum uid index
             uid_index = self._add_datum(uid=groundtruth["datum"]["uid"])
 
+            # initialize bounding box examples
+            self._evaluator.groundtruth_examples[uid_index] = np.zeros(
+                (len(groundtruth["annotations"]), 4), dtype=np.float16
+            )
+            self._evaluator.prediction_examples[uid_index] = np.zeros(
+                (len(prediction["annotations"]), 4), dtype=np.float16
+            )
+
             # cache labels and annotations
             keyed_groundtruths = defaultdict(list)
             keyed_predictions = defaultdict(list)
             for gidx, gann in enumerate(groundtruth["annotations"]):
+                self._evaluator.groundtruth_examples[uid_index][
+                    gidx
+                ] = np.array(_get_bbox_extrema(gann["bounding_box"]))
                 for valor_label in gann["labels"]:
                     glabel = (valor_label["key"], valor_label["value"])
                     label_idx, label_key_idx = self._add_label(glabel)
@@ -799,6 +914,9 @@ class DataLoader:
                         )
                     )
             for pidx, pann in enumerate(prediction["annotations"]):
+                self._evaluator.prediction_examples[uid_index][
+                    pidx
+                ] = np.array(_get_bbox_extrema(pann["bounding_box"]))
                 for valor_label in pann["labels"]:
                     plabel = (valor_label["key"], valor_label["value"])
                     pscore = valor_label["score"]
@@ -813,77 +931,11 @@ class DataLoader:
                         )
                     )
 
-            gt_keys = set(keyed_groundtruths.keys())
-            pd_keys = set(keyed_predictions.keys())
-            joint_keys = gt_keys.intersection(pd_keys)
-            gt_unique_keys = gt_keys - pd_keys
-            pd_unique_keys = pd_keys - gt_keys
-
-            pairs = list()
-            for key in joint_keys:
-                boxes = np.array(
-                    [
-                        np.array([*gextrema, *pextrema])
-                        for _, _, _, pextrema in keyed_predictions[key]
-                        for _, _, gextrema in keyed_groundtruths[key]
-                    ]
-                )
-                ious = compute_iou(boxes)
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                float(gidx),
-                                float(pidx),
-                                ious[
-                                    pidx * len(keyed_groundtruths[key]) + gidx
-                                ],
-                                float(glabel),
-                                float(plabel),
-                                float(score),
-                            ]
-                        )
-                        for pidx, plabel, score, _ in keyed_predictions[key]
-                        for gidx, glabel, _ in keyed_groundtruths[key]
-                    ]
-                )
-            for key in gt_unique_keys:
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                float(gidx),
-                                -1.0,
-                                0.0,
-                                float(glabel),
-                                -1.0,
-                                -1.0,
-                            ]
-                        )
-                        for gidx, glabel, _ in keyed_groundtruths[key]
-                    ]
-                )
-            for key in pd_unique_keys:
-                pairs.extend(
-                    [
-                        np.array(
-                            [
-                                float(uid_index),
-                                -1.0,
-                                float(pidx),
-                                0.0,
-                                -1.0,
-                                float(plabel),
-                                float(score),
-                            ]
-                        )
-                        for pidx, plabel, score, _ in keyed_predictions[key]
-                    ]
-                )
-
-            self.pairs.append(np.array(pairs))
+            self._add_data(
+                uid_index=uid_index,
+                keyed_groundtruths=keyed_groundtruths,
+                keyed_predictions=keyed_predictions,
+            )
 
     def finalize(self) -> Evaluator:
         """

valor_lite/detection/metric.py

@@ -21,6 +21,25 @@ class MetricType(str, Enum):
     PrecisionRecallCurve = "PrecisionRecallCurve"
     DetailedCounts = "DetailedCounts"
 
+    @classmethod
+    def base_metrics(cls):
+        return [
+            cls.Counts,
+            cls.Accuracy,
+            cls.Precision,
+            cls.Recall,
+            cls.F1,
+            cls.AP,
+            cls.AR,
+            cls.mAP,
+            cls.mAR,
+            cls.APAveragedOverIOUs,
+            cls.mAPAveragedOverIOUs,
+            cls.ARAveragedOverScores,
+            cls.mARAveragedOverScores,
+            cls.PrecisionRecallCurve,
+        ]
+
 
 @dataclass
 class Counts:
@@ -316,11 +335,19 @@ class DetailedCounts:
     fp_hallucination: list[int]
     fn_misclassification: list[int]
     fn_missing_prediction: list[int]
-    tp_examples: list[list[str]]
-    fp_misclassification_examples: list[list[str]]
-    fp_hallucination_examples: list[list[str]]
-    fn_misclassification_examples: list[list[str]]
-    fn_missing_prediction_examples: list[list[str]]
+    tp_examples: list[list[tuple[str, tuple[float, float, float, float]]]]
+    fp_misclassification_examples: list[
+        list[tuple[str, tuple[float, float, float, float]]]
+    ]
+    fp_hallucination_examples: list[
+        list[tuple[str, tuple[float, float, float, float]]]
+    ]
+    fn_misclassification_examples: list[
+        list[tuple[str, tuple[float, float, float, float]]]
+    ]
+    fn_missing_prediction_examples: list[
+        list[tuple[str, tuple[float, float, float, float]]]
+    ]
     score_thresholds: list[float]
     iou_threshold: float
     label: tuple[str, str]
@@ -335,13 +362,11 @@ class DetailedCounts:
                 "fp_hallucination": self.fp_hallucination,
                 "fn_misclassification": self.fn_misclassification,
                 "fn_missing_prediction": self.fn_missing_prediction,
-                "tn": None,
                 "tp_examples": self.tp_examples,
                 "fp_misclassification_examples": self.fp_misclassification_examples,
                 "fp_hallucination_examples": self.fp_hallucination_examples,
                 "fn_misclassification_examples": self.fn_misclassification_examples,
                 "fn_missing_prediction_examples": self.fn_missing_prediction_examples,
-                "tn_examples": None,
             },
             parameters={
                 "score_thresholds": self.score_thresholds,

{valor_lite-0.33.2.dist-info → valor_lite-0.33.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.2
+Version: 0.33.3
 Summary: Compute valor metrics directly in your client.
 License: MIT License
         

valor_lite-0.33.3.dist-info/RECORD

@@ -0,0 +1,12 @@
+valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
+valor_lite/detection/__init__.py,sha256=WHLHwHoKzXTBjkjC6E1_lhqB7gRWkiGWVWPqkKn-yK8,997
+valor_lite/detection/annotation.py,sha256=c45pZD1Pp2vf5GeyW_6Kl9JCx5FoaaktCaaa4q3QDUo,1758
+valor_lite/detection/computation.py,sha256=7PttK0VuOWlhRN92wpLVrGzB7RAdfdZyT3b1aTm_WaI,23214
+valor_lite/detection/manager.py,sha256=ziVnukGs-WrkyBEBBO3LVSv4LTbaWFaWqLWarVosj2c,35807
+valor_lite/detection/metric.py,sha256=DLqpODJZOG7SCqt7TCgR4am68PQORRCIQW_SXiTb1IA,9473
+valor_lite-0.33.3.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite-0.33.3.dist-info/METADATA,sha256=FBpd-wMWv-m37EK5BfFuiVmnJXg4GNzCaJrTDHv4-gE,1842
+valor_lite-0.33.3.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+valor_lite-0.33.3.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.33.3.dist-info/RECORD,,

valor_lite-0.33.2.dist-info/RECORD

@@ -1,12 +0,0 @@
-valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
-valor_lite/detection/__init__.py,sha256=WHLHwHoKzXTBjkjC6E1_lhqB7gRWkiGWVWPqkKn-yK8,997
-valor_lite/detection/annotation.py,sha256=ON9iVa33pxysUmZVTCb0wNz-eFX6MDOqDhGDz-ouymc,1466
-valor_lite/detection/computation.py,sha256=L8FIwZ-qxOQnoT7mxgNzLyNyI-Bvga0i-gtbow3hN-o,22575
-valor_lite/detection/manager.py,sha256=Y45Wy3PWi7dQ0VnDERdtpOixUbKVXTZxBcCR92ny0QY,34278
-valor_lite/detection/metric.py,sha256=hHqClS7c71ztoUnfoaW3T7RmGYaVNU1SlM6vUs1P08I,8809
-valor_lite-0.33.2.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
-valor_lite-0.33.2.dist-info/METADATA,sha256=fe-Sj568DB-E9cyC5P8GA_lLjmM1t3MZUHj1f0JF6fM,1842
-valor_lite-0.33.2.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-valor_lite-0.33.2.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
-valor_lite-0.33.2.dist-info/RECORD,,