valor-lite 0.32.2a2__py3-none-any.whl → 0.33.1__py3-none-any.whl

valor_lite/detection/__init__.py

@@ -1,6 +1,6 @@
 from .annotation import Bitmask, BoundingBox, Detection
 from .computation import (
-    compute_detailed_pr_curve,
+    compute_detailed_counts,
     compute_iou,
     compute_metrics,
     compute_ranked_pairs,
@@ -14,8 +14,7 @@ from .metric import (
     APAveragedOverIOUs,
     ARAveragedOverScores,
     Counts,
-    DetailedPrecisionRecallCurve,
-    DetailedPrecisionRecallPoint,
+    DetailedCounts,
     MetricType,
     Precision,
     PrecisionRecallCurve,
@@ -45,12 +44,11 @@ __all__ = [
     "ARAveragedOverScores",
     "mARAveragedOverScores",
     "PrecisionRecallCurve",
-    "DetailedPrecisionRecallPoint",
-    "DetailedPrecisionRecallCurve",
+    "DetailedCounts",
     "compute_iou",
     "compute_ranked_pairs",
     "compute_metrics",
-    "compute_detailed_pr_curve",
+    "compute_detailed_counts",
     "DataLoader",
     "Evaluator",
 ]

valor_lite/detection/computation.py

@@ -49,7 +49,7 @@ def compute_iou(data: NDArray[np.floating]) -> NDArray[np.floating]:
 
 def _compute_ranked_pairs_for_datum(
     data: np.ndarray,
-    label_counts: np.ndarray,
+    label_metadata: np.ndarray,
 ) -> np.ndarray:
     """
     Computes ranked pairs for a datum.
@@ -58,6 +58,12 @@ def _compute_ranked_pairs_for_datum(
     # remove null predictions
     data = data[data[:, 2] >= 0.0]
 
+    # find best fits for prediction
+    mask_label_match = data[:, 4] == data[:, 5]
+    matched_predicitons = np.unique(data[mask_label_match, 2].astype(int))
+    mask_unmatched_predictions = ~np.isin(data[:, 2], matched_predicitons)
+    data = data[mask_label_match | mask_unmatched_predictions]
+
     # sort by gt_id, iou, score
     indices = np.lexsort(
         (
@@ -69,7 +75,7 @@ def _compute_ranked_pairs_for_datum(
     data = data[indices]
 
     # remove ignored predictions
-    for label_idx, count in enumerate(label_counts[:, 0]):
+    for label_idx, count in enumerate(label_metadata[:, 0]):
         if count > 0:
             continue
         data = data[data[:, 5] != label_idx]
@@ -85,13 +91,13 @@ def _compute_ranked_pairs_for_datum(
 
 def compute_ranked_pairs(
     data: list[NDArray[np.floating]],
-    label_counts: NDArray[np.integer],
+    label_metadata: NDArray[np.integer],
 ) -> NDArray[np.floating]:
     pairs = np.concatenate(
         [
             _compute_ranked_pairs_for_datum(
                 datum,
-                label_counts=label_counts,
+                label_metadata=label_metadata,
             )
             for datum in data
         ],
@@ -108,7 +114,7 @@ def compute_ranked_pairs(
 
 def compute_metrics(
     data: np.ndarray,
-    label_counts: np.ndarray,
+    label_metadata: np.ndarray,
     iou_thresholds: np.ndarray,
     score_thresholds: np.ndarray,
 ) -> tuple[
@@ -143,7 +149,7 @@ def compute_metrics(
     """
 
     n_rows = data.shape[0]
-    n_labels = label_counts.shape[0]
+    n_labels = label_metadata.shape[0]
     n_ious = iou_thresholds.shape[0]
     n_scores = score_thresholds.shape[0]
 
@@ -153,7 +159,7 @@ def compute_metrics(
 
     pd_labels = data[:, 5].astype(int)
     unique_pd_labels = np.unique(pd_labels)
-    gt_count = label_counts[:, 0]
+    gt_count = label_metadata[:, 0]
     running_total_count = np.zeros(
         (n_ious, n_rows),
         dtype=np.float64,
@@ -313,8 +319,8 @@ def compute_metrics(
     average_recall /= n_ious
 
     # calculate mAP and mAR
-    label_key_mapping = label_counts[unique_pd_labels, 2]
-    label_keys = np.unique(label_counts[:, 2])
+    label_key_mapping = label_metadata[unique_pd_labels, 2]
+    label_keys = np.unique(label_metadata[:, 2])
     mAP = np.ones((n_ious, label_keys.shape[0])) * -1.0
     mAR = np.ones((n_scores, label_keys.shape[0])) * -1.0
     for key in np.unique(label_key_mapping):
@@ -352,9 +358,9 @@ def compute_metrics(
     )
 
 
-def compute_detailed_pr_curve(
+def compute_detailed_counts(
     data: np.ndarray,
-    label_counts: np.ndarray,
+    label_metadata: np.ndarray,
     iou_thresholds: np.ndarray,
     score_thresholds: np.ndarray,
     n_samples: int,
@@ -368,9 +374,10 @@ def compute_detailed_pr_curve(
     3  fp - 2
     4  fn - misclassification
     5  fn - hallucination
+    6  tn
     """
 
-    n_labels = label_counts.shape[0]
+    n_labels = label_metadata.shape[0]
     n_ious = iou_thresholds.shape[0]
     n_scores = score_thresholds.shape[0]
     n_metrics = 5 * (n_samples + 1)
@@ -386,46 +393,97 @@ def compute_detailed_pr_curve(
     mask_gt_exists = data[:, 1] > -0.5
     mask_pd_exists = data[:, 2] > -0.5
     mask_label_match = np.isclose(data[:, 4], data[:, 5])
+    mask_score_nonzero = data[:, 6] > 1e-9
+    mask_iou_nonzero = data[:, 3] > 1e-9
 
     mask_gt_pd_exists = mask_gt_exists & mask_pd_exists
     mask_gt_pd_match = mask_gt_pd_exists & mask_label_match
     mask_gt_pd_mismatch = mask_gt_pd_exists & ~mask_label_match
 
+    groundtruths = data[:, [0, 1]].astype(int)
+    predictions = data[:, [0, 2]].astype(int)
     for iou_idx in range(n_ious):
-        mask_iou = data[:, 3] >= iou_thresholds[iou_idx]
-        mask_gt_pd_match_iou = mask_gt_pd_match & mask_iou
-        mask_gt_pd_mismatch_iou = mask_gt_pd_mismatch & mask_iou
+        mask_iou_threshold = data[:, 3] >= iou_thresholds[iou_idx]
+        mask_iou = mask_iou_nonzero & mask_iou_threshold
+
+        groundtruths_with_pairs = np.unique(groundtruths[mask_iou], axis=0)
+        mask_groundtruths_with_passing_ious = (
+            groundtruths.reshape(-1, 1, 2)
+            == groundtruths_with_pairs.reshape(1, -1, 2)
+        ).all(axis=2)
+        mask_groundtruths_with_passing_ious = (
+            mask_groundtruths_with_passing_ious.any(axis=1)
+        )
+        mask_groundtruths_without_passing_ious = (
+            ~mask_groundtruths_with_passing_ious & mask_gt_exists
+        )
+
+        predictions_with_passing_ious = np.unique(
+            predictions[mask_iou], axis=0
+        )
+        mask_predictions_with_passing_ious = (
+            predictions.reshape(-1, 1, 2)
+            == predictions_with_passing_ious.reshape(1, -1, 2)
+        ).all(axis=2)
+        mask_predictions_with_passing_ious = (
+            mask_predictions_with_passing_ious.any(axis=1)
+        )
+        mask_predictions_without_passing_ious = (
+            ~mask_predictions_with_passing_ious & mask_pd_exists
+        )
+
         for score_idx in range(n_scores):
-            mask_score = data[:, 6] >= score_thresholds[score_idx]
-            mask_tp = mask_gt_pd_match_iou & mask_score
-            mask_fp_misclf = mask_gt_pd_mismatch_iou & mask_score
-            mask_fn_misclf = mask_gt_pd_match_iou & ~mask_score
-            mask_halluc_missing = ~(
-                mask_gt_pd_match_iou | (mask_gt_pd_mismatch & mask_score)
-            )
-            mask_fp_halluc = mask_halluc_missing & mask_pd_exists
-            mask_fn_misprd = mask_halluc_missing & mask_gt_exists
+            mask_score_threshold = data[:, 6] >= score_thresholds[score_idx]
+            mask_score = mask_score_nonzero & mask_score_threshold
 
-            tp_slice = data[mask_tp]
-            fp_misclf_slice = data[mask_fp_misclf]
-            fp_halluc_slice = data[mask_fp_halluc]
-            fn_misclf_slice = data[mask_fn_misclf]
-            fn_misprd_slice = data[mask_fn_misprd]
+            groundtruths_with_passing_score = np.unique(
+                groundtruths[mask_iou & mask_score], axis=0
+            )
+            mask_groundtruths_with_passing_score = (
+                groundtruths.reshape(-1, 1, 2)
+                == groundtruths_with_passing_score.reshape(1, -1, 2)
+            ).all(axis=2)
+            mask_groundtruths_with_passing_score = (
+                mask_groundtruths_with_passing_score.any(axis=1)
+            )
+            mask_groundtruths_without_passing_score = (
+                ~mask_groundtruths_with_passing_score & mask_gt_exists
+            )
 
-            tp_count = np.bincount(
-                tp_slice[:, 5].astype(int), minlength=n_labels
+            mask_tp = mask_score & mask_iou & mask_gt_pd_match
+            mask_fp_misclf = mask_score & mask_iou & mask_gt_pd_mismatch
+            mask_fn_misclf = mask_iou & (
+                (
+                    ~mask_score
+                    & mask_gt_pd_match
+                    & mask_groundtruths_with_passing_score
+                )
+                | (mask_score & mask_gt_pd_mismatch)
+            )
+            mask_fp_halluc = mask_score & mask_predictions_without_passing_ious
+            mask_fn_misprd = (
+                mask_groundtruths_without_passing_ious
+                | mask_groundtruths_without_passing_score
             )
+
+            tp = np.unique(data[mask_tp][:, [0, 2, 5]], axis=0)
+            fp_misclf = np.unique(data[mask_fp_misclf][:, [0, 2, 5]], axis=0)
+            fp_halluc = np.unique(data[mask_fp_halluc][:, [0, 2, 5]], axis=0)
+            fn_misclf = np.unique(data[mask_fn_misclf][:, [0, 1, 4]], axis=0)
+            fn_misprd = np.unique(data[mask_fn_misprd][:, [0, 1, 4]], axis=0)
+
+            tp_count = np.bincount(tp[:, 2].astype(int), minlength=n_labels)
             fp_misclf_count = np.bincount(
-                fp_misclf_slice[:, 5].astype(int), minlength=n_labels
+                fp_misclf[:, 2].astype(int), minlength=n_labels
             )
             fp_halluc_count = np.bincount(
-                fp_halluc_slice[:, 5].astype(int), minlength=n_labels
+                fp_halluc[:, 2].astype(int), minlength=n_labels
             )
             fn_misclf_count = np.bincount(
-                fn_misclf_slice[:, 4].astype(int), minlength=n_labels
+                fn_misclf[:, 2].astype(int), minlength=n_labels
             )
             fn_misprd_count = np.bincount(
-                fn_misprd_slice[:, 4].astype(int), minlength=n_labels
+                fn_misprd[:, 2].astype(int), minlength=n_labels
             )
 
             detailed_pr_curve[iou_idx, score_idx, :, tp_idx] = tp_count
@@ -444,20 +502,20 @@ def compute_detailed_pr_curve(
 
             if n_samples > 0:
                 for label_idx in range(n_labels):
-                    tp_examples = tp_slice[
-                        tp_slice[:, 5].astype(int) == label_idx
-                    ][:n_samples, 0]
-                    fp_misclf_examples = fp_misclf_slice[
-                        fp_misclf_slice[:, 5].astype(int) == label_idx
+                    tp_examples = tp[tp[:, 2].astype(int) == label_idx][
+                        :n_samples, 0
+                    ]
+                    fp_misclf_examples = fp_misclf[
+                        fp_misclf[:, 2].astype(int) == label_idx
                     ][:n_samples, 0]
-                    fp_halluc_examples = fp_halluc_slice[
-                        fp_halluc_slice[:, 5].astype(int) == label_idx
+                    fp_halluc_examples = fp_halluc[
+                        fp_halluc[:, 2].astype(int) == label_idx
                     ][:n_samples, 0]
-                    fn_misclf_examples = fn_misclf_slice[
-                        fn_misclf_slice[:, 4].astype(int) == label_idx
+                    fn_misclf_examples = fn_misclf[
+                        fn_misclf[:, 2].astype(int) == label_idx
                     ][:n_samples, 0]
-                    fn_misprd_examples = fn_misprd_slice[
-                        fn_misprd_slice[:, 4].astype(int) == label_idx
+                    fn_misprd_examples = fn_misprd[
+                        fn_misprd[:, 2].astype(int) == label_idx
                     ][:n_samples, 0]
 
                     detailed_pr_curve[

valor_lite/detection/manager.py

@@ -6,7 +6,7 @@ from numpy.typing import NDArray
 from tqdm import tqdm
 from valor_lite.detection.annotation import Detection
 from valor_lite.detection.computation import (
-    compute_detailed_pr_curve,
+    compute_detailed_counts,
     compute_iou,
     compute_metrics,
     compute_ranked_pairs,
@@ -19,8 +19,7 @@ from valor_lite.detection.metric import (
     APAveragedOverIOUs,
     ARAveragedOverScores,
     Counts,
-    DetailedPrecisionRecallCurve,
-    DetailedPrecisionRecallPoint,
+    DetailedCounts,
     MetricType,
     Precision,
     PrecisionRecallCurve,
@@ -35,12 +34,12 @@ from valor_lite.detection.metric import (
 Usage
 -----
 
-manager = DataLoader()
-manager.add_data(
+loader = DataLoader()
+loader.add_data(
     groundtruths=groundtruths,
     predictions=predictions,
 )
-evaluator = manager.finalize()
+evaluator = loader.finalize()
 
 metrics = evaluator.evaluate(iou_thresholds=[0.5])
 
@@ -120,7 +119,7 @@ class Evaluator:
         label_keys: list[str] | NDArray[np.int32] | None = None,
     ) -> Filter:
         """
-        Creates a boolean mask that can be passed to an evaluation.
+        Creates a filter that can be passed to an evaluation.
 
         Parameters
         ----------
@@ -208,9 +207,6 @@ class Evaluator:
         return Filter(
             indices=np.where(mask_pairs)[0],
             label_metadata=label_metadata,
-            # uids=datum_uids,
-            # labels=labels,
-            # label_keys=label_keys,
         )
 
     def evaluate(
@@ -255,7 +251,7 @@ class Evaluator:
             pr_curves,
         ) = compute_metrics(
             data=data,
-            label_counts=label_metadata,
+            label_metadata=label_metadata,
             iou_thresholds=np.array(iou_thresholds),
             score_thresholds=np.array(score_thresholds),
         )
@@ -265,7 +261,7 @@ class Evaluator:
         metrics[MetricType.AP] = [
             AP(
                 value=average_precision[iou_idx][label_idx],
-                iou=iou_thresholds[iou_idx],
+                iou_threshold=iou_thresholds[iou_idx],
                 label=self.index_to_label[label_idx],
             )
             for iou_idx in range(average_precision.shape[0])
@@ -276,7 +272,7 @@ class Evaluator:
         metrics[MetricType.mAP] = [
             mAP(
                 value=mean_average_precision[iou_idx][label_key_idx],
-                iou=iou_thresholds[iou_idx],
+                iou_threshold=iou_thresholds[iou_idx],
                 label_key=self.index_to_label_key[label_key_idx],
             )
             for iou_idx in range(mean_average_precision.shape[0])
@@ -286,7 +282,7 @@ class Evaluator:
         metrics[MetricType.APAveragedOverIOUs] = [
             APAveragedOverIOUs(
                 value=average_precision_average_over_ious[label_idx],
-                ious=iou_thresholds,
+                iou_thresholds=iou_thresholds,
                 label=self.index_to_label[label_idx],
             )
             for label_idx in range(self.n_labels)
@@ -296,7 +292,7 @@ class Evaluator:
         metrics[MetricType.mAPAveragedOverIOUs] = [
             mAPAveragedOverIOUs(
                 value=mean_average_precision_average_over_ious[label_key_idx],
-                ious=iou_thresholds,
+                iou_thresholds=iou_thresholds,
                 label_key=self.index_to_label_key[label_key_idx],
             )
             for label_key_idx in range(
@@ -307,8 +303,8 @@ class Evaluator:
         metrics[MetricType.AR] = [
             AR(
                 value=average_recall[score_idx][label_idx],
-                ious=iou_thresholds,
-                score=score_thresholds[score_idx],
+                iou_thresholds=iou_thresholds,
+                score_threshold=score_thresholds[score_idx],
                 label=self.index_to_label[label_idx],
             )
             for score_idx in range(average_recall.shape[0])
@@ -319,8 +315,8 @@ class Evaluator:
         metrics[MetricType.mAR] = [
             mAR(
                 value=mean_average_recall[score_idx][label_key_idx],
-                ious=iou_thresholds,
-                score=score_thresholds[score_idx],
+                iou_thresholds=iou_thresholds,
+                score_threshold=score_thresholds[score_idx],
                 label_key=self.index_to_label_key[label_key_idx],
             )
             for score_idx in range(mean_average_recall.shape[0])
@@ -330,8 +326,8 @@ class Evaluator:
         metrics[MetricType.ARAveragedOverScores] = [
             ARAveragedOverScores(
                 value=average_recall_averaged_over_scores[label_idx],
-                scores=score_thresholds,
-                ious=iou_thresholds,
+                score_thresholds=score_thresholds,
+                iou_thresholds=iou_thresholds,
                 label=self.index_to_label[label_idx],
             )
             for label_idx in range(self.n_labels)
@@ -341,8 +337,8 @@ class Evaluator:
         metrics[MetricType.mARAveragedOverScores] = [
             mARAveragedOverScores(
                 value=mean_average_recall_averaged_over_scores[label_key_idx],
-                scores=score_thresholds,
-                ious=iou_thresholds,
+                score_thresholds=score_thresholds,
+                iou_thresholds=iou_thresholds,
                 label_key=self.index_to_label_key[label_key_idx],
             )
             for label_key_idx in range(
@@ -353,7 +349,7 @@ class Evaluator:
         metrics[MetricType.PrecisionRecallCurve] = [
             PrecisionRecallCurve(
                 precision=list(pr_curves[iou_idx][label_idx]),
-                iou=iou_threshold,
+                iou_threshold=iou_threshold,
                 label=label,
             )
             for iou_idx, iou_threshold in enumerate(iou_thresholds)
@@ -361,14 +357,14 @@ class Evaluator:
             if int(label_metadata[label_idx][0]) > 0
         ]
 
-        for iou_idx, iou_threshold in enumerate(iou_thresholds):
+        for label_idx, label in self.index_to_label.items():
             for score_idx, score_threshold in enumerate(score_thresholds):
-                for label_idx, label in self.index_to_label.items():
+                for iou_idx, iou_threshold in enumerate(iou_thresholds):
                     row = precision_recall[iou_idx][score_idx][label_idx]
                     kwargs = {
                         "label": label,
-                        "iou": iou_threshold,
-                        "score": score_threshold,
+                        "iou_threshold": iou_threshold,
+                        "score_threshold": score_threshold,
                     }
                     metrics[MetricType.Counts].append(
                         Counts(
@@ -405,21 +401,38 @@ class Evaluator:
 
         return metrics
 
-    def compute_detailed_pr_curve(
+    def compute_detailed_counts(
         self,
         iou_thresholds: list[float] = [0.5],
         score_thresholds: list[float] = [
             score / 10.0 for score in range(1, 11)
         ],
         n_samples: int = 0,
-    ) -> list[DetailedPrecisionRecallCurve]:
+    ) -> list[list[DetailedCounts]]:
+        """
+        Computes detailed counting metrics.
+
+        Parameters
+        ----------
+        iou_thresholds : list[float], default=[0.5]
+            List of IoU thresholds to compute metrics for.
+        score_thresholds : list[float], default=[0.1,0.2,...,1.0]
+            List of confidence thresholds to compute metrics for.
+        n_samples : int, default=0
+            Number of datum samples to return per metric.
+
+        Returns
+        -------
+        list[list[DetailedCounts]]
+            Outer list is indexed by label, inner list is by IoU.
+        """
 
         if self._detailed_pairs.size == 0:
             return list()
 
-        metrics = compute_detailed_pr_curve(
+        metrics = compute_detailed_counts(
             self._detailed_pairs,
-            label_counts=self._label_metadata,
+            label_metadata=self._label_metadata,
             iou_thresholds=np.array(iou_thresholds),
             score_thresholds=np.array(score_thresholds),
             n_samples=n_samples,
@@ -431,71 +444,91 @@ class Evaluator:
         fn_misclf_idx = fp_halluc_idx + n_samples + 1
         fn_misprd_idx = fn_misclf_idx + n_samples + 1
 
-        results = list()
-        for label_idx in range(len(metrics)):
-            n_ious, n_scores, _, _ = metrics.shape
-            for iou_idx in range(n_ious):
-                curve = DetailedPrecisionRecallCurve(
-                    iou=iou_thresholds[iou_idx],
-                    value=list(),
+        n_ious, n_scores, n_labels, _ = metrics.shape
+        return [
+            [
+                DetailedCounts(
+                    iou_threshold=iou_thresholds[iou_idx],
                     label=self.index_to_label[label_idx],
-                )
-                for score_idx in range(n_scores):
-                    curve.value.append(
-                        DetailedPrecisionRecallPoint(
-                            score=score_thresholds[score_idx],
-                            tp=metrics[iou_idx][score_idx][label_idx][tp_idx],
-                            tp_examples=[
-                                self.index_to_uid[int(datum_idx)]
-                                for datum_idx in metrics[iou_idx][score_idx][
-                                    label_idx
-                                ][tp_idx + 1 : fp_misclf_idx]
-                                if int(datum_idx) >= 0
-                            ],
-                            fp_misclassification=metrics[iou_idx][score_idx][
+                    score_thresholds=score_thresholds,
+                    tp=metrics[iou_idx, :, label_idx, tp_idx]
+                    .astype(int)
+                    .tolist(),
+                    tp_examples=[
+                        [
+                            self.index_to_uid[int(datum_idx)]
+                            for datum_idx in metrics[iou_idx][score_idx][
                                 label_idx
-                            ][fp_misclf_idx],
-                            fp_misclassification_examples=[
-                                self.index_to_uid[int(datum_idx)]
-                                for datum_idx in metrics[iou_idx][score_idx][
-                                    label_idx
-                                ][fp_misclf_idx + 1 : fp_halluc_idx]
-                                if int(datum_idx) >= 0
-                            ],
-                            fp_hallucination=metrics[iou_idx][score_idx][
+                            ][tp_idx + 1 : fp_misclf_idx]
+                            if int(datum_idx) >= 0
+                        ]
+                        for score_idx in range(n_scores)
+                    ],
+                    fp_misclassification=metrics[
+                        iou_idx, :, label_idx, fp_misclf_idx
+                    ]
+                    .astype(int)
+                    .tolist(),
+                    fp_misclassification_examples=[
+                        [
+                            self.index_to_uid[int(datum_idx)]
+                            for datum_idx in metrics[iou_idx][score_idx][
                                 label_idx
-                            ][fp_halluc_idx],
-                            fp_hallucination_examples=[
-                                self.index_to_uid[int(datum_idx)]
-                                for datum_idx in metrics[iou_idx][score_idx][
-                                    label_idx
-                                ][fp_halluc_idx + 1 : fn_misclf_idx]
-                                if int(datum_idx) >= 0
-                            ],
-                            fn_misclassification=metrics[iou_idx][score_idx][
+                            ][fp_misclf_idx + 1 : fp_halluc_idx]
+                            if int(datum_idx) >= 0
+                        ]
+                        for score_idx in range(n_scores)
+                    ],
+                    fp_hallucination=metrics[
+                        iou_idx, :, label_idx, fp_halluc_idx
+                    ]
+                    .astype(int)
+                    .tolist(),
+                    fp_hallucination_examples=[
+                        [
+                            self.index_to_uid[int(datum_idx)]
+                            for datum_idx in metrics[iou_idx][score_idx][
                                 label_idx
-                            ][fn_misclf_idx],
-                            fn_misclassification_examples=[
-                                self.index_to_uid[int(datum_idx)]
-                                for datum_idx in metrics[iou_idx][score_idx][
-                                    label_idx
-                                ][fn_misclf_idx + 1 : fn_misprd_idx]
-                                if int(datum_idx) >= 0
-                            ],
-                            fn_missing_prediction=metrics[iou_idx][score_idx][
+                            ][fp_halluc_idx + 1 : fn_misclf_idx]
+                            if int(datum_idx) >= 0
+                        ]
+                        for score_idx in range(n_scores)
+                    ],
+                    fn_misclassification=metrics[
+                        iou_idx, :, label_idx, fn_misclf_idx
+                    ]
+                    .astype(int)
+                    .tolist(),
+                    fn_misclassification_examples=[
+                        [
+                            self.index_to_uid[int(datum_idx)]
+                            for datum_idx in metrics[iou_idx][score_idx][
                                 label_idx
-                            ][fn_misprd_idx],
-                            fn_missing_prediction_examples=[
-                                self.index_to_uid[int(datum_idx)]
-                                for datum_idx in metrics[iou_idx][score_idx][
-                                    label_idx
-                                ][fn_misprd_idx + 1 :]
-                                if int(datum_idx) >= 0
-                            ],
-                        )
-                    )
-                results.append(curve)
-        return results
+                            ][fn_misclf_idx + 1 : fn_misprd_idx]
+                            if int(datum_idx) >= 0
+                        ]
+                        for score_idx in range(n_scores)
+                    ],
+                    fn_missing_prediction=metrics[
+                        iou_idx, :, label_idx, fn_misprd_idx
+                    ]
+                    .astype(int)
+                    .tolist(),
+                    fn_missing_prediction_examples=[
+                        [
+                            self.index_to_uid[int(datum_idx)]
+                            for datum_idx in metrics[iou_idx][score_idx][
+                                label_idx
+                            ][fn_misprd_idx + 1 :]
+                            if int(datum_idx) >= 0
+                        ]
+                        for score_idx in range(n_scores)
+                    ],
+                )
+                for iou_idx in range(n_ious)
+            ]
+            for label_idx in range(n_labels)
+        ]
 
 
 class DataLoader:
@@ -839,7 +872,7 @@ class DataLoader:
 
         self._evaluator._ranked_pairs = compute_ranked_pairs(
             self.pairs,
-            label_counts=self._evaluator._label_metadata,
+            label_metadata=self._evaluator._label_metadata,
         )
 
         return self._evaluator

valor_lite/detection/metric.py

@@ -19,7 +19,7 @@ class MetricType(str, Enum):
     ARAveragedOverScores = "ARAveragedOverScores"
     mARAveragedOverScores = "mARAveragedOverScores"
     PrecisionRecallCurve = "PrecisionRecallCurve"
-    DetailedPrecisionRecallCurve = "DetailedPrecisionRecallCurve"
+    DetailedCounts = "DetailedCounts"
 
 
 @dataclass
@@ -28,8 +28,8 @@ class Counts:
     fp: int
     fn: int
     label: tuple[str, str]
-    iou: float
-    score: float
+    iou_threshold: float
+    score_threshold: float
 
     @property
     def metric(self) -> Metric:
@@ -41,8 +41,8 @@ class Counts:
                 "fn": self.fn,
             },
             parameters={
-                "iou": self.iou,
-                "score": self.score,
+                "iou_threshold": self.iou_threshold,
+                "score_threshold": self.score_threshold,
                 "label": {
                     "key": self.label[0],
                     "value": self.label[1],
@@ -58,8 +58,8 @@ class Counts:
 class ClassMetric:
     value: float
     label: tuple[str, str]
-    iou: float
-    score: float
+    iou_threshold: float
+    score_threshold: float
 
     @property
     def metric(self) -> Metric:
@@ -67,8 +67,8 @@ class ClassMetric:
             type=type(self).__name__,
             value=self.value,
             parameters={
-                "iou": self.iou,
-                "score": self.score,
+                "iou_threshold": self.iou_threshold,
+                "score_threshold": self.score_threshold,
                 "label": {
                     "key": self.label[0],
                     "value": self.label[1],
@@ -99,7 +99,7 @@ class F1(ClassMetric):
 @dataclass
 class AP:
     value: float
-    iou: float
+    iou_threshold: float
     label: tuple[str, str]
 
     @property
@@ -108,7 +108,7 @@ class AP:
             type=type(self).__name__,
             value=self.value,
             parameters={
-                "iou": self.iou,
+                "iou_threshold": self.iou_threshold,
                 "label": {
                     "key": self.label[0],
                     "value": self.label[1],
@@ -123,7 +123,7 @@ class AP:
 @dataclass
 class mAP:
     value: float
-    iou: float
+    iou_threshold: float
     label_key: str
 
     @property
@@ -132,7 +132,7 @@ class mAP:
             type=type(self).__name__,
             value=self.value,
             parameters={
-                "iou": self.iou,
+                "iou_threshold": self.iou_threshold,
                 "label_key": self.label_key,
             },
         )
@@ -144,7 +144,7 @@ class mAP:
 @dataclass
 class APAveragedOverIOUs:
     value: float
-    ious: list[float]
+    iou_thresholds: list[float]
     label: tuple[str, str]
 
     @property
@@ -153,7 +153,7 @@ class APAveragedOverIOUs:
             type=type(self).__name__,
             value=self.value,
             parameters={
-                "ious": self.ious,
+                "iou_thresholds": self.iou_thresholds,
                 "label": {
                     "key": self.label[0],
                     "value": self.label[1],
@@ -168,7 +168,7 @@ class APAveragedOverIOUs:
 @dataclass
 class mAPAveragedOverIOUs:
     value: float
-    ious: list[float]
+    iou_thresholds: list[float]
     label_key: str
 
     @property
@@ -177,7 +177,7 @@ class mAPAveragedOverIOUs:
             type=type(self).__name__,
             value=self.value,
             parameters={
-                "ious": self.ious,
+                "iou_thresholds": self.iou_thresholds,
                 "label_key": self.label_key,
             },
         )
@@ -189,8 +189,8 @@ class mAPAveragedOverIOUs:
 @dataclass
 class AR:
     value: float
-    score: float
-    ious: list[float]
+    score_threshold: float
+    iou_thresholds: list[float]
     label: tuple[str, str]
 
     @property
@@ -199,8 +199,8 @@ class AR:
             type=type(self).__name__,
             value=self.value,
             parameters={
-                "score": self.score,
-                "ious": self.ious,
+                "score_threshold": self.score_threshold,
+                "iou_thresholds": self.iou_thresholds,
                 "label": {
                     "key": self.label[0],
                     "value": self.label[1],
@@ -215,8 +215,8 @@ class AR:
 @dataclass
 class mAR:
     value: float
-    score: float
-    ious: list[float]
+    score_threshold: float
+    iou_thresholds: list[float]
     label_key: str
 
     @property
@@ -225,8 +225,8 @@ class mAR:
             type=type(self).__name__,
             value=self.value,
             parameters={
-                "score": self.score,
-                "ious": self.ious,
+                "score_threshold": self.score_threshold,
+                "iou_thresholds": self.iou_thresholds,
                 "label_key": self.label_key,
             },
         )
@@ -238,8 +238,8 @@ class mAR:
 @dataclass
 class ARAveragedOverScores:
     value: float
-    scores: list[float]
-    ious: list[float]
+    score_thresholds: list[float]
+    iou_thresholds: list[float]
     label: tuple[str, str]
 
     @property
@@ -248,8 +248,8 @@ class ARAveragedOverScores:
             type=type(self).__name__,
             value=self.value,
             parameters={
-                "scores": self.scores,
-                "ious": self.ious,
+                "score_thresholds": self.score_thresholds,
+                "iou_thresholds": self.iou_thresholds,
                 "label": {
                     "key": self.label[0],
                     "value": self.label[1],
@@ -264,8 +264,8 @@ class ARAveragedOverScores:
 @dataclass
 class mARAveragedOverScores:
     value: float
-    scores: list[float]
-    ious: list[float]
+    score_thresholds: list[float]
+    iou_thresholds: list[float]
     label_key: str
 
     @property
@@ -274,8 +274,8 @@ class mARAveragedOverScores:
             type=type(self).__name__,
             value=self.value,
             parameters={
-                "scores": self.scores,
-                "ious": self.ious,
+                "score_thresholds": self.score_thresholds,
+                "iou_thresholds": self.iou_thresholds,
                 "label_key": self.label_key,
             },
         )
@@ -291,7 +291,7 @@ class PrecisionRecallCurve:
     """
 
     precision: list[float]
-    iou: float
+    iou_threshold: float
     label: tuple[str, str]
 
     @property
@@ -300,7 +300,7 @@ class PrecisionRecallCurve:
             type=type(self).__name__,
             value=self.precision,
             parameters={
-                "iou": self.iou,
+                "iou_threshold": self.iou_threshold,
                 "label": {"key": self.label[0], "value": self.label[1]},
             },
         )
@@ -310,48 +310,48 @@ class PrecisionRecallCurve:
 
 
 @dataclass
-class DetailedPrecisionRecallPoint:
-    score: float
-    tp: int
-    fp_misclassification: int
-    fp_hallucination: int
-    fn_misclassification: int
-    fn_missing_prediction: int
-    tp_examples: list[str]
-    fp_misclassification_examples: list[str]
-    fp_hallucination_examples: list[str]
-    fn_misclassification_examples: list[str]
-    fn_missing_prediction_examples: list[str]
-
-    def to_dict(self) -> dict:
-        return {
-            "score": self.score,
-            "tp": self.tp,
-            "fp_misclassification": self.fp_misclassification,
-            "fp_hallucination": self.fp_hallucination,
-            "fn_misclassification": self.fn_misclassification,
-            "fn_missing_prediction": self.fn_missing_prediction,
-            "tp_examples": self.tp_examples,
-            "fp_misclassification_examples": self.fp_misclassification_examples,
-            "fp_hallucination_examples": self.fp_hallucination_examples,
-            "fn_misclassification_examples": self.fn_misclassification_examples,
-            "fn_missing_prediction_examples": self.fn_missing_prediction_examples,
-        }
-
-
-@dataclass
-class DetailedPrecisionRecallCurve:
-    iou: float
-    value: list[DetailedPrecisionRecallPoint]
+class DetailedCounts:
+    tp: list[int]
+    fp_misclassification: list[int]
+    fp_hallucination: list[int]
+    fn_misclassification: list[int]
+    fn_missing_prediction: list[int]
+    tp_examples: list[list[str]]
+    fp_misclassification_examples: list[list[str]]
+    fp_hallucination_examples: list[list[str]]
+    fn_misclassification_examples: list[list[str]]
+    fn_missing_prediction_examples: list[list[str]]
+    score_thresholds: list[float]
+    iou_threshold: float
     label: tuple[str, str]
 
-    def to_dict(self) -> dict:
-        return {
-            "value": [pt.to_dict() for pt in self.value],
-            "iou": self.iou,
-            "label": {
-                "key": self.label[0],
-                "value": self.label[1],
+    @property
+    def metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value={
+                "tp": self.tp,
+                "fp_misclassification": self.fp_misclassification,
+                "fp_hallucination": self.fp_hallucination,
+                "fn_misclassification": self.fn_misclassification,
+                "fn_missing_prediction": self.fn_missing_prediction,
+                "tn": None,
+                "tp_examples": self.tp_examples,
+                "fp_misclassification_examples": self.fp_misclassification_examples,
+                "fp_hallucination_examples": self.fp_hallucination_examples,
+                "fn_misclassification_examples": self.fn_misclassification_examples,
+                "fn_missing_prediction_examples": self.fn_missing_prediction_examples,
+                "tn_examples": None,
             },
-            "type": "DetailedPrecisionRecallCurve",
-        }
+            parameters={
+                "score_thresholds": self.score_thresholds,
+                "iou_threshold": self.iou_threshold,
+                "label": {
+                    "key": self.label[0],
+                    "value": self.label[1],
+                },
+            },
+        )
+
+    def to_dict(self) -> dict:
+        return self.metric.to_dict()

{valor_lite-0.32.2a2.dist-info → valor_lite-0.33.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.32.2a2
+Version: 0.33.1
 Summary: Compute valor metrics directly in your client.
 License: MIT License
         

valor_lite-0.33.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
+valor_lite/detection/__init__.py,sha256=WHLHwHoKzXTBjkjC6E1_lhqB7gRWkiGWVWPqkKn-yK8,997
+valor_lite/detection/annotation.py,sha256=ON9iVa33pxysUmZVTCb0wNz-eFX6MDOqDhGDz-ouymc,1466
+valor_lite/detection/computation.py,sha256=2FnVw6_dcAOvwCqpU9bIkeD7gPqDzfW48WSARnvKeOg,18873
+valor_lite/detection/manager.py,sha256=HyODoIkmj92Kfspnpojp1pUY7noAw3FuCgQ36r6vMa4,32356
+valor_lite/detection/metric.py,sha256=hHqClS7c71ztoUnfoaW3T7RmGYaVNU1SlM6vUs1P08I,8809
+valor_lite-0.33.1.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite-0.33.1.dist-info/METADATA,sha256=W36vWkCaas8e0H5RqfGwwlh5FritdeNO7bBj8r-lf6s,1842
+valor_lite-0.33.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+valor_lite-0.33.1.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.33.1.dist-info/RECORD,,

valor_lite-0.32.2a2.dist-info/RECORD

@@ -1,12 +0,0 @@
-valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
-valor_lite/detection/__init__.py,sha256=vkV907Sjx09tgOHpDaLyR_-aFGfu2c1Kpb7hg220vBY,1099
-valor_lite/detection/annotation.py,sha256=ON9iVa33pxysUmZVTCb0wNz-eFX6MDOqDhGDz-ouymc,1466
-valor_lite/detection/computation.py,sha256=VIYZUeBd3KpwCPDBQCKCa0cY0hVb4mq_yGtY2ZP9gGE,16512
-valor_lite/detection/manager.py,sha256=i-C72aQfuakeYFWsERQX-KoOGdGsDMrKMVQsqN82TnY,31527
-valor_lite/detection/metric.py,sha256=wn9JAZMNbUIXUvH2C79jNKJswca1QEyXrCfs7isi1hU,8144
-valor_lite-0.32.2a2.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
-valor_lite-0.32.2a2.dist-info/METADATA,sha256=1Q-jyOmW3wUs4NUYCbvqEt0suQfYRbTSrx5TkkWzbHA,1844
-valor_lite-0.32.2a2.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-valor_lite-0.32.2a2.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
-valor_lite-0.32.2a2.dist-info/RECORD,,