valor-lite 0.33.12__py3-none-any.whl → 0.33.14__py3-none-any.whl

valor_lite/classification/utilities.py

@@ -0,0 +1,222 @@
+from collections import defaultdict
+
+import numpy as np
+from numpy.typing import NDArray
+from valor_lite.classification.metric import Metric, MetricType
+
+
+def unpack_precision_recall_rocauc_into_metric_lists(
+    results: tuple[
+        NDArray[np.int32],
+        NDArray[np.float64],
+        NDArray[np.float64],
+        NDArray[np.float64],
+        NDArray[np.float64],
+        NDArray[np.float64],
+        float,
+    ],
+    score_thresholds: list[float],
+    hardmax: bool,
+    label_metadata: NDArray[np.int32],
+    index_to_label: dict[int, str],
+) -> dict[MetricType, list[Metric]]:
+    (
+        counts,
+        precision,
+        recall,
+        accuracy,
+        f1_score,
+        rocauc,
+        mean_rocauc,
+    ) = results
+
+    metrics = defaultdict(list)
+
+    metrics[MetricType.ROCAUC] = [
+        Metric.roc_auc(
+            value=float(rocauc[label_idx]),
+            label=label,
+        )
+        for label_idx, label in index_to_label.items()
+        if label_metadata[label_idx, 0] > 0
+    ]
+
+    metrics[MetricType.mROCAUC] = [
+        Metric.mean_roc_auc(
+            value=float(mean_rocauc),
+        )
+    ]
+
+    metrics[MetricType.Accuracy] = [
+        Metric.accuracy(
+            value=float(accuracy[score_idx]),
+            score_threshold=score_threshold,
+            hardmax=hardmax,
+        )
+        for score_idx, score_threshold in enumerate(score_thresholds)
+    ]
+
+    for label_idx, label in index_to_label.items():
+        for score_idx, score_threshold in enumerate(score_thresholds):
+
+            kwargs = {
+                "label": label,
+                "hardmax": hardmax,
+                "score_threshold": score_threshold,
+            }
+            row = counts[:, label_idx]
+            metrics[MetricType.Counts].append(
+                Metric.counts(
+                    tp=int(row[score_idx, 0]),
+                    fp=int(row[score_idx, 1]),
+                    fn=int(row[score_idx, 2]),
+                    tn=int(row[score_idx, 3]),
+                    **kwargs,
+                )
+            )
+
+            # if no groundtruths exists for a label, skip it.
+            if label_metadata[label_idx, 0] == 0:
+                continue
+
+            metrics[MetricType.Precision].append(
+                Metric.precision(
+                    value=float(precision[score_idx, label_idx]),
+                    **kwargs,
+                )
+            )
+            metrics[MetricType.Recall].append(
+                Metric.recall(
+                    value=float(recall[score_idx, label_idx]),
+                    **kwargs,
+                )
+            )
+            metrics[MetricType.F1].append(
+                Metric.f1_score(
+                    value=float(f1_score[score_idx, label_idx]),
+                    **kwargs,
+                )
+            )
+    return metrics
+
+
+def _unpack_confusion_matrix_value(
+    confusion_matrix: NDArray[np.float64],
+    number_of_labels: int,
+    number_of_examples: int,
+    index_to_uid: dict[int, str],
+    index_to_label: dict[int, str],
+) -> dict[str, dict[str, dict[str, int | list[dict[str, str | float]]]]]:
+    """
+    Unpacks a numpy array of confusion matrix counts and examples.
+    """
+
+    datum_idx = lambda gt_label_idx, pd_label_idx, example_idx: int(  # noqa: E731 - lambda fn
+        confusion_matrix[
+            gt_label_idx,
+            pd_label_idx,
+            example_idx * 2 + 1,
+        ]
+    )
+
+    score_idx = lambda gt_label_idx, pd_label_idx, example_idx: float(  # noqa: E731 - lambda fn
+        confusion_matrix[
+            gt_label_idx,
+            pd_label_idx,
+            example_idx * 2 + 2,
+        ]
+    )
+
+    return {
+        index_to_label[gt_label_idx]: {
+            index_to_label[pd_label_idx]: {
+                "count": max(
+                    int(confusion_matrix[gt_label_idx, pd_label_idx, 0]),
+                    0,
+                ),
+                "examples": [
+                    {
+                        "datum": index_to_uid[
+                            datum_idx(gt_label_idx, pd_label_idx, example_idx)
+                        ],
+                        "score": score_idx(
+                            gt_label_idx, pd_label_idx, example_idx
+                        ),
+                    }
+                    for example_idx in range(number_of_examples)
+                    if datum_idx(gt_label_idx, pd_label_idx, example_idx) >= 0
+                ],
+            }
+            for pd_label_idx in range(number_of_labels)
+        }
+        for gt_label_idx in range(number_of_labels)
+    }
+
+
+def _unpack_missing_predictions_value(
+    missing_predictions: NDArray[np.int32],
+    number_of_labels: int,
+    number_of_examples: int,
+    index_to_uid: dict[int, str],
+    index_to_label: dict[int, str],
+) -> dict[str, dict[str, int | list[dict[str, str]]]]:
+    """
+    Unpacks a numpy array of missing prediction counts and examples.
+    """
+
+    datum_idx = (
+        lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
+            missing_predictions[
+                gt_label_idx,
+                example_idx + 1,
+            ]
+        )
+    )
+
+    return {
+        index_to_label[gt_label_idx]: {
+            "count": max(
+                int(missing_predictions[gt_label_idx, 0]),
+                0,
+            ),
+            "examples": [
+                {"datum": index_to_uid[datum_idx(gt_label_idx, example_idx)]}
+                for example_idx in range(number_of_examples)
+                if datum_idx(gt_label_idx, example_idx) >= 0
+            ],
+        }
+        for gt_label_idx in range(number_of_labels)
+    }
+
+
+def unpack_confusion_matrix_into_metric_list(
+    results: tuple[NDArray[np.float64], NDArray[np.int32]],
+    score_thresholds: list[float],
+    number_of_examples: int,
+    index_to_uid: dict[int, str],
+    index_to_label: dict[int, str],
+) -> list[Metric]:
+
+    (confusion_matrix, missing_predictions) = results
+    n_scores, n_labels, _, _ = confusion_matrix.shape
+    return [
+        Metric.confusion_matrix(
+            score_threshold=score_threshold,
+            maximum_number_of_examples=number_of_examples,
+            confusion_matrix=_unpack_confusion_matrix_value(
+                confusion_matrix=confusion_matrix[score_idx, :, :, :],
+                number_of_labels=n_labels,
+                number_of_examples=number_of_examples,
+                index_to_label=index_to_label,
+                index_to_uid=index_to_uid,
+            ),
+            missing_predictions=_unpack_missing_predictions_value(
+                missing_predictions=missing_predictions[score_idx, :, :],
+                number_of_labels=n_labels,
+                number_of_examples=number_of_examples,
+                index_to_label=index_to_label,
+                index_to_uid=index_to_uid,
+            ),
+        )
+        for score_idx, score_threshold in enumerate(score_thresholds)
+    ]

valor_lite/object_detection/__init__.py

@@ -3,56 +3,25 @@ from .computation import (
     compute_bbox_iou,
     compute_bitmask_iou,
     compute_confusion_matrix,
-    compute_metrics,
     compute_polygon_iou,
+    compute_precion_recall,
     compute_ranked_pairs,
 )
 from .manager import DataLoader, Evaluator
-from .metric import (
-    AP,
-    AR,
-    F1,
-    Accuracy,
-    APAveragedOverIOUs,
-    ARAveragedOverScores,
-    ConfusionMatrix,
-    Counts,
-    MetricType,
-    Precision,
-    PrecisionRecallCurve,
-    Recall,
-    mAP,
-    mAPAveragedOverIOUs,
-    mAR,
-    mARAveragedOverScores,
-)
+from .metric import Metric, MetricType
 
 __all__ = [
     "Bitmask",
     "BoundingBox",
     "Detection",
     "Polygon",
+    "Metric",
     "MetricType",
-    "Counts",
-    "Precision",
-    "Recall",
-    "Accuracy",
-    "F1",
-    "AP",
-    "mAP",
-    "AR",
-    "mAR",
-    "APAveragedOverIOUs",
-    "mAPAveragedOverIOUs",
-    "ARAveragedOverScores",
-    "mARAveragedOverScores",
-    "PrecisionRecallCurve",
-    "ConfusionMatrix",
     "compute_bbox_iou",
     "compute_bitmask_iou",
     "compute_polygon_iou",
     "compute_ranked_pairs",
-    "compute_metrics",
+    "compute_precion_recall",
     "compute_confusion_matrix",
     "DataLoader",
     "Evaluator",

valor_lite/object_detection/computation.py

@@ -5,7 +5,7 @@ from numpy.typing import NDArray
 
 def compute_bbox_iou(data: NDArray[np.float64]) -> NDArray[np.float64]:
     """
-    Computes intersection-over-union (IoU) for axis-aligned bounding boxes.
+    Computes intersection-over-union (IOU) for axis-aligned bounding boxes.
 
     Takes data with shape (N, 8):
 
@@ -20,7 +20,7 @@ def compute_bbox_iou(data: NDArray[np.float64]) -> NDArray[np.float64]:
 
     Returns data with shape (N, 1):
 
-    Index 0 - IoU
+    Index 0 - IOU
 
     Parameters
     ----------
@@ -30,7 +30,7 @@ def compute_bbox_iou(data: NDArray[np.float64]) -> NDArray[np.float64]:
     Returns
     -------
     NDArray[np.float64]
-        Computed IoU's.
+        Computed IOU's.
     """
     if data.size == 0:
         return np.array([], dtype=np.float64)
@@ -76,7 +76,7 @@ def compute_bbox_iou(data: NDArray[np.float64]) -> NDArray[np.float64]:
 
 def compute_bitmask_iou(data: NDArray[np.bool_]) -> NDArray[np.float64]:
     """
-    Computes intersection-over-union (IoU) for bitmasks.
+    Computes intersection-over-union (IOU) for bitmasks.
 
     Takes data with shape (N, 2):
 
@@ -85,7 +85,7 @@ def compute_bitmask_iou(data: NDArray[np.bool_]) -> NDArray[np.float64]:
 
     Returns data with shape (N, 1):
 
-    Index 0 - IoU
+    Index 0 - IOU
 
     Parameters
     ----------
@@ -95,7 +95,7 @@ def compute_bitmask_iou(data: NDArray[np.bool_]) -> NDArray[np.float64]:
     Returns
     -------
     NDArray[np.float64]
-        Computed IoU's.
+        Computed IOU's.
     """
 
     if data.size == 0:
@@ -125,7 +125,7 @@ def compute_polygon_iou(
     data: NDArray[np.float64],
 ) -> NDArray[np.float64]:
     """
-    Computes intersection-over-union (IoU) for shapely polygons.
+    Computes intersection-over-union (IOU) for shapely polygons.
 
     Takes data with shape (N, 2):
 
@@ -134,7 +134,7 @@ def compute_polygon_iou(
 
     Returns data with shape (N, 1):
 
-    Index 0 - IoU
+    Index 0 - IOU
 
     Parameters
     ----------
@@ -144,7 +144,7 @@ def compute_polygon_iou(
     Returns
     -------
     NDArray[np.float64]
-        Computed IoU's.
+        Computed IOU's.
     """
 
     if data.size == 0:
@@ -225,7 +225,7 @@ def compute_ranked_pairs(
     Index 0 - Datum Index
     Index 1 - GroundTruth Index
     Index 2 - Prediction Index
-    Index 3 - IoU
+    Index 3 - IOU
     Index 4 - GroundTruth Label Index
     Index 5 - Prediction Label Index
     Index 6 - Score
@@ -262,7 +262,7 @@ def compute_ranked_pairs(
     return ranked_pairs[indices]
 
 
-def compute_metrics(
+def compute_precion_recall(
     data: NDArray[np.float64],
     label_metadata: NDArray[np.int32],
     iou_thresholds: NDArray[np.float64],
@@ -282,6 +282,7 @@ def compute_metrics(
     ],
     NDArray[np.float64],
     NDArray[np.float64],
+    NDArray[np.float64],
 ]:
     """
     Computes Object Detection metrics.
@@ -291,7 +292,7 @@ def compute_metrics(
     Index 0 - Datum Index
     Index 1 - GroundTruth Index
     Index 2 - Prediction Index
-    Index 3 - IoU
+    Index 3 - IOU
     Index 4 - GroundTruth Label Index
     Index 5 - Prediction Label Index
     Index 6 - Score
@@ -303,19 +304,21 @@ def compute_metrics(
     label_metadata : NDArray[np.int32]
         An array containing metadata related to labels.
     iou_thresholds : NDArray[np.float64]
-        A 1-D array containing IoU thresholds.
+        A 1-D array containing IOU thresholds.
     score_thresholds : NDArray[np.float64]
         A 1-D array containing score thresholds.
 
     Returns
     -------
-    tuple[NDArray, NDArray, NDArray, float]
+    tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64], float]
         Average Precision results.
-    tuple[NDArray, NDArray, NDArray, float]
+    tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64], float]
         Average Recall results.
-    np.ndarray
-        Precision, Recall, TP, FP, FN, F1 Score, Accuracy.
-    np.ndarray
+    NDArray[np.float64]
+        Accuracy.
+    NDArray[np.float64]
+        Precision, Recall, TP, FP, FN, F1 Score.
+    NDArray[np.float64]
         Interpolated Precision-Recall Curves.
     """
 
@@ -325,13 +328,14 @@ def compute_metrics(
     n_scores = score_thresholds.shape[0]
 
     if n_ious == 0:
-        raise ValueError("At least one IoU threshold must be passed.")
+        raise ValueError("At least one IOU threshold must be passed.")
     elif n_scores == 0:
         raise ValueError("At least one score threshold must be passed.")
 
-    average_precision = np.zeros((n_ious, n_labels))
-    average_recall = np.zeros((n_scores, n_labels))
-    counts = np.zeros((n_ious, n_scores, n_labels, 7))
+    average_precision = np.zeros((n_ious, n_labels), dtype=np.float64)
+    average_recall = np.zeros((n_scores, n_labels), dtype=np.float64)
+    accuracy = np.zeros((n_ious, n_scores), dtype=np.float64)
+    counts = np.zeros((n_ious, n_scores, n_labels, 6), dtype=np.float64)
 
     pd_labels = data[:, 5].astype(np.int32)
     scores = data[:, 6]
@@ -417,14 +421,6 @@ def compute_metrics(
                 out=f1_score,
             )
 
-            accuracy = np.zeros_like(tp_count)
-            np.divide(
-                tp_count,
-                (gt_count + pd_count),
-                where=(gt_count + pd_count) > 1e-9,
-                out=accuracy,
-            )
-
             counts[iou_idx][score_idx] = np.concatenate(
                 (
                     tp_count[:, np.newaxis],
@@ -433,11 +429,18 @@ def compute_metrics(
                     precision[:, np.newaxis],
                     recall[:, np.newaxis],
                     f1_score[:, np.newaxis],
-                    accuracy[:, np.newaxis],
                 ),
                 axis=1,
             )
 
+            # caluculate accuracy
+            total_pd_count = label_metadata[:, 1].sum()
+            accuracy[iou_idx, score_idx] = (
+                (tp_count.sum() / total_pd_count)
+                if total_pd_count > 1e-9
+                else 0.0
+            )
+
             # calculate recall for AR
             average_recall[score_idx] += recall
 
@@ -529,18 +532,18 @@ def compute_metrics(
         mAR = np.zeros(n_scores, dtype=np.float64)
 
     # calculate AR and AR averaged over thresholds
-    APAveragedOverIoUs = average_precision.mean(axis=0)
+    APAveragedOverIOUs = average_precision.mean(axis=0)
     ARAveragedOverScores = average_recall.mean(axis=0)
 
     # calculate mAP and mAR averaged over thresholds
-    mAPAveragedOverIoUs = mAP.mean(axis=0)
+    mAPAveragedOverIOUs = mAP.mean(axis=0)
     mARAveragedOverScores = mAR.mean(axis=0)
 
     ap_results = (
         average_precision,
         mAP,
-        APAveragedOverIoUs,
-        mAPAveragedOverIoUs,
+        APAveragedOverIOUs,
+        mAPAveragedOverIOUs,
     )
     ar_results = (
         average_recall,
@@ -552,6 +555,7 @@ def compute_metrics(
     return (
         ap_results,
         ar_results,
+        accuracy,
         counts,
         pr_curve,
     )
@@ -639,7 +643,7 @@ def compute_confusion_matrix(
     Index 0 - Datum Index
     Index 1 - GroundTruth Index
     Index 2 - Prediction Index
-    Index 3 - IoU
+    Index 3 - IOU
     Index 4 - GroundTruth Label Index
     Index 5 - Prediction Label Index
     Index 6 - Score
@@ -651,7 +655,7 @@ def compute_confusion_matrix(
     label_metadata : NDArray[np.int32]
         An array containing metadata related to labels.
     iou_thresholds : NDArray[np.float64]
-        A 1-D array containing IoU thresholds.
+        A 1-D array containing IOU thresholds.
     score_thresholds : NDArray[np.float64]
         A 1-D array containing score thresholds.
     n_examples : int