valor-lite 0.33.7__py3-none-any.whl → 0.33.8__py3-none-any.whl

valor_lite/detection/metric.py

@@ -46,7 +46,7 @@ class Counts:
     tp: int
     fp: int
     fn: int
-    label: tuple[str, str]
+    label: str
     iou_threshold: float
     score_threshold: float
 
@@ -62,10 +62,7 @@ class Counts:
             parameters={
                 "iou_threshold": self.iou_threshold,
                 "score_threshold": self.score_threshold,
-                "label": {
-                    "key": self.label[0],
-                    "value": self.label[1],
-                },
+                "label": self.label,
             },
         )
 
@@ -76,7 +73,7 @@ class Counts:
 @dataclass
 class ClassMetric:
     value: float
-    label: tuple[str, str]
+    label: str
     iou_threshold: float
     score_threshold: float
 
@@ -88,10 +85,7 @@ class ClassMetric:
             parameters={
                 "iou_threshold": self.iou_threshold,
                 "score_threshold": self.score_threshold,
-                "label": {
-                    "key": self.label[0],
-                    "value": self.label[1],
-                },
+                "label": self.label,
             },
         )
 
@@ -119,7 +113,7 @@ class F1(ClassMetric):
 class AP:
     value: float
     iou_threshold: float
-    label: tuple[str, str]
+    label: str
 
     @property
     def metric(self) -> Metric:
@@ -128,10 +122,7 @@ class AP:
             value=self.value,
             parameters={
                 "iou_threshold": self.iou_threshold,
-                "label": {
-                    "key": self.label[0],
-                    "value": self.label[1],
-                },
+                "label": self.label,
             },
         )
 
@@ -143,7 +134,6 @@ class AP:
 class mAP:
     value: float
     iou_threshold: float
-    label_key: str
 
     @property
     def metric(self) -> Metric:
@@ -152,7 +142,6 @@ class mAP:
             value=self.value,
             parameters={
                 "iou_threshold": self.iou_threshold,
-                "label_key": self.label_key,
             },
         )
 
@@ -164,7 +153,7 @@ class mAP:
 class APAveragedOverIOUs:
     value: float
     iou_thresholds: list[float]
-    label: tuple[str, str]
+    label: str
 
     @property
     def metric(self) -> Metric:
@@ -173,10 +162,7 @@ class APAveragedOverIOUs:
             value=self.value,
             parameters={
                 "iou_thresholds": self.iou_thresholds,
-                "label": {
-                    "key": self.label[0],
-                    "value": self.label[1],
-                },
+                "label": self.label,
             },
         )
 
@@ -188,7 +174,6 @@ class APAveragedOverIOUs:
 class mAPAveragedOverIOUs:
     value: float
     iou_thresholds: list[float]
-    label_key: str
 
     @property
     def metric(self) -> Metric:
@@ -197,7 +182,6 @@ class mAPAveragedOverIOUs:
             value=self.value,
             parameters={
                 "iou_thresholds": self.iou_thresholds,
-                "label_key": self.label_key,
             },
         )
 
@@ -210,7 +194,7 @@ class AR:
     value: float
     score_threshold: float
     iou_thresholds: list[float]
-    label: tuple[str, str]
+    label: str
 
     @property
     def metric(self) -> Metric:
@@ -220,10 +204,7 @@ class AR:
             parameters={
                 "score_threshold": self.score_threshold,
                 "iou_thresholds": self.iou_thresholds,
-                "label": {
-                    "key": self.label[0],
-                    "value": self.label[1],
-                },
+                "label": self.label,
             },
         )
 
@@ -236,7 +217,6 @@ class mAR:
     value: float
     score_threshold: float
     iou_thresholds: list[float]
-    label_key: str
 
     @property
     def metric(self) -> Metric:
@@ -246,7 +226,6 @@ class mAR:
             parameters={
                 "score_threshold": self.score_threshold,
                 "iou_thresholds": self.iou_thresholds,
-                "label_key": self.label_key,
             },
         )
 
@@ -259,7 +238,7 @@ class ARAveragedOverScores:
     value: float
     score_thresholds: list[float]
     iou_thresholds: list[float]
-    label: tuple[str, str]
+    label: str
 
     @property
     def metric(self) -> Metric:
@@ -269,10 +248,7 @@ class ARAveragedOverScores:
             parameters={
                 "score_thresholds": self.score_thresholds,
                 "iou_thresholds": self.iou_thresholds,
-                "label": {
-                    "key": self.label[0],
-                    "value": self.label[1],
-                },
+                "label": self.label,
             },
         )
 
@@ -285,7 +261,6 @@ class mARAveragedOverScores:
     value: float
     score_thresholds: list[float]
     iou_thresholds: list[float]
-    label_key: str
 
     @property
     def metric(self) -> Metric:
@@ -295,7 +270,6 @@ class mARAveragedOverScores:
             parameters={
                 "score_thresholds": self.score_thresholds,
                 "iou_thresholds": self.iou_thresholds,
-                "label_key": self.label_key,
             },
         )
 
@@ -311,7 +285,7 @@ class PrecisionRecallCurve:
 
     precision: list[float]
     iou_threshold: float
-    label: tuple[str, str]
+    label: str
 
     @property
     def metric(self) -> Metric:
@@ -320,7 +294,7 @@ class PrecisionRecallCurve:
             value=self.precision,
             parameters={
                 "iou_threshold": self.iou_threshold,
-                "label": {"key": self.label[0], "value": self.label[1]},
+                "label": self.label,
             },
         )
 
@@ -341,8 +315,8 @@ class ConfusionMatrix:
                     dict[
                         str,  # either `datum`, `groundtruth`, `prediction` or score
                         str  # datum uid
-                        | tuple[
-                            float, float, float, float
+                        | dict[
+                            str, float
                         ]  # bounding box (xmin, xmax, ymin, ymax)
                         | float,  # prediction score
                     ]
@@ -360,8 +334,8 @@ class ConfusionMatrix:
                     str,  # either `datum`, `prediction` or score
                     str  # datum uid
                     | float  # prediction score
-                    | tuple[
-                        float, float, float, float
+                    | dict[
+                        str, float
                     ],  # bounding box (xmin, xmax, ymin, ymax)
                 ]
             ],
@@ -376,8 +350,8 @@ class ConfusionMatrix:
                 dict[
                     str,  # either `datum` or `groundtruth`
                     str  # datum uid
-                    | tuple[
-                        float, float, float, float
+                    | dict[
+                        str, float
                     ],  # bounding box (xmin, xmax, ymin, ymax)
                 ]
             ],
@@ -385,7 +359,6 @@ class ConfusionMatrix:
     ]
     score_threshold: float
     iou_threshold: float
-    label_key: str
     number_of_examples: int
 
     @property
@@ -400,7 +373,6 @@ class ConfusionMatrix:
             parameters={
                 "score_threshold": self.score_threshold,
                 "iou_threshold": self.iou_threshold,
-                "label_key": self.label_key,
             },
         )
 

valor_lite/segmentation/__init__.py

@@ -0,0 +1,27 @@
+from .annotation import Bitmask, Segmentation
+from .manager import DataLoader, Evaluator
+from .metric import (
+    F1,
+    Accuracy,
+    ConfusionMatrix,
+    IoU,
+    MetricType,
+    Precision,
+    Recall,
+    mIoU,
+)
+
+__all__ = [
+    "DataLoader",
+    "Evaluator",
+    "Segmentation",
+    "Bitmask",
+    "MetricType",
+    "Precision",
+    "Recall",
+    "Accuracy",
+    "F1",
+    "IoU",
+    "mIoU",
+    "ConfusionMatrix",
+]

valor_lite/segmentation/annotation.py

@@ -0,0 +1,49 @@
+from dataclasses import dataclass, field
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+@dataclass
+class Bitmask:
+    mask: NDArray[np.bool_]
+    label: str
+
+    def __post_init__(self):
+        if self.mask.dtype != np.bool_:
+            raise ValueError(
+                f"Bitmask recieved mask with dtype `{self.mask.dtype}`."
+            )
+
+
+@dataclass
+class Segmentation:
+    uid: str
+    groundtruths: list[Bitmask]
+    predictions: list[Bitmask]
+    shape: tuple[int, ...] = field(default_factory=lambda: (0, 0))
+    size: int = field(default=0)
+
+    def __post_init__(self):
+
+        groundtruth_shape = {
+            groundtruth.mask.shape for groundtruth in self.groundtruths
+        }
+        prediction_shape = {
+            prediction.mask.shape for prediction in self.predictions
+        }
+        if len(groundtruth_shape) == 0:
+            raise ValueError("The segmenation is missing ground truths.")
+        elif len(prediction_shape) == 0:
+            raise ValueError("The segmenation is missing predictions.")
+        elif (
+            len(groundtruth_shape) != 1
+            or len(prediction_shape) != 1
+            or groundtruth_shape != prediction_shape
+        ):
+            raise ValueError(
+                "A shape mismatch exists within the segmentation."
+            )
+
+        self.shape = groundtruth_shape.pop()
+        self.size = int(np.prod(np.array(self.shape)))

valor_lite/segmentation/computation.py

@@ -0,0 +1,186 @@
+import numpy as np
+from numpy.typing import NDArray
+
+
+def compute_intermediate_confusion_matrices(
+    groundtruths: NDArray[np.bool_],
+    predictions: NDArray[np.bool_],
+    groundtruth_labels: NDArray[np.int32],
+    prediction_labels: NDArray[np.int32],
+    n_labels: int,
+) -> NDArray[np.int32]:
+    """
+    Computes an intermediate confusion matrix containing label counts.
+
+    Parameters
+    ----------
+    groundtruths : NDArray[np.bool_]
+        A 2-D array containing flattened bitmasks for each label.
+    predictions : NDArray[np.bool_]
+        A 2-D array containing flattened bitmasks for each label.
+    groundtruth_labels : NDArray[np.int32]
+        A 1-D array containing label indices.
+    groundtruth_labels : NDArray[np.int32]
+        A 1-D array containing label indices.
+    n_labels : int
+        The number of unique labels.
+
+    Returns
+    -------
+    NDArray[np.int32]
+        A 2-D confusion matrix with shape (n_labels + 1, n_labels + 1).
+    """
+
+    n_gt_labels = groundtruth_labels.size
+    n_pd_labels = prediction_labels.size
+
+    groundtruth_counts = groundtruths.sum(axis=1)
+    prediction_counts = predictions.sum(axis=1)
+
+    background_counts = np.logical_not(
+        groundtruths.any(axis=0) | predictions.any(axis=0)
+    ).sum()
+
+    intersection_counts = np.logical_and(
+        groundtruths.reshape(n_gt_labels, 1, -1),
+        predictions.reshape(1, n_pd_labels, -1),
+    ).sum(axis=2)
+
+    intersected_groundtruth_counts = intersection_counts.sum(axis=0)
+    intersected_prediction_counts = intersection_counts.sum(axis=1)
+
+    confusion_matrix = np.zeros((n_labels + 1, n_labels + 1), dtype=np.int32)
+    confusion_matrix[0, 0] = background_counts
+    for gidx in range(n_gt_labels):
+        gt_label_idx = groundtruth_labels[gidx]
+        for pidx in range(n_pd_labels):
+            pd_label_idx = prediction_labels[pidx]
+            confusion_matrix[
+                gt_label_idx + 1,
+                pd_label_idx + 1,
+            ] = intersection_counts[gidx, pidx]
+
+            if gidx == 0:
+                confusion_matrix[0, pd_label_idx + 1] = (
+                    prediction_counts[pidx]
+                    - intersected_prediction_counts[pidx]
+                )
+
+        confusion_matrix[gt_label_idx + 1, 0] = (
+            groundtruth_counts[gidx] - intersected_groundtruth_counts[gidx]
+        )
+
+    return confusion_matrix
+
+
+def compute_metrics(
+    data: NDArray[np.float64],
+    label_metadata: NDArray[np.int32],
+    n_pixels: int,
+) -> tuple[
+    NDArray[np.float64],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    float,
+    NDArray[np.float64],
+    NDArray[np.float64],
+    NDArray[np.float64],
+]:
+    """
+    Computes semantic segmentation metrics.
+
+    Takes data with shape (3, N).
+
+    Parameters
+    ----------
+    data : NDArray[np.float64]
+        A 3-D array containing confusion matrices for each datum.
+    label_metadata : NDArray[np.int32]
+        A 2-D array containing label metadata.
+
+    Returns
+    -------
+    NDArray[np.float64]
+        Precision.
+    NDArray[np.float64]
+        Recall.
+    NDArray[np.float64]
+        F1 Score.
+    float
+        Accuracy
+    NDArray[np.float64]
+        Confusion matrix containing IoU values.
+    NDArray[np.float64]
+        Hallucination ratios.
+    NDArray[np.float64]
+        Missing prediction ratios.
+    """
+    n_labels = label_metadata.shape[0]
+    gt_counts = label_metadata[:, 0]
+    pd_counts = label_metadata[:, 1]
+
+    counts = data.sum(axis=0)
+
+    # compute iou, missing_predictions and hallucinations
+    intersection_ = counts[1:, 1:]
+    union_ = (
+        gt_counts[:, np.newaxis] + pd_counts[np.newaxis, :] - intersection_
+    )
+
+    ious = np.zeros((n_labels, n_labels), dtype=np.float64)
+    np.divide(
+        intersection_,
+        union_,
+        where=union_ > 1e-9,
+        out=ious,
+    )
+
+    hallucination_ratio = np.zeros((n_labels), dtype=np.float64)
+    np.divide(
+        counts[0, 1:],
+        pd_counts,
+        where=pd_counts > 1e-9,
+        out=hallucination_ratio,
+    )
+
+    missing_prediction_ratio = np.zeros((n_labels), dtype=np.float64)
+    np.divide(
+        counts[1:, 0],
+        gt_counts,
+        where=gt_counts > 1e-9,
+        out=missing_prediction_ratio,
+    )
+
+    # compute precision, recall, f1
+    tp_counts = counts.diagonal()[1:]
+
+    precision = np.zeros(n_labels, dtype=np.float64)
+    np.divide(tp_counts, pd_counts, where=pd_counts > 1e-9, out=precision)
+
+    recall = np.zeros_like(precision)
+    np.divide(tp_counts, gt_counts, where=gt_counts > 1e-9, out=recall)
+
+    f1_score = np.zeros_like(precision)
+    np.divide(
+        2 * (precision * recall),
+        (precision + recall),
+        where=(precision + recall) > 0,
+        out=f1_score,
+    )
+
+    # compute accuracy
+    tp_count = counts[1:, 1:].diagonal().sum()
+    background_count = counts[0, 0]
+    accuracy = (
+        (tp_count + background_count) / n_pixels if n_pixels > 0 else 0.0
+    )
+
+    return (
+        precision,
+        recall,
+        f1_score,
+        accuracy,
+        ious,
+        hallucination_ratio,
+        missing_prediction_ratio,
+    )