valor-lite 0.33.1__py3-none-any.whl → 0.33.2__py3-none-any.whl

valor_lite/detection/computation.py

@@ -1,16 +1,38 @@
 import numpy as np
 from numpy.typing import NDArray
 
-# datum id  0
-# gt        1
-# pd        2
-# iou       3
-# gt label  4
-# pd label  5
-# score     6
-
 
 def compute_iou(data: NDArray[np.floating]) -> NDArray[np.floating]:
+    """
+    Computes intersection-over-union (IoU) for axis-aligned bounding boxes.
+
+    Takes data with shape (N, 8):
+
+    Index 0 - xmin for Box 1
+    Index 1 - xmax for Box 1
+    Index 2 - ymin for Box 1
+    Index 3 - ymax for Box 1
+    Index 4 - xmin for Box 2
+    Index 5 - xmax for Box 2
+    Index 6 - ymin for Box 2
+    Index 7 - ymax for Box 2
+
+    Returns data with shape (N, 1):
+
+    Index 0 - IoU
+
+    Parameters
+    ----------
+    data : NDArray[np.floating]
+        A sorted array of classification pairs.
+    label_metadata : NDArray[np.int32]
+        An array containing metadata related to labels.
+
+    Returns
+    -------
+    NDArray[np.floating]
+        Compute IoU's.
+    """
 
     xmin1, xmax1, ymin1, ymax1 = (
         data[:, 0],
@@ -93,6 +115,33 @@ def compute_ranked_pairs(
     data: list[NDArray[np.floating]],
     label_metadata: NDArray[np.integer],
 ) -> NDArray[np.floating]:
+    """
+    Performs pair ranking on input data.
+
+    Takes data with shape (N, 7):
+
+    Index 0 - Datum Index
+    Index 1 - GroundTruth Index
+    Index 2 - Prediction Index
+    Index 3 - IoU
+    Index 4 - GroundTruth Label Index
+    Index 5 - Prediction Label Index
+    Index 6 - Score
+
+    Returns data with shape (N - M, 7)
+
+    Parameters
+    ----------
+    data : NDArray[np.floating]
+        A sorted array of classification pairs.
+    label_metadata : NDArray[np.int32]
+        An array containing metadata related to labels.
+
+    Returns
+    -------
+    NDArray[np.floating]
+        A filtered array containing only ranked pairs.
+    """
     pairs = np.concatenate(
         [
             _compute_ranked_pairs_for_datum(
@@ -136,6 +185,27 @@ def compute_metrics(
     """
     Computes Object Detection metrics.
 
+    Takes data with shape (N, 7):
+
+    Index 0 - Datum Index
+    Index 1 - GroundTruth Index
+    Index 2 - Prediction Index
+    Index 3 - IoU
+    Index 4 - GroundTruth Label Index
+    Index 5 - Prediction Label Index
+    Index 6 - Score
+
+    Parameters
+    ----------
+    data : NDArray[np.floating]
+        A sorted array of classification pairs.
+    label_metadata : NDArray[np.int32]
+        An array containing metadata related to labels.
+    iou_thresholds : NDArray[np.floating]
+        A 1-D array containing IoU thresholds.
+    score_thresholds : NDArray[np.floating]
+        A 1-D array containing score thresholds.
+
     Returns
     -------
     tuple[NDArray, NDArray, NDArray NDArray]
@@ -155,7 +225,7 @@ def compute_metrics(
 
     average_precision = np.zeros((n_ious, n_labels))
     average_recall = np.zeros((n_scores, n_labels))
-    precision_recall = np.zeros((n_ious, n_scores, n_labels, 7))
+    counts = np.zeros((n_ious, n_scores, n_labels, 7))
 
     pd_labels = data[:, 5].astype(int)
     unique_pd_labels = np.unique(pd_labels)
@@ -245,7 +315,7 @@ def compute_metrics(
                 out=accuracy,
             )
 
-            precision_recall[iou_idx][score_idx] = np.concatenate(
+            counts[iou_idx][score_idx] = np.concatenate(
                 (
                     tp_count[:, np.newaxis],
                     fp_count[:, np.newaxis],
@@ -353,7 +423,7 @@ def compute_metrics(
     return (
         ap_results,
         ar_results,
-        precision_recall,
+        counts,
         pr_curve,
     )
 
@@ -365,16 +435,49 @@ def compute_detailed_counts(
     score_thresholds: np.ndarray,
     n_samples: int,
 ) -> np.ndarray:
-
     """
-    0  label
-    1  tp
-    ...
-    2  fp - 1
-    3  fp - 2
-    4  fn - misclassification
-    5  fn - hallucination
-    6  tn
+    Compute detailed counts.
+
+    Takes data with shape (N, 7):
+
+    Index 0 - Datum Index
+    Index 1 - GroundTruth Index
+    Index 2 - Prediction Index
+    Index 3 - IoU
+    Index 4 - GroundTruth Label Index
+    Index 5 - Prediction Label Index
+    Index 6 - Score
+
+    Outputs an array with shape (N_IoUs, N_Score, N_Labels, 5 * n_samples + 5):
+
+    Index 0 - True Positive Count
+    ... Datum ID Examples
+    Index n_samples + 1 - False Positive Misclassification Count
+    ... Datum ID Examples
+    Index 2 * n_samples + 2 - False Positive Hallucination Count
+    ... Datum ID Examples
+    Index 3 * n_samples + 3 - False Negative Misclassification Count
+    ... Datum ID Examples
+    Index 4 * n_samples + 4 - False Negative Missing Prediction Count
+    ... Datum ID Examples
+
+    Parameters
+    ----------
+    data : NDArray[np.floating]
+        A sorted array of classification pairs.
+    label_metadata : NDArray[np.int32]
+        An array containing metadata related to labels.
+    iou_thresholds : NDArray[np.floating]
+        A 1-D array containing IoU thresholds.
+    score_thresholds : NDArray[np.floating]
+        A 1-D array containing score thresholds.
+    n_samples : int
+        The number of examples to return per count.
+
+    Returns
+    -------
+    NDArray[np.floating]
+        The detailed counts with optional examples.
     """
 
     n_labels = label_metadata.shape[0]
@@ -466,12 +569,30 @@ def compute_detailed_counts(
                 | mask_groundtruths_without_passing_score
             )
 
-            tp = np.unique(data[mask_tp][:, [0, 2, 5]], axis=0)
+            tp_pds = np.unique(data[mask_tp][:, [0, 2, 5]], axis=0)
+            tp_gts = np.unique(data[mask_tp][:, [0, 1, 4]], axis=0)
             fp_misclf = np.unique(data[mask_fp_misclf][:, [0, 2, 5]], axis=0)
             fp_halluc = np.unique(data[mask_fp_halluc][:, [0, 2, 5]], axis=0)
             fn_misclf = np.unique(data[mask_fn_misclf][:, [0, 1, 4]], axis=0)
             fn_misprd = np.unique(data[mask_fn_misprd][:, [0, 1, 4]], axis=0)
 
+            mask_fp_misclf_is_tp = (
+                (fp_misclf.reshape(-1, 1, 3) == tp_pds.reshape(1, -1, 3))
+                .all(axis=2)
+                .any(axis=1)
+            )
+            mask_fn_misclf_is_tp = (
+                (fn_misclf.reshape(-1, 1, 3) == tp_gts.reshape(1, -1, 3))
+                .all(axis=2)
+                .any(axis=1)
+            )
+
+            tp = tp_pds
+            fp_misclf = fp_misclf[~mask_fp_misclf_is_tp]
+            fp_halluc = fp_halluc
+            fn_misclf = fn_misclf[~mask_fn_misclf_is_tp]
+            fn_misprd = fn_misprd
+
             tp_count = np.bincount(tp[:, 2].astype(int), minlength=n_labels)
             fp_misclf_count = np.bincount(
                 fp_misclf[:, 2].astype(int), minlength=n_labels

valor_lite/detection/manager.py

@@ -58,6 +58,10 @@ class Filter:
 
 
 class Evaluator:
+    """
+    Object Detection Evaluator
+    """
+
     def __init__(self):
 
         # metadata
@@ -87,6 +91,9 @@ class Evaluator:
 
     @property
     def ignored_prediction_labels(self) -> list[tuple[str, str]]:
+        """
+        Prediction labels that are not present in the ground truth set.
+        """
         glabels = set(np.where(self._label_metadata[:, 0] > 0)[0])
         plabels = set(np.where(self._label_metadata[:, 1] > 0)[0])
         return [
@@ -95,6 +102,9 @@ class Evaluator:
 
     @property
     def missing_prediction_labels(self) -> list[tuple[str, str]]:
+        """
+        Ground truth labels that are not present in the prediction set.
+        """
         glabels = set(np.where(self._label_metadata[:, 0] > 0)[0])
         plabels = set(np.where(self._label_metadata[:, 1] > 0)[0])
         return [
@@ -103,6 +113,9 @@ class Evaluator:
 
     @property
     def metadata(self) -> dict:
+        """
+        Evaluation metadata.
+        """
         return {
             "n_datums": self.n_datums,
             "n_groundtruths": self.n_groundtruths,
@@ -216,16 +229,21 @@ class Evaluator:
         filter_: Filter | None = None,
     ) -> dict[MetricType, list]:
         """
-        Runs evaluation over cached data.
+        Performs an evaluation and returns metrics.
 
         Parameters
         ----------
         iou_thresholds : list[float]
-            A list of iou thresholds to compute over.
+            A list of IoU thresholds to compute metrics over.
         score_thresholds : list[float]
-            A list of score thresholds to compute over.
-        filter_mask : NDArray[bool], optional
-            A boolean mask that filters the cached data.
+            A list of score thresholds to compute metrics over.
+        filter_ : Filter, optional
+            An optional filter object.
+
+        Returns
+        -------
+        dict[MetricType, list]
+            A dictionary mapping MetricType enumerations to lists of computed metrics.
         """
 
         data = self._ranked_pairs
@@ -360,6 +378,10 @@ class Evaluator:
         for label_idx, label in self.index_to_label.items():
             for score_idx, score_threshold in enumerate(score_thresholds):
                 for iou_idx, iou_threshold in enumerate(iou_thresholds):
+
+                    if label_metadata[label_idx, 0] == 0:
+                        continue
+
                     row = precision_recall[iou_idx][score_idx][label_idx]
                     kwargs = {
                         "label": label,
@@ -374,6 +396,7 @@ class Evaluator:
                             **kwargs,
                         )
                     )
+
                     metrics[MetricType.Precision].append(
                         Precision(
                             value=row[3],
@@ -532,6 +555,10 @@ class Evaluator:
 
 
 class DataLoader:
+    """
+    Object Detection DataLoader
+    """
+
     def __init__(self):
         self._evaluator = Evaluator()
         self.pairs = list()
@@ -539,6 +566,19 @@ class DataLoader:
         self.prediction_count = defaultdict(lambda: defaultdict(int))
 
     def _add_datum(self, uid: str) -> int:
+        """
+        Helper function for adding a datum to the cache.
+
+        Parameters
+        ----------
+        uid : str
+            The datum uid.
+
+        Returns
+        -------
+        int
+            The datum index.
+        """
         if uid not in self._evaluator.uid_to_index:
             index = len(self._evaluator.uid_to_index)
             self._evaluator.uid_to_index[uid] = index
@@ -546,6 +586,22 @@ class DataLoader:
         return self._evaluator.uid_to_index[uid]
 
     def _add_label(self, label: tuple[str, str]) -> tuple[int, int]:
+        """
+        Helper function for adding a label to the cache.
+
+        Parameters
+        ----------
+        label : tuple[str, str]
+            The label as a tuple in format (key, value).
+
+        Returns
+        -------
+        int
+            Label index.
+        int
+            Label key index.
+        """
+
         label_id = len(self._evaluator.index_to_label)
         label_key_id = len(self._evaluator.index_to_label_key)
         if label not in self._evaluator.label_to_index:
@@ -573,6 +629,16 @@ class DataLoader:
         detections: list[Detection],
         show_progress: bool = False,
     ):
+        """
+        Adds detections to the cache.
+
+        Parameters
+        ----------
+        detections : list[Detection]
+            A list of Detection objects.
+        show_progress : bool, default=False
+            Toggle for tqdm progress bar.
+        """
         disable_tqdm = not show_progress
         for detection in tqdm(detections, disable=disable_tqdm):
 
@@ -688,6 +754,17 @@ class DataLoader:
         detections: list[tuple[dict, dict]],
         show_progress: bool = False,
     ):
+        """
+        Adds Valor-format detections to the cache.
+
+        Parameters
+        ----------
+        detections : list[tuple[dict, dict]]
+            A list of groundtruth, prediction pairs in Valor-format dictionaries.
+        show_progress : bool, default=False
+            Toggle for tqdm progress bar.
+        """
+
         def _get_bbox_extrema(
             data: list[list[list[float]]],
         ) -> tuple[float, float, float, float]:
@@ -809,6 +886,14 @@ class DataLoader:
             self.pairs.append(np.array(pairs))
 
     def finalize(self) -> Evaluator:
+        """
+        Performs data finalization and some preprocessing steps.
+
+        Returns
+        -------
+        Evaluator
+            A ready-to-use evaluator object.
+        """
 
         self.pairs = [pair for pair in self.pairs if pair.size > 0]
         if len(self.pairs) == 0:

{valor_lite-0.33.1.dist-info → valor_lite-0.33.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.1
+Version: 0.33.2
 Summary: Compute valor metrics directly in your client.
 License: MIT License
         

valor_lite-0.33.2.dist-info/RECORD

@@ -0,0 +1,12 @@
+valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
+valor_lite/detection/__init__.py,sha256=WHLHwHoKzXTBjkjC6E1_lhqB7gRWkiGWVWPqkKn-yK8,997
+valor_lite/detection/annotation.py,sha256=ON9iVa33pxysUmZVTCb0wNz-eFX6MDOqDhGDz-ouymc,1466
+valor_lite/detection/computation.py,sha256=L8FIwZ-qxOQnoT7mxgNzLyNyI-Bvga0i-gtbow3hN-o,22575
+valor_lite/detection/manager.py,sha256=Y45Wy3PWi7dQ0VnDERdtpOixUbKVXTZxBcCR92ny0QY,34278
+valor_lite/detection/metric.py,sha256=hHqClS7c71ztoUnfoaW3T7RmGYaVNU1SlM6vUs1P08I,8809
+valor_lite-0.33.2.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite-0.33.2.dist-info/METADATA,sha256=fe-Sj568DB-E9cyC5P8GA_lLjmM1t3MZUHj1f0JF6fM,1842
+valor_lite-0.33.2.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+valor_lite-0.33.2.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.33.2.dist-info/RECORD,,

valor_lite-0.33.1.dist-info/RECORD

@@ -1,12 +0,0 @@
-valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
-valor_lite/detection/__init__.py,sha256=WHLHwHoKzXTBjkjC6E1_lhqB7gRWkiGWVWPqkKn-yK8,997
-valor_lite/detection/annotation.py,sha256=ON9iVa33pxysUmZVTCb0wNz-eFX6MDOqDhGDz-ouymc,1466
-valor_lite/detection/computation.py,sha256=2FnVw6_dcAOvwCqpU9bIkeD7gPqDzfW48WSARnvKeOg,18873
-valor_lite/detection/manager.py,sha256=HyODoIkmj92Kfspnpojp1pUY7noAw3FuCgQ36r6vMa4,32356
-valor_lite/detection/metric.py,sha256=hHqClS7c71ztoUnfoaW3T7RmGYaVNU1SlM6vUs1P08I,8809
-valor_lite-0.33.1.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
-valor_lite-0.33.1.dist-info/METADATA,sha256=W36vWkCaas8e0H5RqfGwwlh5FritdeNO7bBj8r-lf6s,1842
-valor_lite-0.33.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-valor_lite-0.33.1.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
-valor_lite-0.33.1.dist-info/RECORD,,