valor-lite 0.33.6__py3-none-any.whl → 0.33.8__py3-none-any.whl

valor_lite/classification/annotation.py

@@ -4,10 +4,14 @@ from dataclasses import dataclass
 @dataclass
 class Classification:
     uid: str
-    groundtruths: list[tuple[str, str]]
-    predictions: list[tuple[str, str]]
+    groundtruth: str
+    predictions: list[str]
     scores: list[float]
 
     def __post_init__(self):
+        if not isinstance(self.groundtruth, str):
+            raise ValueError(
+                "A classification must contain a single groundtruth."
+            )
         if len(self.predictions) != len(self.scores):
             raise ValueError("There must be a score per prediction label.")

valor_lite/classification/computation.py

@@ -3,28 +3,18 @@ from numpy.typing import NDArray
 
 
 def _compute_rocauc(
-    data: NDArray[np.floating],
+    data: NDArray[np.float64],
     label_metadata: NDArray[np.int32],
     n_datums: int,
     n_labels: int,
-    n_label_keys: int,
     mask_matching_labels: NDArray[np.bool_],
     pd_labels: NDArray[np.int32],
 ):
     """
     Compute ROCAUC and mean ROCAUC.
     """
-    count_labels_per_key = np.bincount(label_metadata[:, 2])
-    count_groundtruths_per_key = np.bincount(
-        label_metadata[:, 2],
-        weights=label_metadata[:, 0],
-        minlength=n_label_keys,
-    )
-
     positive_count = label_metadata[:, 0]
-    negative_count = (
-        count_groundtruths_per_key[label_metadata[:, 2]] - label_metadata[:, 0]
-    )
+    negative_count = label_metadata[:, 1] - label_metadata[:, 0]
 
     true_positives = np.zeros((n_labels, n_datums), dtype=np.int32)
     false_positives = np.zeros_like(true_positives)
@@ -35,7 +25,6 @@ def _compute_rocauc(
             continue
 
         mask_pds = pd_labels == label_idx
-
         true_positives[label_idx] = mask_matching_labels[mask_pds]
         false_positives[label_idx] = ~mask_matching_labels[mask_pds]
         scores[label_idx] = data[mask_pds, 3]
@@ -70,32 +59,25 @@ def _compute_rocauc(
     rocauc = np.trapz(x=fpr, y=tpr, axis=1)  # type: ignore - numpy will be switching to `trapezoid` in the future.
 
     # compute mean rocauc
-    summed_rocauc = np.bincount(label_metadata[:, 2], weights=rocauc)
-    mean_rocauc = np.zeros(n_label_keys, dtype=np.float64)
-    np.divide(
-        summed_rocauc,
-        count_labels_per_key,
-        where=count_labels_per_key > 1e-9,
-        out=mean_rocauc,
-    )
+    mean_rocauc = rocauc.mean()
 
     return rocauc, mean_rocauc
 
 
 def compute_metrics(
-    data: NDArray[np.floating],
+    data: NDArray[np.float64],
     label_metadata: NDArray[np.int32],
-    score_thresholds: NDArray[np.floating],
+    score_thresholds: NDArray[np.float64],
     hardmax: bool,
     n_datums: int,
 ) -> tuple[
     NDArray[np.int32],
-    NDArray[np.floating],
-    NDArray[np.floating],
-    NDArray[np.floating],
-    NDArray[np.floating],
-    NDArray[np.floating],
-    NDArray[np.floating],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    float,
 ]:
     """
     Computes classification metrics.
@@ -110,14 +92,14 @@ def compute_metrics(
 
     Parameters
     ----------
-    data : NDArray[np.floating]
+    data : NDArray[np.float64]
         A sorted array of classification pairs.
     label_metadata : NDArray[np.int32]
         An array containing metadata related to labels.
-    score_thresholds : NDArray[np.floating]
+    score_thresholds : NDArray[np.float64]
         A 1-D array contains score thresholds to compute metrics over.
     hardmax : bool
-        Option to only allow a single positive prediction per label key.
+        Option to only allow a single positive prediction.
     n_datums : int
         The number of datums being operated over.
 
@@ -125,22 +107,21 @@ def compute_metrics(
     -------
     NDArray[np.int32]
         TP, FP, FN, TN counts.
-    NDArray[np.floating]
+    NDArray[np.float64]
         Precision.
-    NDArray[np.floating]
+    NDArray[np.float64]
         Recall.
-    NDArray[np.floating]
+    NDArray[np.float64]
         Accuracy
-    NDArray[np.floating]
+    NDArray[np.float64]
         F1 Score
-    NDArray[np.floating]
+    NDArray[np.float64]
         ROCAUC.
-    NDArray[np.floating]
+    float
         mROCAUC.
     """
 
     n_labels = label_metadata.shape[0]
-    n_label_keys = np.unique(label_metadata[:, 2]).size
     n_scores = score_thresholds.shape[0]
 
     pd_labels = data[:, 2].astype(int)
@@ -155,7 +136,6 @@ def compute_metrics(
         label_metadata=label_metadata,
         n_datums=n_datums,
         n_labels=n_labels,
-        n_label_keys=n_label_keys,
         mask_matching_labels=mask_matching_labels,
         pd_labels=pd_labels,
     )
@@ -229,16 +209,16 @@ def compute_metrics(
 
 
 def _count_with_examples(
-    data: NDArray[np.floating],
+    data: NDArray[np.float64],
     unique_idx: int | list[int],
     label_idx: int | list[int],
-) -> tuple[NDArray[np.floating], NDArray[np.int32], NDArray[np.int32]]:
+) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.int32]]:
     """
     Helper function for counting occurences of unique detailed pairs.
 
     Parameters
     ----------
-    data : NDArray[np.floating]
+    data : NDArray[np.float64]
         A masked portion of a detailed pairs array.
     unique_idx : int | list[int]
         The index or indices upon which uniqueness is constrained.
@@ -247,7 +227,7 @@ def _count_with_examples(
 
     Returns
     -------
-    NDArray[np.floating]
+    NDArray[np.float64]
         Examples drawn from the data input.
     NDArray[np.int32]
         Unique label indices.
@@ -267,13 +247,12 @@ def _count_with_examples(
 
 
 def compute_confusion_matrix(
-    data: NDArray[np.floating],
+    data: NDArray[np.float64],
     label_metadata: NDArray[np.int32],
-    score_thresholds: NDArray[np.floating],
+    score_thresholds: NDArray[np.float64],
     hardmax: bool,
     n_examples: int,
-) -> tuple[NDArray[np.floating], NDArray[np.int32]]:
-
+) -> tuple[NDArray[np.float64], NDArray[np.int32]]:
     """
     Compute detailed confusion matrix.
 
@@ -287,20 +266,20 @@ def compute_confusion_matrix(
 
     Parameters
     ----------
-    data : NDArray[np.floating]
+    data : NDArray[np.float64]
         A sorted array summarizing the IOU calculations of one or more pairs.
     label_metadata : NDArray[np.int32]
         An array containing metadata related to labels.
-    iou_thresholds : NDArray[np.floating]
+    iou_thresholds : NDArray[np.float64]
         A 1-D array containing IoU thresholds.
-    score_thresholds : NDArray[np.floating]
+    score_thresholds : NDArray[np.float64]
         A 1-D array containing score thresholds.
     n_examples : int
         The maximum number of examples to return per count.
 
     Returns
     -------
-    NDArray[np.floating]
+    NDArray[np.float64]
         Confusion matrix.
     NDArray[np.int32]
         Ground truths with missing predictions.

valor_lite/classification/manager.py

@@ -67,13 +67,8 @@ class Evaluator:
         self.index_to_uid: dict[int, str] = dict()
 
         # label reference
-        self.label_to_index: dict[tuple[str, str], int] = dict()
-        self.index_to_label: dict[int, tuple[str, str]] = dict()
-
-        # label key reference
-        self.index_to_label_key: dict[int, str] = dict()
-        self.label_key_to_index: dict[str, int] = dict()
-        self.label_index_to_label_key_index: dict[int, int] = dict()
+        self.label_to_index: dict[str, int] = dict()
+        self.index_to_label: dict[int, str] = dict()
 
         # computation caches
         self._detailed_pairs = np.array([])
@@ -81,7 +76,7 @@ class Evaluator:
         self._label_metadata_per_datum = np.array([], dtype=np.int32)
 
     @property
-    def ignored_prediction_labels(self) -> list[tuple[str, str]]:
+    def ignored_prediction_labels(self) -> list[str]:
         """
         Prediction labels that are not present in the ground truth set.
         """
@@ -92,7 +87,7 @@ class Evaluator:
         ]
 
     @property
-    def missing_prediction_labels(self) -> list[tuple[str, str]]:
+    def missing_prediction_labels(self) -> list[str]:
         """
         Ground truth labels that are not present in the prediction set.
         """
@@ -119,8 +114,7 @@ class Evaluator:
     def create_filter(
         self,
         datum_uids: list[str] | NDArray[np.int32] | None = None,
-        labels: list[tuple[str, str]] | NDArray[np.int32] | None = None,
-        label_keys: list[str] | NDArray[np.int32] | None = None,
+        labels: list[str] | NDArray[np.int32] | None = None,
     ) -> Filter:
         """
         Creates a boolean mask that can be passed to an evaluation.
@@ -129,10 +123,8 @@ class Evaluator:
         ----------
         datum_uids : list[str] | NDArray[np.int32], optional
             An optional list of string uids or a numpy array of uid indices.
-        labels : list[tuple[str, str]] | NDArray[np.int32], optional
+        labels : list[str] | NDArray[np.int32], optional
             An optional list of labels or a numpy array of label indices.
-        label_keys : list[str] | NDArray[np.int32], optional
-            An optional list of label keys or a numpy array of label key indices.
 
         Returns
         -------
@@ -179,36 +171,18 @@ class Evaluator:
             mask[labels] = True
             mask_labels &= mask
 
-        if label_keys is not None:
-            if isinstance(label_keys, list):
-                label_keys = np.array(
-                    [self.label_key_to_index[key] for key in label_keys]
-                )
-            label_indices = np.where(
-                np.isclose(self._label_metadata[:, 2], label_keys)
-            )[0]
-            mask = np.zeros_like(mask_pairs, dtype=np.bool_)
-            mask[
-                np.isin(self._detailed_pairs[:, 1].astype(int), label_indices)
-            ] = True
-            mask_pairs &= mask
-
-            mask = np.zeros_like(mask_labels, dtype=np.bool_)
-            mask[label_indices] = True
-            mask_labels &= mask
-
         mask = mask_datums[:, np.newaxis] & mask_labels[np.newaxis, :]
         label_metadata_per_datum = self._label_metadata_per_datum.copy()
         label_metadata_per_datum[:, ~mask] = 0
 
         label_metadata = np.zeros_like(self._label_metadata, dtype=np.int32)
-        label_metadata[:, :2] = np.transpose(
+        label_metadata = np.transpose(
             np.sum(
                 label_metadata_per_datum,
                 axis=1,
             )
         )
-        label_metadata[:, 2] = self._label_metadata[:, 2]
+
         n_datums = int(np.sum(label_metadata[:, 0]))
 
         return Filter(
@@ -241,6 +215,8 @@ class Evaluator:
             Maximum number of annotation examples to return in ConfusionMatrix.
         filter_ : Filter, optional
             An optional filter object.
+        as_dict : bool, default=False
+            An option to return metrics as dictionaries.
 
         Returns
         -------
@@ -286,10 +262,8 @@ class Evaluator:
 
         metrics[MetricType.mROCAUC] = [
             mROCAUC(
-                value=mean_rocauc[label_key_idx],
-                label_key=self.index_to_label_key[label_key_idx],
+                value=mean_rocauc,
             )
-            for label_key_idx in range(len(self.label_key_to_index))
         ]
 
         for label_idx, label in self.index_to_label.items():
@@ -364,8 +338,7 @@ class Evaluator:
 
     def _unpack_confusion_matrix(
         self,
-        confusion_matrix: NDArray[np.floating],
-        label_key_idx: int,
+        confusion_matrix: NDArray[np.float64],
         number_of_labels: int,
         number_of_examples: int,
     ) -> dict[
@@ -405,8 +378,8 @@ class Evaluator:
         )
 
         return {
-            self.index_to_label[gt_label_idx][1]: {
-                self.index_to_label[pd_label_idx][1]: {
+            self.index_to_label[gt_label_idx]: {
+                self.index_to_label[pd_label_idx]: {
                     "count": max(
                         int(confusion_matrix[gt_label_idx, pd_label_idx, 0]),
                         0,
@@ -428,22 +401,13 @@ class Evaluator:
                     ],
                 }
                 for pd_label_idx in range(number_of_labels)
-                if (
-                    self.label_index_to_label_key_index[pd_label_idx]
-                    == label_key_idx
-                )
             }
             for gt_label_idx in range(number_of_labels)
-            if (
-                self.label_index_to_label_key_index[gt_label_idx]
-                == label_key_idx
-            )
         }
 
     def _unpack_missing_predictions(
         self,
         missing_predictions: NDArray[np.int32],
-        label_key_idx: int,
         number_of_labels: int,
         number_of_examples: int,
     ) -> dict[str, dict[str, int | list[dict[str, str]]]]:
@@ -461,7 +425,7 @@ class Evaluator:
         )
 
         return {
-            self.index_to_label[gt_label_idx][1]: {
+            self.index_to_label[gt_label_idx]: {
                 "count": max(
                     int(missing_predictions[gt_label_idx, 0]),
                     0,
@@ -477,15 +441,11 @@ class Evaluator:
                 ],
             }
             for gt_label_idx in range(number_of_labels)
-            if (
-                self.label_index_to_label_key_index[gt_label_idx]
-                == label_key_idx
-            )
         }
 
     def _compute_confusion_matrix(
         self,
-        data: NDArray[np.floating],
+        data: NDArray[np.float64],
         label_metadata: NDArray[np.int32],
         score_thresholds: list[float],
         hardmax: bool,
@@ -496,7 +456,7 @@ class Evaluator:
 
         Parameters
         ----------
-        data : NDArray[np.floating]
+        data : NDArray[np.float64]
             A data array containing classification pairs.
         label_metadata : NDArray[np.int32]
             An integer array containing label metadata.
@@ -510,7 +470,7 @@ class Evaluator:
         Returns
         -------
         list[ConfusionMatrix]
-            A list of ConfusionMatrix per label key.
+            A list of ConfusionMatrix objects.
         """
 
         if data.size == 0:
@@ -528,22 +488,18 @@ class Evaluator:
         return [
             ConfusionMatrix(
                 score_threshold=score_thresholds[score_idx],
-                label_key=label_key,
                 number_of_examples=number_of_examples,
                 confusion_matrix=self._unpack_confusion_matrix(
                     confusion_matrix=confusion_matrix[score_idx, :, :, :],
-                    label_key_idx=label_key_idx,
                     number_of_labels=n_labels,
                     number_of_examples=number_of_examples,
                 ),
                 missing_predictions=self._unpack_missing_predictions(
                     missing_predictions=missing_predictions[score_idx, :, :],
-                    label_key_idx=label_key_idx,
                     number_of_labels=n_labels,
                     number_of_examples=number_of_examples,
                 ),
             )
-            for label_key_idx, label_key in self.index_to_label_key.items()
             for score_idx in range(n_scores)
         ]
 
@@ -578,77 +534,50 @@ class DataLoader:
             self._evaluator.index_to_uid[index] = uid
         return self._evaluator.uid_to_index[uid]
 
-    def _add_label(self, label: tuple[str, str]) -> tuple[int, int]:
+    def _add_label(self, label: str) -> int:
         """
         Helper function for adding a label to the cache.
 
         Parameters
         ----------
-        label : tuple[str, str]
-            The label as a tuple in format (key, value).
+        label : str
+            A string representing a label.
 
         Returns
         -------
         int
             Label index.
-        int
-            Label key index.
         """
         label_id = len(self._evaluator.index_to_label)
-        label_key_id = len(self._evaluator.index_to_label_key)
         if label not in self._evaluator.label_to_index:
             self._evaluator.label_to_index[label] = label_id
             self._evaluator.index_to_label[label_id] = label
 
-            # update label key index
-            if label[0] not in self._evaluator.label_key_to_index:
-                self._evaluator.label_key_to_index[label[0]] = label_key_id
-                self._evaluator.index_to_label_key[label_key_id] = label[0]
-                label_key_id += 1
-
-            self._evaluator.label_index_to_label_key_index[
-                label_id
-            ] = self._evaluator.label_key_to_index[label[0]]
             label_id += 1
 
-        return (
-            self._evaluator.label_to_index[label],
-            self._evaluator.label_key_to_index[label[0]],
-        )
+        return self._evaluator.label_to_index[label]
 
     def _add_data(
         self,
         uid_index: int,
-        keyed_groundtruths: dict[int, int],
-        keyed_predictions: dict[int, list[tuple[int, float]]],
+        groundtruth: int,
+        predictions: list[tuple[int, float]],
     ):
-        gt_keys = set(keyed_groundtruths.keys())
-        pd_keys = set(keyed_predictions.keys())
-        joint_keys = gt_keys.intersection(pd_keys)
-
-        gt_unique_keys = gt_keys - pd_keys
-        pd_unique_keys = pd_keys - gt_keys
-        if gt_unique_keys or pd_unique_keys:
-            raise ValueError(
-                "Label keys must match between ground truths and predictions."
-            )
 
         pairs = list()
-        for key in joint_keys:
-            scores = np.array([score for _, score in keyed_predictions[key]])
-            max_score_idx = np.argmax(scores)
-
-            glabel = keyed_groundtruths[key]
-            for idx, (plabel, score) in enumerate(keyed_predictions[key]):
-                pairs.append(
-                    (
-                        float(uid_index),
-                        float(glabel),
-                        float(plabel),
-                        float(score),
-                        float(max_score_idx == idx),
-                    )
+        scores = np.array([score for _, score in predictions])
+        max_score_idx = np.argmax(scores)
+
+        for idx, (plabel, score) in enumerate(predictions):
+            pairs.append(
+                (
+                    float(uid_index),
+                    float(groundtruth),
+                    float(plabel),
+                    float(score),
+                    float(max_score_idx == idx),
                 )
+            )
 
         if self._evaluator._detailed_pairs.size == 0:
             self._evaluator._detailed_pairs = np.array(pairs)
@@ -680,27 +609,29 @@ class DataLoader:
         disable_tqdm = not show_progress
         for classification in tqdm(classifications, disable=disable_tqdm):
 
+            if len(classification.predictions) == 0:
+                raise ValueError(
+                    "Classifications must contain at least one prediction."
+                )
             # update metadata
             self._evaluator.n_datums += 1
-            self._evaluator.n_groundtruths += len(classification.groundtruths)
+            self._evaluator.n_groundtruths += 1
             self._evaluator.n_predictions += len(classification.predictions)
 
             # update datum uid index
             uid_index = self._add_datum(uid=classification.uid)
 
             # cache labels and annotations
-            keyed_groundtruths = defaultdict(int)
-            keyed_predictions = defaultdict(list)
-            for glabel in classification.groundtruths:
-                label_idx, label_key_idx = self._add_label(glabel)
-                self.groundtruth_count[label_idx][uid_index] += 1
-                keyed_groundtruths[label_key_idx] = label_idx
-            for idx, (plabel, pscore) in enumerate(
-                zip(classification.predictions, classification.scores)
+            groundtruth = self._add_label(classification.groundtruth)
+            self.groundtruth_count[groundtruth][uid_index] += 1
+
+            predictions = list()
+            for plabel, pscore in zip(
+                classification.predictions, classification.scores
             ):
-                label_idx, label_key_idx = self._add_label(plabel)
+                label_idx = self._add_label(plabel)
                 self.prediction_count[label_idx][uid_index] += 1
-                keyed_predictions[label_key_idx].append(
+                predictions.append(
                     (
                         label_idx,
                         pscore,
@@ -709,8 +640,8 @@ class DataLoader:
 
             self._add_data(
                 uid_index=uid_index,
-                keyed_groundtruths=keyed_groundtruths,
-                keyed_predictions=keyed_predictions,
+                groundtruth=groundtruth,
+                predictions=predictions,
             )
 
     def add_data_from_valor_dict(
@@ -743,31 +674,38 @@ class DataLoader:
             uid_index = self._add_datum(uid=groundtruth["datum"]["uid"])
 
             # cache labels and annotations
-            keyed_groundtruths = defaultdict(int)
-            keyed_predictions = defaultdict(list)
+            predictions = list()
+            groundtruths = None
             for gann in groundtruth["annotations"]:
                 for valor_label in gann["labels"]:
-                    glabel = (valor_label["key"], valor_label["value"])
-                    label_idx, label_key_idx = self._add_label(glabel)
+                    glabel = f'{valor_label["key"]}_{valor_label["value"]}'
+                    label_idx = self._add_label(glabel)
                     self.groundtruth_count[label_idx][uid_index] += 1
-                    keyed_groundtruths[label_key_idx] = label_idx
+                    groundtruths = label_idx
             for pann in prediction["annotations"]:
                 for valor_label in pann["labels"]:
-                    plabel = (valor_label["key"], valor_label["value"])
+                    plabel = f'{valor_label["key"]}_{valor_label["value"]}'
                     pscore = valor_label["score"]
-                    label_idx, label_key_idx = self._add_label(plabel)
+                    label_idx = self._add_label(plabel)
                     self.prediction_count[label_idx][uid_index] += 1
-                    keyed_predictions[label_key_idx].append(
+                    predictions.append(
                         (
                             label_idx,
                             pscore,
                         )
                     )
 
+            # fix type error where groundtruths can possibly be unbound now that it's a float
+            # in practice, this error should never be hit since groundtruths can't be empty without throwing a ValueError earlier in the flow
+            if groundtruths is None:
+                raise ValueError(
+                    "Expected a value for groundtruths, but got None."
+                )
+
             self._add_data(
                 uid_index=uid_index,
-                keyed_groundtruths=keyed_groundtruths,
-                keyed_predictions=keyed_predictions,
+                groundtruth=groundtruths,
+                predictions=predictions,
             )
 
     def finalize(self) -> Evaluator:
@@ -820,7 +758,6 @@ class DataLoader:
                             1, :, label_idx
                         ]
                     ),
-                    self._evaluator.label_index_to_label_key_index[label_idx],
                 ]
                 for label_idx in range(n_labels)
             ],