valor-lite 0.36.2__tar.gz → 0.36.3__tar.gz

{valor_lite-0.36.2 → valor_lite-0.36.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: valor-lite
-Version: 0.36.2
+Version: 0.36.3
 Summary: Evaluate machine learning models.
 Project-URL: homepage, https://www.striveworks.com
 Requires-Python: >=3.10

{valor_lite-0.36.2 → valor_lite-0.36.3}/valor_lite/classification/computation.py

@@ -1,3 +1,5 @@
+from enum import IntFlag, auto
+
 import numpy as np
 from numpy.typing import NDArray
 
@@ -318,56 +320,20 @@ def compute_precision_recall_rocauc(
     )
 
 
-def _count_with_examples(
-    data: NDArray[np.float64],
-    unique_idx: int | list[int],
-    label_idx: int | list[int],
-) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.intp]]:
-    """
-    Helper function for counting occurences of unique detailed pairs.
-
-    Parameters
-    ----------
-    data : NDArray[np.float64]
-        A masked portion of a detailed pairs array.
-    unique_idx : int | list[int]
-        The index or indices upon which uniqueness is constrained.
-    label_idx : int | list[int]
-        The index or indices within the unique index or indices that encode labels.
-
-    Returns
-    -------
-    NDArray[np.float64]
-        Examples drawn from the data input.
-    NDArray[np.int32]
-        Unique label indices.
-    NDArray[np.intp]
-        Counts for each unique label index.
-    """
-    unique_rows, indices = np.unique(
-        data.astype(int)[:, unique_idx],
-        return_index=True,
-        axis=0,
-    )
-    examples = data[indices]
-    labels, counts = np.unique(
-        unique_rows[:, label_idx], return_counts=True, axis=0
-    )
-    return examples, labels, counts
+class PairClassification(IntFlag):
+    TP = auto()
+    FP_FN_MISCLF = auto()
+    FN_UNMATCHED = auto()
 
 
 def compute_confusion_matrix(
     detailed_pairs: NDArray[np.float64],
-    label_metadata: NDArray[np.int32],
     score_thresholds: NDArray[np.float64],
     hardmax: bool,
-    n_examples: int,
-) -> tuple[NDArray[np.float64], NDArray[np.int32]]:
+) -> NDArray[np.uint8]:
     """
     Compute detailed confusion matrix.
 
-    Takes data with shape (N, 5):
-
     Parameters
     ----------
     detailed_pairs : NDArray[np.float64]
@@ -377,37 +343,22 @@ def compute_confusion_matrix(
             Index 2 - Prediction Label Index
             Index 3 - Score
             Index 4 - Hard Max Score
-    label_metadata : NDArray[np.int32]
-        A 2-D array containing metadata related to labels with shape (n_labels, 2).
-            Index 0 - GroundTruth Label Count
-            Index 1 - Prediction Label Count
     iou_thresholds : NDArray[np.float64]
         A 1-D array containing IOU thresholds.
     score_thresholds : NDArray[np.float64]
         A 1-D array containing score thresholds.
-    n_examples : int
-        The maximum number of examples to return per count.
 
     Returns
     -------
-    NDArray[np.float64]
-        Confusion matrix.
-    NDArray[np.int32]
-        Unmatched Ground Truths.
+    NDArray[uint8]
+        Row-wise classification of pairs.
     """
-
-    n_labels = label_metadata.shape[0]
+    n_pairs = detailed_pairs.shape[0]
     n_scores = score_thresholds.shape[0]
 
-    confusion_matrix = np.full(
-        (n_scores, n_labels, n_labels, 2 * n_examples + 1),
-        fill_value=-1.0,
-        dtype=np.float32,
-    )
-    unmatched_ground_truths = np.full(
-        (n_scores, n_labels, n_examples + 1),
-        fill_value=-1,
-        dtype=np.int32,
+    pair_classifications = np.zeros(
+        (n_scores, n_pairs),
+        dtype=np.uint8,
     )
 
     mask_label_match = np.isclose(detailed_pairs[:, 1], detailed_pairs[:, 2])
@@ -420,9 +371,9 @@ def compute_confusion_matrix(
         if hardmax:
             mask_score &= detailed_pairs[:, 4] > 0.5
 
-        mask_tp = mask_label_match & mask_score
-        mask_misclf = ~mask_label_match & mask_score
-        mask_misprd = ~(
+        mask_true_positives = mask_label_match & mask_score
+        mask_misclassifications = ~mask_label_match & mask_score
+        mask_unmatched_groundtruths = ~(
             (
                 groundtruths.reshape(-1, 1, 2)
                 == groundtruths[mask_score].reshape(1, -1, 2)
@@ -431,73 +382,15 @@ def compute_confusion_matrix(
             .any(axis=1)
         )
 
-        tp_examples, tp_labels, tp_counts = _count_with_examples(
-            data=detailed_pairs[mask_tp],
-            unique_idx=[0, 2],
-            label_idx=1,
-        )
-        misclf_examples, misclf_labels, misclf_counts = _count_with_examples(
-            data=detailed_pairs[mask_misclf],
-            unique_idx=[0, 1, 2],
-            label_idx=[1, 2],
+        # classify pairings
+        pair_classifications[score_idx, mask_true_positives] |= np.uint8(
+            PairClassification.TP
         )
-        misprd_examples, misprd_labels, misprd_counts = _count_with_examples(
-            data=detailed_pairs[mask_misprd],
-            unique_idx=[0, 1],
-            label_idx=1,
+        pair_classifications[score_idx, mask_misclassifications] |= np.uint8(
+            PairClassification.FP_FN_MISCLF
         )
+        pair_classifications[
+            score_idx, mask_unmatched_groundtruths
+        ] |= np.uint8(PairClassification.FN_UNMATCHED)
 
-        confusion_matrix[score_idx, tp_labels, tp_labels, 0] = tp_counts
-        confusion_matrix[
-            score_idx, misclf_labels[:, 0], misclf_labels[:, 1], 0
-        ] = misclf_counts
-
-        unmatched_ground_truths[score_idx, misprd_labels, 0] = misprd_counts
-
-        if n_examples > 0:
-            for label_idx in range(n_labels):
-                # true-positive examples
-                mask_tp_label = tp_examples[:, 2] == label_idx
-                if mask_tp_label.sum() > 0:
-                    tp_label_examples = tp_examples[mask_tp_label][:n_examples]
-                    confusion_matrix[
-                        score_idx,
-                        label_idx,
-                        label_idx,
-                        1 : 2 * tp_label_examples.shape[0] + 1,
-                    ] = tp_label_examples[:, [0, 3]].flatten()
-
-                # misclassification examples
-                mask_misclf_gt_label = misclf_examples[:, 1] == label_idx
-                if mask_misclf_gt_label.sum() > 0:
-                    for pd_label_idx in range(n_labels):
-                        mask_misclf_pd_label = (
-                            misclf_examples[:, 2] == pd_label_idx
-                        )
-                        mask_misclf_label_combo = (
-                            mask_misclf_gt_label & mask_misclf_pd_label
-                        )
-                        if mask_misclf_label_combo.sum() > 0:
-                            misclf_label_examples = misclf_examples[
-                                mask_misclf_label_combo
-                            ][:n_examples]
-                            confusion_matrix[
-                                score_idx,
-                                label_idx,
-                                pd_label_idx,
-                                1 : 2 * misclf_label_examples.shape[0] + 1,
-                            ] = misclf_label_examples[:, [0, 3]].flatten()
-
-                # unmatched ground truth examples
-                mask_misprd_label = misprd_examples[:, 1] == label_idx
-                if misprd_examples.size > 0:
-                    misprd_label_examples = misprd_examples[mask_misprd_label][
-                        :n_examples
-                    ]
-                    unmatched_ground_truths[
-                        score_idx,
-                        label_idx,
-                        1 : misprd_label_examples.shape[0] + 1,
-                    ] = misprd_label_examples[:, 0].flatten()
-
-    return confusion_matrix, unmatched_ground_truths  # type: ignore[reportReturnType]
+    return pair_classifications

{valor_lite-0.36.2 → valor_lite-0.36.3}/valor_lite/classification/manager.py

@@ -16,7 +16,7 @@ from valor_lite.classification.utilities import (
     unpack_confusion_matrix_into_metric_list,
     unpack_precision_recall_rocauc_into_metric_lists,
 )
-from valor_lite.exceptions import EmptyEvaluatorException, EmptyFilterException
+from valor_lite.exceptions import EmptyEvaluatorError, EmptyFilterError
 
 """
 Usage
@@ -88,14 +88,14 @@ class Filter:
     def __post_init__(self):
         # validate datum mask
         if not self.datum_mask.any():
-            raise EmptyFilterException("filter removes all datums")
+            raise EmptyFilterError("filter removes all datums")
 
         # validate label indices
         if (
             self.valid_label_indices is not None
             and self.valid_label_indices.size == 0
         ):
-            raise EmptyFilterException("filter removes all labels")
+            raise EmptyFilterError("filter removes all labels")
 
 
 class Evaluator:
@@ -292,7 +292,6 @@ class Evaluator:
         self,
         score_thresholds: list[float] = [0.0],
         hardmax: bool = True,
-        number_of_examples: int = 0,
         filter_: Filter | None = None,
     ) -> list[Metric]:
         """
@@ -304,8 +303,6 @@ class Evaluator:
             A list of score thresholds to compute metrics over.
         hardmax : bool
             Toggles whether a hardmax is applied to predictions.
-        number_of_examples : int, default=0
-            The number of examples to return per count.
         filter_ : Filter, optional
             Applies a filter to the internal cache.
 
@@ -316,25 +313,22 @@ class Evaluator:
         """
         # apply filters
         if filter_ is not None:
-            detailed_pairs, label_metadata = self.filter(filter_=filter_)
+            detailed_pairs, _ = self.filter(filter_=filter_)
         else:
             detailed_pairs = self._detailed_pairs
-            label_metadata = self._label_metadata
 
         if detailed_pairs.size == 0:
             return list()
 
-        results = compute_confusion_matrix(
+        result = compute_confusion_matrix(
             detailed_pairs=detailed_pairs,
-            label_metadata=label_metadata,
             score_thresholds=np.array(score_thresholds),
             hardmax=hardmax,
-            n_examples=number_of_examples,
         )
         return unpack_confusion_matrix_into_metric_list(
-            results=results,
+            detailed_pairs=detailed_pairs,
+            result=result,
             score_thresholds=score_thresholds,
-            number_of_examples=number_of_examples,
             index_to_datum_id=self.index_to_datum_id,
             index_to_label=self.index_to_label,
         )
@@ -343,7 +337,6 @@ class Evaluator:
         self,
         score_thresholds: list[float] = [0.0],
         hardmax: bool = True,
-        number_of_examples: int = 0,
         filter_: Filter | None = None,
     ) -> dict[MetricType, list[Metric]]:
         """
@@ -355,8 +348,6 @@ class Evaluator:
             A list of score thresholds to compute metrics over.
         hardmax : bool
             Toggles whether a hardmax is applied to predictions.
-        number_of_examples : int, default=0
-            The number of examples to return per count.
         filter_ : Filter, optional
             Applies a filter to the internal cache.
 
@@ -373,7 +364,6 @@ class Evaluator:
         metrics[MetricType.ConfusionMatrix] = self.compute_confusion_matrix(
             score_thresholds=score_thresholds,
             hardmax=hardmax,
-            number_of_examples=number_of_examples,
             filter_=filter_,
         )
         return metrics
@@ -391,11 +381,17 @@ class Evaluator:
         -------
         int
             The datum index.
+
+        Raises
+        ------
+        ValueError
+            If datum id already exists.
         """
-        if uid not in self.datum_id_to_index:
-            index = len(self.datum_id_to_index)
-            self.datum_id_to_index[uid] = index
-            self.index_to_datum_id.append(uid)
+        if uid in self.datum_id_to_index:
+            raise ValueError("datum with id '{uid}' already exists")
+        index = len(self.datum_id_to_index)
+        self.datum_id_to_index[uid] = index
+        self.index_to_datum_id.append(uid)
         return self.datum_id_to_index[uid]
 
     def _add_label(self, label: str) -> int:
@@ -497,7 +493,7 @@ class Evaluator:
             A ready-to-use evaluator object.
         """
         if self._detailed_pairs.size == 0:
-            raise EmptyEvaluatorException()
+            raise EmptyEvaluatorError()
 
         self._label_metadata = compute_label_metadata(
             ids=self._detailed_pairs[:, :3].astype(np.int32),

{valor_lite-0.36.2 → valor_lite-0.36.3}/valor_lite/classification/metric.py

@@ -329,7 +329,6 @@ class Metric(BaseMetric):
             ],
         ],
         score_threshold: float,
-        maximum_number_of_examples: int,
     ):
         """
         The confusion matrix and related metrics for the classification task.
@@ -382,8 +381,6 @@ class Metric(BaseMetric):
             Each example includes the datum UID.
         score_threshold : float
             The confidence score threshold used to filter predictions.
-        maximum_number_of_examples : int
-            The maximum number of examples per element.
 
         Returns
         -------
@@ -397,6 +394,5 @@ class Metric(BaseMetric):
             },
             parameters={
                 "score_threshold": score_threshold,
-                "maximum_number_of_examples": maximum_number_of_examples,
             },
         )

valor_lite-0.36.3/valor_lite/classification/utilities.py

@@ -0,0 +1,211 @@
+from collections import defaultdict
+
+import numpy as np
+from numpy.typing import NDArray
+
+from valor_lite.classification.computation import PairClassification
+from valor_lite.classification.metric import Metric, MetricType
+
+
+def unpack_precision_recall_rocauc_into_metric_lists(
+    results: tuple[
+        NDArray[np.int32],
+        NDArray[np.float64],
+        NDArray[np.float64],
+        NDArray[np.float64],
+        NDArray[np.float64],
+        NDArray[np.float64],
+        float,
+    ],
+    score_thresholds: list[float],
+    hardmax: bool,
+    label_metadata: NDArray[np.int32],
+    index_to_label: list[str],
+) -> dict[MetricType, list[Metric]]:
+    (
+        counts,
+        precision,
+        recall,
+        accuracy,
+        f1_score,
+        rocauc,
+        mean_rocauc,
+    ) = results
+
+    metrics = defaultdict(list)
+
+    metrics[MetricType.ROCAUC] = [
+        Metric.roc_auc(
+            value=float(rocauc[label_idx]),
+            label=label,
+        )
+        for label_idx, label in enumerate(index_to_label)
+        if label_metadata[label_idx, 0] > 0
+    ]
+
+    metrics[MetricType.mROCAUC] = [
+        Metric.mean_roc_auc(
+            value=float(mean_rocauc),
+        )
+    ]
+
+    metrics[MetricType.Accuracy] = [
+        Metric.accuracy(
+            value=float(accuracy[score_idx]),
+            score_threshold=score_threshold,
+            hardmax=hardmax,
+        )
+        for score_idx, score_threshold in enumerate(score_thresholds)
+    ]
+
+    for label_idx, label in enumerate(index_to_label):
+        for score_idx, score_threshold in enumerate(score_thresholds):
+
+            kwargs = {
+                "label": label,
+                "hardmax": hardmax,
+                "score_threshold": score_threshold,
+            }
+            row = counts[:, label_idx]
+            metrics[MetricType.Counts].append(
+                Metric.counts(
+                    tp=int(row[score_idx, 0]),
+                    fp=int(row[score_idx, 1]),
+                    fn=int(row[score_idx, 2]),
+                    tn=int(row[score_idx, 3]),
+                    **kwargs,
+                )
+            )
+
+            # if no groundtruths exists for a label, skip it.
+            if label_metadata[label_idx, 0] == 0:
+                continue
+
+            metrics[MetricType.Precision].append(
+                Metric.precision(
+                    value=float(precision[score_idx, label_idx]),
+                    **kwargs,
+                )
+            )
+            metrics[MetricType.Recall].append(
+                Metric.recall(
+                    value=float(recall[score_idx, label_idx]),
+                    **kwargs,
+                )
+            )
+            metrics[MetricType.F1].append(
+                Metric.f1_score(
+                    value=float(f1_score[score_idx, label_idx]),
+                    **kwargs,
+                )
+            )
+    return metrics
+
+
+def _create_empty_confusion_matrix(index_to_labels: list[str]):
+    unmatched_ground_truths = dict()
+    confusion_matrix = dict()
+    for label in index_to_labels:
+        unmatched_ground_truths[label] = {"count": 0, "examples": []}
+        confusion_matrix[label] = {}
+        for plabel in index_to_labels:
+            confusion_matrix[label][plabel] = {"count": 0, "examples": []}
+    return (
+        confusion_matrix,
+        unmatched_ground_truths,
+    )
+
+
+def _unpack_confusion_matrix(
+    ids: NDArray[np.int32],
+    scores: NDArray[np.float64],
+    mask_matched: NDArray[np.bool_],
+    mask_fn_unmatched: NDArray[np.bool_],
+    index_to_datum_id: list[str],
+    index_to_label: list[str],
+    score_threshold: float,
+):
+    (
+        confusion_matrix,
+        unmatched_ground_truths,
+    ) = _create_empty_confusion_matrix(index_to_label)
+
+    unique_matches, unique_match_indices = np.unique(
+        ids[np.ix_(mask_matched, (0, 1, 2))],  # type: ignore - numpy ix_ typing
+        axis=0,
+        return_index=True,
+    )
+    (
+        unique_unmatched_groundtruths,
+        unique_unmatched_groundtruth_indices,
+    ) = np.unique(
+        ids[np.ix_(mask_fn_unmatched, (0, 1))],  # type: ignore - numpy ix_ typing
+        axis=0,
+        return_index=True,
+    )
+
+    n_matched = unique_matches.shape[0]
+    n_unmatched_groundtruths = unique_unmatched_groundtruths.shape[0]
+    n_max = max(n_matched, n_unmatched_groundtruths)
+
+    for idx in range(n_max):
+        if idx < n_matched:
+            glabel = index_to_label[unique_matches[idx, 1]]
+            plabel = index_to_label[unique_matches[idx, 2]]
+            confusion_matrix[glabel][plabel]["count"] += 1
+            confusion_matrix[glabel][plabel]["examples"].append(
+                {
+                    "datum_id": index_to_datum_id[unique_matches[idx, 0]],
+                    "score": float(scores[unique_match_indices[idx]]),
+                }
+            )
+        if idx < n_unmatched_groundtruths:
+            label = index_to_label[unique_unmatched_groundtruths[idx, 1]]
+            unmatched_ground_truths[label]["count"] += 1
+            unmatched_ground_truths[label]["examples"].append(
+                {
+                    "datum_id": index_to_datum_id[
+                        unique_unmatched_groundtruths[idx, 0]
+                    ],
+                }
+            )
+
+    return Metric.confusion_matrix(
+        confusion_matrix=confusion_matrix,
+        unmatched_ground_truths=unmatched_ground_truths,
+        score_threshold=score_threshold,
+    )
+
+
+def unpack_confusion_matrix_into_metric_list(
+    result: NDArray[np.uint8],
+    detailed_pairs: NDArray[np.float64],
+    score_thresholds: list[float],
+    index_to_datum_id: list[str],
+    index_to_label: list[str],
+) -> list[Metric]:
+
+    ids = detailed_pairs[:, :3].astype(np.int32)
+
+    mask_matched = (
+        np.bitwise_and(
+            result, PairClassification.TP | PairClassification.FP_FN_MISCLF
+        )
+        > 0
+    )
+    mask_fn_unmatched = (
+        np.bitwise_and(result, PairClassification.FN_UNMATCHED) > 0
+    )
+
+    return [
+        _unpack_confusion_matrix(
+            ids=ids,
+            scores=detailed_pairs[:, 3],
+            mask_matched=mask_matched[score_idx, :],
+            mask_fn_unmatched=mask_fn_unmatched[score_idx, :],
+            index_to_datum_id=index_to_datum_id,
+            index_to_label=index_to_label,
+            score_threshold=score_threshold,
+        )
+        for score_idx, score_threshold in enumerate(score_thresholds)
+    ]

{valor_lite-0.36.2 → valor_lite-0.36.3}/valor_lite/exceptions.py

@@ -1,15 +1,15 @@
-class EmptyEvaluatorException(Exception):
+class EmptyEvaluatorError(Exception):
     def __init__(self):
         super().__init__(
             "evaluator cannot be finalized as it contains no data"
         )
 
 
-class EmptyFilterException(Exception):
+class EmptyFilterError(Exception):
     def __init__(self, message: str):
         super().__init__(message)
 
 
-class InternalCacheException(Exception):
+class InternalCacheError(Exception):
     def __init__(self, message: str):
         super().__init__(message)

{valor_lite-0.36.2 → valor_lite-0.36.3}/valor_lite/object_detection/manager.py

@@ -6,9 +6,9 @@ from numpy.typing import NDArray
 from tqdm import tqdm
 
 from valor_lite.exceptions import (
-    EmptyEvaluatorException,
-    EmptyFilterException,
-    InternalCacheException,
+    EmptyEvaluatorError,
+    EmptyFilterError,
+    InternalCacheError,
 )
 from valor_lite.object_detection.annotation import (
     Bitmask,
@@ -102,13 +102,13 @@ class Filter:
     def __post_init__(self):
         # validate datums mask
         if not self.mask_datums.any():
-            raise EmptyFilterException("filter removes all datums")
+            raise EmptyFilterError("filter removes all datums")
 
         # validate annotation masks
         no_gts = self.mask_groundtruths.all()
         no_pds = self.mask_predictions.all()
         if no_gts and no_pds:
-            raise EmptyFilterException("filter removes all annotations")
+            raise EmptyFilterError("filter removes all annotations")
         elif no_gts:
             warnings.warn("filter removes all ground truths")
         elif no_pds:
@@ -197,7 +197,7 @@ class Evaluator:
         # filter datums
         if datum_ids is not None:
             if not datum_ids:
-                raise EmptyFilterException("filter removes all datums")
+                raise EmptyFilterError("filter removes all datums")
             valid_datum_indices = np.array(
                 [self.datum_id_to_index[uid] for uid in datum_ids],
                 dtype=np.int32,
@@ -240,7 +240,7 @@ class Evaluator:
         # filter by labels
         if labels is not None:
             if not labels:
-                raise EmptyFilterException("filter removes all labels")
+                raise EmptyFilterError("filter removes all labels")
             valid_label_indices = np.array(
                 [self.label_to_index[label] for label in labels] + [-1]
             )
@@ -467,7 +467,7 @@ class DataLoader:
             if len(self._evaluator.datum_id_to_index) != len(
                 self._evaluator.index_to_datum_id
             ):
-                raise InternalCacheException("datum cache size mismatch")
+                raise InternalCacheError("datum cache size mismatch")
             idx = len(self._evaluator.datum_id_to_index)
             self._evaluator.datum_id_to_index[datum_id] = idx
             self._evaluator.index_to_datum_id.append(datum_id)
@@ -491,9 +491,7 @@ class DataLoader:
             if len(self._evaluator.groundtruth_id_to_index) != len(
                 self._evaluator.index_to_groundtruth_id
             ):
-                raise InternalCacheException(
-                    "ground truth cache size mismatch"
-                )
+                raise InternalCacheError("ground truth cache size mismatch")
             idx = len(self._evaluator.groundtruth_id_to_index)
             self._evaluator.groundtruth_id_to_index[annotation_id] = idx
             self._evaluator.index_to_groundtruth_id.append(annotation_id)
@@ -517,7 +515,7 @@ class DataLoader:
             if len(self._evaluator.prediction_id_to_index) != len(
                 self._evaluator.index_to_prediction_id
             ):
-                raise InternalCacheException("prediction cache size mismatch")
+                raise InternalCacheError("prediction cache size mismatch")
             idx = len(self._evaluator.prediction_id_to_index)
             self._evaluator.prediction_id_to_index[annotation_id] = idx
             self._evaluator.index_to_prediction_id.append(annotation_id)
@@ -542,7 +540,7 @@ class DataLoader:
             if len(self._evaluator.label_to_index) != len(
                 self._evaluator.index_to_label
             ):
-                raise InternalCacheException("label cache size mismatch")
+                raise InternalCacheError("label cache size mismatch")
             self._evaluator.label_to_index[label] = label_id
             self._evaluator.index_to_label.append(label)
             label_id += 1
@@ -768,14 +766,14 @@ class DataLoader:
             A ready-to-use evaluator object.
         """
         if not self.pairs:
-            raise EmptyEvaluatorException()
+            raise EmptyEvaluatorError()
 
         n_labels = len(self._evaluator.index_to_label)
         n_datums = len(self._evaluator.index_to_datum_id)
 
         self._evaluator._detailed_pairs = np.concatenate(self.pairs, axis=0)
         if self._evaluator._detailed_pairs.size == 0:
-            raise EmptyEvaluatorException()
+            raise EmptyEvaluatorError()
 
         # order pairs by descending score, iou
         indices = np.lexsort(

{valor_lite-0.36.2 → valor_lite-0.36.3}/valor_lite/semantic_segmentation/manager.py

@@ -4,7 +4,7 @@ import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
 
-from valor_lite.exceptions import EmptyEvaluatorException, EmptyFilterException
+from valor_lite.exceptions import EmptyEvaluatorError, EmptyFilterError
 from valor_lite.semantic_segmentation.annotation import Segmentation
 from valor_lite.semantic_segmentation.computation import (
     compute_intermediate_confusion_matrices,
@@ -74,11 +74,11 @@ class Filter:
     def __post_init__(self):
         # validate datum mask
         if not self.datum_mask.any():
-            raise EmptyFilterException("filter removes all datums")
+            raise EmptyFilterError("filter removes all datums")
 
         # validate label mask
         if self.label_mask.all():
-            raise EmptyFilterException("filter removes all labels")
+            raise EmptyFilterError("filter removes all labels")
 
 
 class Evaluator:
@@ -403,7 +403,7 @@ class DataLoader:
         """
 
         if len(self.matrices) == 0:
-            raise EmptyEvaluatorException()
+            raise EmptyEvaluatorError()
 
         n_labels = len(self._evaluator.index_to_label)
         n_datums = len(self._evaluator.index_to_datum_id)

{valor_lite-0.36.2 → valor_lite-0.36.3}/valor_lite.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: valor-lite
-Version: 0.36.2
+Version: 0.36.3
 Summary: Evaluate machine learning models.
 Project-URL: homepage, https://www.striveworks.com
 Requires-Python: >=3.10

valor_lite-0.36.2/valor_lite/classification/utilities.py

@@ -1,229 +0,0 @@
-from collections import defaultdict
-
-import numpy as np
-from numpy.typing import NDArray
-
-from valor_lite.classification.metric import Metric, MetricType
-
-
-def unpack_precision_recall_rocauc_into_metric_lists(
-    results: tuple[
-        NDArray[np.int32],
-        NDArray[np.float64],
-        NDArray[np.float64],
-        NDArray[np.float64],
-        NDArray[np.float64],
-        NDArray[np.float64],
-        float,
-    ],
-    score_thresholds: list[float],
-    hardmax: bool,
-    label_metadata: NDArray[np.int32],
-    index_to_label: list[str],
-) -> dict[MetricType, list[Metric]]:
-    (
-        counts,
-        precision,
-        recall,
-        accuracy,
-        f1_score,
-        rocauc,
-        mean_rocauc,
-    ) = results
-
-    metrics = defaultdict(list)
-
-    metrics[MetricType.ROCAUC] = [
-        Metric.roc_auc(
-            value=float(rocauc[label_idx]),
-            label=label,
-        )
-        for label_idx, label in enumerate(index_to_label)
-        if label_metadata[label_idx, 0] > 0
-    ]
-
-    metrics[MetricType.mROCAUC] = [
-        Metric.mean_roc_auc(
-            value=float(mean_rocauc),
-        )
-    ]
-
-    metrics[MetricType.Accuracy] = [
-        Metric.accuracy(
-            value=float(accuracy[score_idx]),
-            score_threshold=score_threshold,
-            hardmax=hardmax,
-        )
-        for score_idx, score_threshold in enumerate(score_thresholds)
-    ]
-
-    for label_idx, label in enumerate(index_to_label):
-        for score_idx, score_threshold in enumerate(score_thresholds):
-
-            kwargs = {
-                "label": label,
-                "hardmax": hardmax,
-                "score_threshold": score_threshold,
-            }
-            row = counts[:, label_idx]
-            metrics[MetricType.Counts].append(
-                Metric.counts(
-                    tp=int(row[score_idx, 0]),
-                    fp=int(row[score_idx, 1]),
-                    fn=int(row[score_idx, 2]),
-                    tn=int(row[score_idx, 3]),
-                    **kwargs,
-                )
-            )
-
-            # if no groundtruths exists for a label, skip it.
-            if label_metadata[label_idx, 0] == 0:
-                continue
-
-            metrics[MetricType.Precision].append(
-                Metric.precision(
-                    value=float(precision[score_idx, label_idx]),
-                    **kwargs,
-                )
-            )
-            metrics[MetricType.Recall].append(
-                Metric.recall(
-                    value=float(recall[score_idx, label_idx]),
-                    **kwargs,
-                )
-            )
-            metrics[MetricType.F1].append(
-                Metric.f1_score(
-                    value=float(f1_score[score_idx, label_idx]),
-                    **kwargs,
-                )
-            )
-    return metrics
-
-
-def _unpack_confusion_matrix_value(
-    confusion_matrix: NDArray[np.float64],
-    number_of_labels: int,
-    number_of_examples: int,
-    index_to_datum_id: list[str],
-    index_to_label: list[str],
-) -> dict[str, dict[str, dict[str, int | list[dict[str, str | float]]]]]:
-    """
-    Unpacks a numpy array of confusion matrix counts and examples.
-    """
-
-    datum_idx = lambda gt_label_idx, pd_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-        confusion_matrix[
-            gt_label_idx,
-            pd_label_idx,
-            example_idx * 2 + 1,
-        ]
-    )
-
-    score_idx = lambda gt_label_idx, pd_label_idx, example_idx: float(  # noqa: E731 - lambda fn
-        confusion_matrix[
-            gt_label_idx,
-            pd_label_idx,
-            example_idx * 2 + 2,
-        ]
-    )
-
-    return {
-        index_to_label[gt_label_idx]: {
-            index_to_label[pd_label_idx]: {
-                "count": max(
-                    int(confusion_matrix[gt_label_idx, pd_label_idx, 0]),
-                    0,
-                ),
-                "examples": [
-                    {
-                        "datum_id": index_to_datum_id[
-                            datum_idx(gt_label_idx, pd_label_idx, example_idx)
-                        ],
-                        "score": score_idx(
-                            gt_label_idx, pd_label_idx, example_idx
-                        ),
-                    }
-                    for example_idx in range(number_of_examples)
-                    if datum_idx(gt_label_idx, pd_label_idx, example_idx) >= 0
-                ],
-            }
-            for pd_label_idx in range(number_of_labels)
-        }
-        for gt_label_idx in range(number_of_labels)
-    }
-
-
-def _unpack_unmatched_ground_truths_value(
-    unmatched_ground_truths: NDArray[np.int32],
-    number_of_labels: int,
-    number_of_examples: int,
-    index_to_datum_id: list[str],
-    index_to_label: list[str],
-) -> dict[str, dict[str, int | list[dict[str, str]]]]:
-    """
-    Unpacks a numpy array of unmatched ground truth counts and examples.
-    """
-
-    datum_idx = (
-        lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            unmatched_ground_truths[
-                gt_label_idx,
-                example_idx + 1,
-            ]
-        )
-    )
-
-    return {
-        index_to_label[gt_label_idx]: {
-            "count": max(
-                int(unmatched_ground_truths[gt_label_idx, 0]),
-                0,
-            ),
-            "examples": [
-                {
-                    "datum_id": index_to_datum_id[
-                        datum_idx(gt_label_idx, example_idx)
-                    ]
-                }
-                for example_idx in range(number_of_examples)
-                if datum_idx(gt_label_idx, example_idx) >= 0
-            ],
-        }
-        for gt_label_idx in range(number_of_labels)
-    }
-
-
-def unpack_confusion_matrix_into_metric_list(
-    results: tuple[NDArray[np.float64], NDArray[np.int32]],
-    score_thresholds: list[float],
-    number_of_examples: int,
-    index_to_datum_id: list[str],
-    index_to_label: list[str],
-) -> list[Metric]:
-
-    (confusion_matrix, unmatched_ground_truths) = results
-    _, n_labels, _, _ = confusion_matrix.shape
-    return [
-        Metric.confusion_matrix(
-            score_threshold=score_threshold,
-            maximum_number_of_examples=number_of_examples,
-            confusion_matrix=_unpack_confusion_matrix_value(
-                confusion_matrix=confusion_matrix[score_idx, :, :, :],
-                number_of_labels=n_labels,
-                number_of_examples=number_of_examples,
-                index_to_label=index_to_label,
-                index_to_datum_id=index_to_datum_id,
-            ),
-            unmatched_ground_truths=_unpack_unmatched_ground_truths_value(
-                unmatched_ground_truths=unmatched_ground_truths[
-                    score_idx, :, :
-                ],
-                number_of_labels=n_labels,
-                number_of_examples=number_of_examples,
-                index_to_label=index_to_label,
-                index_to_datum_id=index_to_datum_id,
-            ),
-        )
-        for score_idx, score_threshold in enumerate(score_thresholds)
-    ]