valor-lite 0.36.2__tar.gz → 0.36.4__tar.gz

{valor_lite-0.36.2 → valor_lite-0.36.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: valor-lite
-Version: 0.36.2
+Version: 0.36.4
 Summary: Evaluate machine learning models.
 Project-URL: homepage, https://www.striveworks.com
 Requires-Python: >=3.10

{valor_lite-0.36.2 → valor_lite-0.36.4}/valor_lite/classification/computation.py

@@ -1,3 +1,5 @@
+from enum import IntFlag, auto
+
 import numpy as np
 from numpy.typing import NDArray
 
@@ -318,56 +320,20 @@ def compute_precision_recall_rocauc(
     )
 
 
-def _count_with_examples(
-    data: NDArray[np.float64],
-    unique_idx: int | list[int],
-    label_idx: int | list[int],
-) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.intp]]:
-    """
-    Helper function for counting occurences of unique detailed pairs.
-
-    Parameters
-    ----------
-    data : NDArray[np.float64]
-        A masked portion of a detailed pairs array.
-    unique_idx : int | list[int]
-        The index or indices upon which uniqueness is constrained.
-    label_idx : int | list[int]
-        The index or indices within the unique index or indices that encode labels.
-
-    Returns
-    -------
-    NDArray[np.float64]
-        Examples drawn from the data input.
-    NDArray[np.int32]
-        Unique label indices.
-    NDArray[np.intp]
-        Counts for each unique label index.
-    """
-    unique_rows, indices = np.unique(
-        data.astype(int)[:, unique_idx],
-        return_index=True,
-        axis=0,
-    )
-    examples = data[indices]
-    labels, counts = np.unique(
-        unique_rows[:, label_idx], return_counts=True, axis=0
-    )
-    return examples, labels, counts
+class PairClassification(IntFlag):
+    TP = auto()
+    FP_FN_MISCLF = auto()
+    FN_UNMATCHED = auto()
 
 
 def compute_confusion_matrix(
     detailed_pairs: NDArray[np.float64],
-    label_metadata: NDArray[np.int32],
     score_thresholds: NDArray[np.float64],
     hardmax: bool,
-    n_examples: int,
-) -> tuple[NDArray[np.float64], NDArray[np.int32]]:
+) -> NDArray[np.uint8]:
     """
     Compute detailed confusion matrix.
 
-    Takes data with shape (N, 5):
-
     Parameters
     ----------
     detailed_pairs : NDArray[np.float64]
@@ -377,37 +343,22 @@ def compute_confusion_matrix(
             Index 2 - Prediction Label Index
             Index 3 - Score
             Index 4 - Hard Max Score
-    label_metadata : NDArray[np.int32]
-        A 2-D array containing metadata related to labels with shape (n_labels, 2).
-            Index 0 - GroundTruth Label Count
-            Index 1 - Prediction Label Count
     iou_thresholds : NDArray[np.float64]
         A 1-D array containing IOU thresholds.
     score_thresholds : NDArray[np.float64]
         A 1-D array containing score thresholds.
-    n_examples : int
-        The maximum number of examples to return per count.
 
     Returns
     -------
-    NDArray[np.float64]
-        Confusion matrix.
-    NDArray[np.int32]
-        Unmatched Ground Truths.
+    NDArray[uint8]
+        Row-wise classification of pairs.
     """
-
-    n_labels = label_metadata.shape[0]
+    n_pairs = detailed_pairs.shape[0]
     n_scores = score_thresholds.shape[0]
 
-    confusion_matrix = np.full(
-        (n_scores, n_labels, n_labels, 2 * n_examples + 1),
-        fill_value=-1.0,
-        dtype=np.float32,
-    )
-    unmatched_ground_truths = np.full(
-        (n_scores, n_labels, n_examples + 1),
-        fill_value=-1,
-        dtype=np.int32,
+    pair_classifications = np.zeros(
+        (n_scores, n_pairs),
+        dtype=np.uint8,
     )
 
     mask_label_match = np.isclose(detailed_pairs[:, 1], detailed_pairs[:, 2])
@@ -420,9 +371,9 @@ def compute_confusion_matrix(
         if hardmax:
             mask_score &= detailed_pairs[:, 4] > 0.5
 
-        mask_tp = mask_label_match & mask_score
-        mask_misclf = ~mask_label_match & mask_score
-        mask_misprd = ~(
+        mask_true_positives = mask_label_match & mask_score
+        mask_misclassifications = ~mask_label_match & mask_score
+        mask_unmatched_groundtruths = ~(
             (
                 groundtruths.reshape(-1, 1, 2)
                 == groundtruths[mask_score].reshape(1, -1, 2)
@@ -431,73 +382,15 @@ def compute_confusion_matrix(
             .any(axis=1)
         )
 
-        tp_examples, tp_labels, tp_counts = _count_with_examples(
-            data=detailed_pairs[mask_tp],
-            unique_idx=[0, 2],
-            label_idx=1,
-        )
-        misclf_examples, misclf_labels, misclf_counts = _count_with_examples(
-            data=detailed_pairs[mask_misclf],
-            unique_idx=[0, 1, 2],
-            label_idx=[1, 2],
+        # classify pairings
+        pair_classifications[score_idx, mask_true_positives] |= np.uint8(
+            PairClassification.TP
         )
-        misprd_examples, misprd_labels, misprd_counts = _count_with_examples(
-            data=detailed_pairs[mask_misprd],
-            unique_idx=[0, 1],
-            label_idx=1,
+        pair_classifications[score_idx, mask_misclassifications] |= np.uint8(
+            PairClassification.FP_FN_MISCLF
         )
+        pair_classifications[
+            score_idx, mask_unmatched_groundtruths
+        ] |= np.uint8(PairClassification.FN_UNMATCHED)
 
-        confusion_matrix[score_idx, tp_labels, tp_labels, 0] = tp_counts
-        confusion_matrix[
-            score_idx, misclf_labels[:, 0], misclf_labels[:, 1], 0
-        ] = misclf_counts
-
-        unmatched_ground_truths[score_idx, misprd_labels, 0] = misprd_counts
-
-        if n_examples > 0:
-            for label_idx in range(n_labels):
-                # true-positive examples
-                mask_tp_label = tp_examples[:, 2] == label_idx
-                if mask_tp_label.sum() > 0:
-                    tp_label_examples = tp_examples[mask_tp_label][:n_examples]
-                    confusion_matrix[
-                        score_idx,
-                        label_idx,
-                        label_idx,
-                        1 : 2 * tp_label_examples.shape[0] + 1,
-                    ] = tp_label_examples[:, [0, 3]].flatten()
-
-                # misclassification examples
-                mask_misclf_gt_label = misclf_examples[:, 1] == label_idx
-                if mask_misclf_gt_label.sum() > 0:
-                    for pd_label_idx in range(n_labels):
-                        mask_misclf_pd_label = (
-                            misclf_examples[:, 2] == pd_label_idx
-                        )
-                        mask_misclf_label_combo = (
-                            mask_misclf_gt_label & mask_misclf_pd_label
-                        )
-                        if mask_misclf_label_combo.sum() > 0:
-                            misclf_label_examples = misclf_examples[
-                                mask_misclf_label_combo
-                            ][:n_examples]
-                            confusion_matrix[
-                                score_idx,
-                                label_idx,
-                                pd_label_idx,
-                                1 : 2 * misclf_label_examples.shape[0] + 1,
-                            ] = misclf_label_examples[:, [0, 3]].flatten()
-
-                # unmatched ground truth examples
-                mask_misprd_label = misprd_examples[:, 1] == label_idx
-                if misprd_examples.size > 0:
-                    misprd_label_examples = misprd_examples[mask_misprd_label][
-                        :n_examples
-                    ]
-                    unmatched_ground_truths[
-                        score_idx,
-                        label_idx,
-                        1 : misprd_label_examples.shape[0] + 1,
-                    ] = misprd_label_examples[:, 0].flatten()
-
-    return confusion_matrix, unmatched_ground_truths  # type: ignore[reportReturnType]
+    return pair_classifications

{valor_lite-0.36.2 → valor_lite-0.36.4}/valor_lite/classification/manager.py

@@ -16,7 +16,7 @@ from valor_lite.classification.utilities import (
     unpack_confusion_matrix_into_metric_list,
     unpack_precision_recall_rocauc_into_metric_lists,
 )
-from valor_lite.exceptions import EmptyEvaluatorException, EmptyFilterException
+from valor_lite.exceptions import EmptyEvaluatorError, EmptyFilterError
 
 """
 Usage
@@ -88,14 +88,14 @@ class Filter:
     def __post_init__(self):
         # validate datum mask
         if not self.datum_mask.any():
-            raise EmptyFilterException("filter removes all datums")
+            raise EmptyFilterError("filter removes all datums")
 
         # validate label indices
         if (
             self.valid_label_indices is not None
             and self.valid_label_indices.size == 0
         ):
-            raise EmptyFilterException("filter removes all labels")
+            raise EmptyFilterError("filter removes all labels")
 
 
 class Evaluator:
@@ -144,18 +144,18 @@ class Evaluator:
 
     def create_filter(
         self,
-        datum_ids: list[str] | None = None,
-        labels: list[str] | None = None,
+        datums: list[str] | NDArray[np.int32] | None = None,
+        labels: list[str] | NDArray[np.int32] | None = None,
     ) -> Filter:
         """
         Creates a filter object.
 
         Parameters
         ----------
-        datum_uids : list[str], optional
-            An optional list of string uids representing datums.
-        labels : list[str], optional
-            An optional list of labels.
+        datums : list[str] | NDArray[int32], optional
+            An optional list of string uids or integer indices representing datums.
+        labels : list[str] | NDArray[int32], optional
+            An optional list of strings or integer indices representing labels.
 
         Returns
         -------
@@ -165,50 +165,72 @@ class Evaluator:
         # create datum mask
         n_pairs = self._detailed_pairs.shape[0]
         datum_mask = np.ones(n_pairs, dtype=np.bool_)
-        if datum_ids is not None:
-            if not datum_ids:
-                return Filter(
-                    datum_mask=np.zeros_like(datum_mask),
-                    valid_label_indices=None,
-                    metadata=Metadata(),
+        if datums is not None:
+            # convert to array of valid datum indices
+            if isinstance(datums, list):
+                datums = np.array(
+                    [self.datum_id_to_index[uid] for uid in datums],
+                    dtype=np.int32,
                 )
-            valid_datum_indices = np.array(
-                [self.datum_id_to_index[uid] for uid in datum_ids],
-                dtype=np.int32,
-            )
-            datum_mask = np.isin(
-                self._detailed_pairs[:, 0], valid_datum_indices
-            )
+
+            # return early if all data removed
+            if datums.size == 0:
+                raise EmptyFilterError("filter removes all datums")
+
+            # validate indices
+            if datums.max() >= len(self.index_to_datum_id):
+                raise ValueError(
+                    f"datum index '{datums.max()}' exceeds total number of datums"
+                )
+            elif datums.min() < 0:
+                raise ValueError(
+                    f"datum index '{datums.min()}' is a negative value"
+                )
+
+            # create datum mask
+            datum_mask = np.isin(self._detailed_pairs[:, 0], datums)
 
         # collect valid label indices
-        valid_label_indices = None
         if labels is not None:
-            if not labels:
-                return Filter(
-                    datum_mask=datum_mask,
-                    valid_label_indices=np.array([], dtype=np.int32),
-                    metadata=Metadata(),
+            # convert to array of valid label indices
+            if isinstance(labels, list):
+                labels = np.array(
+                    [self.label_to_index[label] for label in labels]
                 )
-            valid_label_indices = np.array(
-                [self.label_to_index[label] for label in labels] + [-1]
-            )
+
+            # return early if all data removed
+            if labels.size == 0:
+                raise EmptyFilterError("filter removes all labels")
+
+            # validate indices
+            if labels.max() >= len(self.index_to_label):
+                raise ValueError(
+                    f"label index '{labels.max()}' exceeds total number of labels"
+                )
+            elif labels.min() < 0:
+                raise ValueError(
+                    f"label index '{labels.min()}' is a negative value"
+                )
+
+            # add -1 to represent null labels which should not be filtered
+            labels = np.concatenate([labels, np.array([-1])])
 
         filtered_detailed_pairs, _ = filter_cache(
             detailed_pairs=self._detailed_pairs,
             datum_mask=datum_mask,
-            valid_label_indices=valid_label_indices,
+            valid_label_indices=labels,
             n_labels=self.metadata.number_of_labels,
         )
 
         number_of_datums = (
-            len(datum_ids)
-            if datum_ids is not None
+            datums.size
+            if datums is not None
             else self.metadata.number_of_datums
         )
 
         return Filter(
             datum_mask=datum_mask,
-            valid_label_indices=valid_label_indices,
+            valid_label_indices=labels,
             metadata=Metadata.create(
                 detailed_pairs=filtered_detailed_pairs,
                 number_of_datums=number_of_datums,
@@ -292,7 +314,6 @@ class Evaluator:
         self,
         score_thresholds: list[float] = [0.0],
         hardmax: bool = True,
-        number_of_examples: int = 0,
         filter_: Filter | None = None,
     ) -> list[Metric]:
         """
@@ -304,8 +325,6 @@ class Evaluator:
             A list of score thresholds to compute metrics over.
         hardmax : bool
             Toggles whether a hardmax is applied to predictions.
-        number_of_examples : int, default=0
-            The number of examples to return per count.
         filter_ : Filter, optional
             Applies a filter to the internal cache.
 
@@ -316,25 +335,22 @@ class Evaluator:
         """
         # apply filters
         if filter_ is not None:
-            detailed_pairs, label_metadata = self.filter(filter_=filter_)
+            detailed_pairs, _ = self.filter(filter_=filter_)
         else:
             detailed_pairs = self._detailed_pairs
-            label_metadata = self._label_metadata
 
         if detailed_pairs.size == 0:
             return list()
 
-        results = compute_confusion_matrix(
+        result = compute_confusion_matrix(
             detailed_pairs=detailed_pairs,
-            label_metadata=label_metadata,
             score_thresholds=np.array(score_thresholds),
             hardmax=hardmax,
-            n_examples=number_of_examples,
         )
         return unpack_confusion_matrix_into_metric_list(
-            results=results,
+            detailed_pairs=detailed_pairs,
+            result=result,
             score_thresholds=score_thresholds,
-            number_of_examples=number_of_examples,
             index_to_datum_id=self.index_to_datum_id,
             index_to_label=self.index_to_label,
         )
@@ -343,7 +359,6 @@ class Evaluator:
         self,
         score_thresholds: list[float] = [0.0],
         hardmax: bool = True,
-        number_of_examples: int = 0,
         filter_: Filter | None = None,
     ) -> dict[MetricType, list[Metric]]:
         """
@@ -355,8 +370,6 @@ class Evaluator:
             A list of score thresholds to compute metrics over.
         hardmax : bool
             Toggles whether a hardmax is applied to predictions.
-        number_of_examples : int, default=0
-            The number of examples to return per count.
         filter_ : Filter, optional
             Applies a filter to the internal cache.
 
@@ -373,7 +386,6 @@ class Evaluator:
         metrics[MetricType.ConfusionMatrix] = self.compute_confusion_matrix(
             score_thresholds=score_thresholds,
             hardmax=hardmax,
-            number_of_examples=number_of_examples,
             filter_=filter_,
         )
         return metrics
@@ -391,11 +403,17 @@ class Evaluator:
         -------
         int
             The datum index.
+
+        Raises
+        ------
+        ValueError
+            If datum id already exists.
         """
-        if uid not in self.datum_id_to_index:
-            index = len(self.datum_id_to_index)
-            self.datum_id_to_index[uid] = index
-            self.index_to_datum_id.append(uid)
+        if uid in self.datum_id_to_index:
+            raise ValueError("datum with id '{uid}' already exists")
+        index = len(self.datum_id_to_index)
+        self.datum_id_to_index[uid] = index
+        self.index_to_datum_id.append(uid)
         return self.datum_id_to_index[uid]
 
     def _add_label(self, label: str) -> int:
@@ -497,7 +515,7 @@ class Evaluator:
             A ready-to-use evaluator object.
         """
         if self._detailed_pairs.size == 0:
-            raise EmptyEvaluatorException()
+            raise EmptyEvaluatorError()
 
         self._label_metadata = compute_label_metadata(
             ids=self._detailed_pairs[:, :3].astype(np.int32),

{valor_lite-0.36.2 → valor_lite-0.36.4}/valor_lite/classification/metric.py

@@ -329,7 +329,6 @@ class Metric(BaseMetric):
             ],
         ],
         score_threshold: float,
-        maximum_number_of_examples: int,
     ):
         """
         The confusion matrix and related metrics for the classification task.
@@ -382,8 +381,6 @@ class Metric(BaseMetric):
             Each example includes the datum UID.
         score_threshold : float
             The confidence score threshold used to filter predictions.
-        maximum_number_of_examples : int
-            The maximum number of examples per element.
 
         Returns
         -------
@@ -397,6 +394,5 @@ class Metric(BaseMetric):
             },
             parameters={
                 "score_threshold": score_threshold,
-                "maximum_number_of_examples": maximum_number_of_examples,
             },
         )