valor-lite 0.36.2__py3-none-any.whl → 0.36.4__py3-none-any.whl

valor_lite/classification/computation.py

@@ -1,3 +1,5 @@
+from enum import IntFlag, auto
+
 import numpy as np
 from numpy.typing import NDArray
 
@@ -318,56 +320,20 @@ def compute_precision_recall_rocauc(
     )
 
 
-def _count_with_examples(
-    data: NDArray[np.float64],
-    unique_idx: int | list[int],
-    label_idx: int | list[int],
-) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.intp]]:
-    """
-    Helper function for counting occurences of unique detailed pairs.
-
-    Parameters
-    ----------
-    data : NDArray[np.float64]
-        A masked portion of a detailed pairs array.
-    unique_idx : int | list[int]
-        The index or indices upon which uniqueness is constrained.
-    label_idx : int | list[int]
-        The index or indices within the unique index or indices that encode labels.
-
-    Returns
-    -------
-    NDArray[np.float64]
-        Examples drawn from the data input.
-    NDArray[np.int32]
-        Unique label indices.
-    NDArray[np.intp]
-        Counts for each unique label index.
-    """
-    unique_rows, indices = np.unique(
-        data.astype(int)[:, unique_idx],
-        return_index=True,
-        axis=0,
-    )
-    examples = data[indices]
-    labels, counts = np.unique(
-        unique_rows[:, label_idx], return_counts=True, axis=0
-    )
-    return examples, labels, counts
+class PairClassification(IntFlag):
+    TP = auto()
+    FP_FN_MISCLF = auto()
+    FN_UNMATCHED = auto()
 
 
 def compute_confusion_matrix(
     detailed_pairs: NDArray[np.float64],
-    label_metadata: NDArray[np.int32],
     score_thresholds: NDArray[np.float64],
     hardmax: bool,
-    n_examples: int,
-) -> tuple[NDArray[np.float64], NDArray[np.int32]]:
+) -> NDArray[np.uint8]:
     """
     Compute detailed confusion matrix.
 
-    Takes data with shape (N, 5):
-
     Parameters
     ----------
     detailed_pairs : NDArray[np.float64]
@@ -377,37 +343,22 @@ def compute_confusion_matrix(
             Index 2 - Prediction Label Index
             Index 3 - Score
             Index 4 - Hard Max Score
-    label_metadata : NDArray[np.int32]
-        A 2-D array containing metadata related to labels with shape (n_labels, 2).
-            Index 0 - GroundTruth Label Count
-            Index 1 - Prediction Label Count
     iou_thresholds : NDArray[np.float64]
         A 1-D array containing IOU thresholds.
     score_thresholds : NDArray[np.float64]
         A 1-D array containing score thresholds.
-    n_examples : int
-        The maximum number of examples to return per count.
 
     Returns
     -------
-    NDArray[np.float64]
-        Confusion matrix.
-    NDArray[np.int32]
-        Unmatched Ground Truths.
+    NDArray[uint8]
+        Row-wise classification of pairs.
     """
-
-    n_labels = label_metadata.shape[0]
+    n_pairs = detailed_pairs.shape[0]
     n_scores = score_thresholds.shape[0]
 
-    confusion_matrix = np.full(
-        (n_scores, n_labels, n_labels, 2 * n_examples + 1),
-        fill_value=-1.0,
-        dtype=np.float32,
-    )
-    unmatched_ground_truths = np.full(
-        (n_scores, n_labels, n_examples + 1),
-        fill_value=-1,
-        dtype=np.int32,
+    pair_classifications = np.zeros(
+        (n_scores, n_pairs),
+        dtype=np.uint8,
     )
 
     mask_label_match = np.isclose(detailed_pairs[:, 1], detailed_pairs[:, 2])
@@ -420,9 +371,9 @@ def compute_confusion_matrix(
         if hardmax:
             mask_score &= detailed_pairs[:, 4] > 0.5
 
-        mask_tp = mask_label_match & mask_score
-        mask_misclf = ~mask_label_match & mask_score
-        mask_misprd = ~(
+        mask_true_positives = mask_label_match & mask_score
+        mask_misclassifications = ~mask_label_match & mask_score
+        mask_unmatched_groundtruths = ~(
             (
                 groundtruths.reshape(-1, 1, 2)
                 == groundtruths[mask_score].reshape(1, -1, 2)
@@ -431,73 +382,15 @@ def compute_confusion_matrix(
             .any(axis=1)
         )
 
-        tp_examples, tp_labels, tp_counts = _count_with_examples(
-            data=detailed_pairs[mask_tp],
-            unique_idx=[0, 2],
-            label_idx=1,
-        )
-        misclf_examples, misclf_labels, misclf_counts = _count_with_examples(
-            data=detailed_pairs[mask_misclf],
-            unique_idx=[0, 1, 2],
-            label_idx=[1, 2],
+        # classify pairings
+        pair_classifications[score_idx, mask_true_positives] |= np.uint8(
+            PairClassification.TP
         )
-        misprd_examples, misprd_labels, misprd_counts = _count_with_examples(
-            data=detailed_pairs[mask_misprd],
-            unique_idx=[0, 1],
-            label_idx=1,
+        pair_classifications[score_idx, mask_misclassifications] |= np.uint8(
+            PairClassification.FP_FN_MISCLF
         )
+        pair_classifications[
+            score_idx, mask_unmatched_groundtruths
+        ] |= np.uint8(PairClassification.FN_UNMATCHED)
 
-        confusion_matrix[score_idx, tp_labels, tp_labels, 0] = tp_counts
-        confusion_matrix[
-            score_idx, misclf_labels[:, 0], misclf_labels[:, 1], 0
-        ] = misclf_counts
-
-        unmatched_ground_truths[score_idx, misprd_labels, 0] = misprd_counts
-
-        if n_examples > 0:
-            for label_idx in range(n_labels):
-                # true-positive examples
-                mask_tp_label = tp_examples[:, 2] == label_idx
-                if mask_tp_label.sum() > 0:
-                    tp_label_examples = tp_examples[mask_tp_label][:n_examples]
-                    confusion_matrix[
-                        score_idx,
-                        label_idx,
-                        label_idx,
-                        1 : 2 * tp_label_examples.shape[0] + 1,
-                    ] = tp_label_examples[:, [0, 3]].flatten()
-
-                # misclassification examples
-                mask_misclf_gt_label = misclf_examples[:, 1] == label_idx
-                if mask_misclf_gt_label.sum() > 0:
-                    for pd_label_idx in range(n_labels):
-                        mask_misclf_pd_label = (
-                            misclf_examples[:, 2] == pd_label_idx
-                        )
-                        mask_misclf_label_combo = (
-                            mask_misclf_gt_label & mask_misclf_pd_label
-                        )
-                        if mask_misclf_label_combo.sum() > 0:
-                            misclf_label_examples = misclf_examples[
-                                mask_misclf_label_combo
-                            ][:n_examples]
-                            confusion_matrix[
-                                score_idx,
-                                label_idx,
-                                pd_label_idx,
-                                1 : 2 * misclf_label_examples.shape[0] + 1,
-                            ] = misclf_label_examples[:, [0, 3]].flatten()
-
-                # unmatched ground truth examples
-                mask_misprd_label = misprd_examples[:, 1] == label_idx
-                if misprd_examples.size > 0:
-                    misprd_label_examples = misprd_examples[mask_misprd_label][
-                        :n_examples
-                    ]
-                    unmatched_ground_truths[
-                        score_idx,
-                        label_idx,
-                        1 : misprd_label_examples.shape[0] + 1,
-                    ] = misprd_label_examples[:, 0].flatten()
-
-    return confusion_matrix, unmatched_ground_truths  # type: ignore[reportReturnType]
+    return pair_classifications

valor_lite/classification/manager.py

@@ -16,7 +16,7 @@ from valor_lite.classification.utilities import (
     unpack_confusion_matrix_into_metric_list,
     unpack_precision_recall_rocauc_into_metric_lists,
 )
-from valor_lite.exceptions import EmptyEvaluatorException, EmptyFilterException
+from valor_lite.exceptions import EmptyEvaluatorError, EmptyFilterError
 
 """
 Usage
@@ -88,14 +88,14 @@ class Filter:
     def __post_init__(self):
         # validate datum mask
         if not self.datum_mask.any():
-            raise EmptyFilterException("filter removes all datums")
+            raise EmptyFilterError("filter removes all datums")
 
         # validate label indices
         if (
             self.valid_label_indices is not None
             and self.valid_label_indices.size == 0
         ):
-            raise EmptyFilterException("filter removes all labels")
+            raise EmptyFilterError("filter removes all labels")
 
 
 class Evaluator:
@@ -144,18 +144,18 @@ class Evaluator:
 
     def create_filter(
         self,
-        datum_ids: list[str] | None = None,
-        labels: list[str] | None = None,
+        datums: list[str] | NDArray[np.int32] | None = None,
+        labels: list[str] | NDArray[np.int32] | None = None,
     ) -> Filter:
         """
         Creates a filter object.
 
         Parameters
         ----------
-        datum_uids : list[str], optional
-            An optional list of string uids representing datums.
-        labels : list[str], optional
-            An optional list of labels.
+        datums : list[str] | NDArray[int32], optional
+            An optional list of string uids or integer indices representing datums.
+        labels : list[str] | NDArray[int32], optional
+            An optional list of strings or integer indices representing labels.
 
         Returns
         -------
@@ -165,50 +165,72 @@ class Evaluator:
         # create datum mask
         n_pairs = self._detailed_pairs.shape[0]
         datum_mask = np.ones(n_pairs, dtype=np.bool_)
-        if datum_ids is not None:
-            if not datum_ids:
-                return Filter(
-                    datum_mask=np.zeros_like(datum_mask),
-                    valid_label_indices=None,
-                    metadata=Metadata(),
+        if datums is not None:
+            # convert to array of valid datum indices
+            if isinstance(datums, list):
+                datums = np.array(
+                    [self.datum_id_to_index[uid] for uid in datums],
+                    dtype=np.int32,
                 )
-            valid_datum_indices = np.array(
-                [self.datum_id_to_index[uid] for uid in datum_ids],
-                dtype=np.int32,
-            )
-            datum_mask = np.isin(
-                self._detailed_pairs[:, 0], valid_datum_indices
-            )
+
+            # return early if all data removed
+            if datums.size == 0:
+                raise EmptyFilterError("filter removes all datums")
+
+            # validate indices
+            if datums.max() >= len(self.index_to_datum_id):
+                raise ValueError(
+                    f"datum index '{datums.max()}' exceeds total number of datums"
+                )
+            elif datums.min() < 0:
+                raise ValueError(
+                    f"datum index '{datums.min()}' is a negative value"
+                )
+
+            # create datum mask
+            datum_mask = np.isin(self._detailed_pairs[:, 0], datums)
 
         # collect valid label indices
-        valid_label_indices = None
         if labels is not None:
-            if not labels:
-                return Filter(
-                    datum_mask=datum_mask,
-                    valid_label_indices=np.array([], dtype=np.int32),
-                    metadata=Metadata(),
+            # convert to array of valid label indices
+            if isinstance(labels, list):
+                labels = np.array(
+                    [self.label_to_index[label] for label in labels]
                 )
-            valid_label_indices = np.array(
-                [self.label_to_index[label] for label in labels] + [-1]
-            )
+
+            # return early if all data removed
+            if labels.size == 0:
+                raise EmptyFilterError("filter removes all labels")
+
+            # validate indices
+            if labels.max() >= len(self.index_to_label):
+                raise ValueError(
+                    f"label index '{labels.max()}' exceeds total number of labels"
+                )
+            elif labels.min() < 0:
+                raise ValueError(
+                    f"label index '{labels.min()}' is a negative value"
+                )
+
+            # add -1 to represent null labels which should not be filtered
+            labels = np.concatenate([labels, np.array([-1])])
 
         filtered_detailed_pairs, _ = filter_cache(
             detailed_pairs=self._detailed_pairs,
             datum_mask=datum_mask,
-            valid_label_indices=valid_label_indices,
+            valid_label_indices=labels,
             n_labels=self.metadata.number_of_labels,
         )
 
         number_of_datums = (
-            len(datum_ids)
-            if datum_ids is not None
+            datums.size
+            if datums is not None
             else self.metadata.number_of_datums
         )
 
         return Filter(
             datum_mask=datum_mask,
-            valid_label_indices=valid_label_indices,
+            valid_label_indices=labels,
             metadata=Metadata.create(
                 detailed_pairs=filtered_detailed_pairs,
                 number_of_datums=number_of_datums,
@@ -292,7 +314,6 @@ class Evaluator:
         self,
         score_thresholds: list[float] = [0.0],
         hardmax: bool = True,
-        number_of_examples: int = 0,
         filter_: Filter | None = None,
     ) -> list[Metric]:
         """
@@ -304,8 +325,6 @@ class Evaluator:
             A list of score thresholds to compute metrics over.
         hardmax : bool
             Toggles whether a hardmax is applied to predictions.
-        number_of_examples : int, default=0
-            The number of examples to return per count.
         filter_ : Filter, optional
             Applies a filter to the internal cache.
 
@@ -316,25 +335,22 @@ class Evaluator:
         """
         # apply filters
         if filter_ is not None:
-            detailed_pairs, label_metadata = self.filter(filter_=filter_)
+            detailed_pairs, _ = self.filter(filter_=filter_)
         else:
             detailed_pairs = self._detailed_pairs
-            label_metadata = self._label_metadata
 
         if detailed_pairs.size == 0:
             return list()
 
-        results = compute_confusion_matrix(
+        result = compute_confusion_matrix(
             detailed_pairs=detailed_pairs,
-            label_metadata=label_metadata,
             score_thresholds=np.array(score_thresholds),
             hardmax=hardmax,
-            n_examples=number_of_examples,
         )
         return unpack_confusion_matrix_into_metric_list(
-            results=results,
+            detailed_pairs=detailed_pairs,
+            result=result,
             score_thresholds=score_thresholds,
-            number_of_examples=number_of_examples,
             index_to_datum_id=self.index_to_datum_id,
             index_to_label=self.index_to_label,
         )
@@ -343,7 +359,6 @@ class Evaluator:
         self,
         score_thresholds: list[float] = [0.0],
         hardmax: bool = True,
-        number_of_examples: int = 0,
         filter_: Filter | None = None,
     ) -> dict[MetricType, list[Metric]]:
         """
@@ -355,8 +370,6 @@ class Evaluator:
             A list of score thresholds to compute metrics over.
         hardmax : bool
             Toggles whether a hardmax is applied to predictions.
-        number_of_examples : int, default=0
-            The number of examples to return per count.
         filter_ : Filter, optional
             Applies a filter to the internal cache.
 
@@ -373,7 +386,6 @@ class Evaluator:
         metrics[MetricType.ConfusionMatrix] = self.compute_confusion_matrix(
             score_thresholds=score_thresholds,
             hardmax=hardmax,
-            number_of_examples=number_of_examples,
             filter_=filter_,
         )
         return metrics
@@ -391,11 +403,17 @@ class Evaluator:
         -------
         int
             The datum index.
+
+        Raises
+        ------
+        ValueError
+            If datum id already exists.
         """
-        if uid not in self.datum_id_to_index:
-            index = len(self.datum_id_to_index)
-            self.datum_id_to_index[uid] = index
-            self.index_to_datum_id.append(uid)
+        if uid in self.datum_id_to_index:
+            raise ValueError("datum with id '{uid}' already exists")
+        index = len(self.datum_id_to_index)
+        self.datum_id_to_index[uid] = index
+        self.index_to_datum_id.append(uid)
         return self.datum_id_to_index[uid]
 
     def _add_label(self, label: str) -> int:
@@ -497,7 +515,7 @@ class Evaluator:
             A ready-to-use evaluator object.
         """
         if self._detailed_pairs.size == 0:
-            raise EmptyEvaluatorException()
+            raise EmptyEvaluatorError()
 
         self._label_metadata = compute_label_metadata(
             ids=self._detailed_pairs[:, :3].astype(np.int32),

valor_lite/classification/metric.py

@@ -329,7 +329,6 @@ class Metric(BaseMetric):
             ],
         ],
         score_threshold: float,
-        maximum_number_of_examples: int,
     ):
         """
         The confusion matrix and related metrics for the classification task.
@@ -382,8 +381,6 @@ class Metric(BaseMetric):
             Each example includes the datum UID.
         score_threshold : float
             The confidence score threshold used to filter predictions.
-        maximum_number_of_examples : int
-            The maximum number of examples per element.
 
         Returns
         -------
@@ -397,6 +394,5 @@ class Metric(BaseMetric):
             },
             parameters={
                 "score_threshold": score_threshold,
-                "maximum_number_of_examples": maximum_number_of_examples,
             },
         )

valor_lite/classification/utilities.py

@@ -3,6 +3,7 @@ from collections import defaultdict
 import numpy as np
 from numpy.typing import NDArray
 
+from valor_lite.classification.computation import PairClassification
 from valor_lite.classification.metric import Metric, MetricType
 
 
@@ -101,129 +102,110 @@ def unpack_precision_recall_rocauc_into_metric_lists(
     return metrics
 
 
-def _unpack_confusion_matrix_value(
-    confusion_matrix: NDArray[np.float64],
-    number_of_labels: int,
-    number_of_examples: int,
-    index_to_datum_id: list[str],
-    index_to_label: list[str],
-) -> dict[str, dict[str, dict[str, int | list[dict[str, str | float]]]]]:
-    """
-    Unpacks a numpy array of confusion matrix counts and examples.
-    """
-
-    datum_idx = lambda gt_label_idx, pd_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-        confusion_matrix[
-            gt_label_idx,
-            pd_label_idx,
-            example_idx * 2 + 1,
-        ]
-    )
-
-    score_idx = lambda gt_label_idx, pd_label_idx, example_idx: float(  # noqa: E731 - lambda fn
-        confusion_matrix[
-            gt_label_idx,
-            pd_label_idx,
-            example_idx * 2 + 2,
-        ]
+def _create_empty_confusion_matrix(index_to_labels: list[str]):
+    unmatched_ground_truths = dict()
+    confusion_matrix = dict()
+    for label in index_to_labels:
+        unmatched_ground_truths[label] = {"count": 0, "examples": []}
+        confusion_matrix[label] = {}
+        for plabel in index_to_labels:
+            confusion_matrix[label][plabel] = {"count": 0, "examples": []}
+    return (
+        confusion_matrix,
+        unmatched_ground_truths,
     )
 
-    return {
-        index_to_label[gt_label_idx]: {
-            index_to_label[pd_label_idx]: {
-                "count": max(
-                    int(confusion_matrix[gt_label_idx, pd_label_idx, 0]),
-                    0,
-                ),
-                "examples": [
-                    {
-                        "datum_id": index_to_datum_id[
-                            datum_idx(gt_label_idx, pd_label_idx, example_idx)
-                        ],
-                        "score": score_idx(
-                            gt_label_idx, pd_label_idx, example_idx
-                        ),
-                    }
-                    for example_idx in range(number_of_examples)
-                    if datum_idx(gt_label_idx, pd_label_idx, example_idx) >= 0
-                ],
-            }
-            for pd_label_idx in range(number_of_labels)
-        }
-        for gt_label_idx in range(number_of_labels)
-    }
-
 
-def _unpack_unmatched_ground_truths_value(
-    unmatched_ground_truths: NDArray[np.int32],
-    number_of_labels: int,
-    number_of_examples: int,
+def _unpack_confusion_matrix(
+    ids: NDArray[np.int32],
+    scores: NDArray[np.float64],
+    mask_matched: NDArray[np.bool_],
+    mask_fn_unmatched: NDArray[np.bool_],
     index_to_datum_id: list[str],
     index_to_label: list[str],
-) -> dict[str, dict[str, int | list[dict[str, str]]]]:
-    """
-    Unpacks a numpy array of unmatched ground truth counts and examples.
-    """
-
-    datum_idx = (
-        lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            unmatched_ground_truths[
-                gt_label_idx,
-                example_idx + 1,
-            ]
-        )
+    score_threshold: float,
+):
+    (
+        confusion_matrix,
+        unmatched_ground_truths,
+    ) = _create_empty_confusion_matrix(index_to_label)
+
+    unique_matches, unique_match_indices = np.unique(
+        ids[np.ix_(mask_matched, (0, 1, 2))],  # type: ignore - numpy ix_ typing
+        axis=0,
+        return_index=True,
     )
+    (
+        unique_unmatched_groundtruths,
+        unique_unmatched_groundtruth_indices,
+    ) = np.unique(
+        ids[np.ix_(mask_fn_unmatched, (0, 1))],  # type: ignore - numpy ix_ typing
+        axis=0,
+        return_index=True,
+    )
+
+    n_matched = unique_matches.shape[0]
+    n_unmatched_groundtruths = unique_unmatched_groundtruths.shape[0]
+    n_max = max(n_matched, n_unmatched_groundtruths)
 
-    return {
-        index_to_label[gt_label_idx]: {
-            "count": max(
-                int(unmatched_ground_truths[gt_label_idx, 0]),
-                0,
-            ),
-            "examples": [
+    for idx in range(n_max):
+        if idx < n_matched:
+            glabel = index_to_label[unique_matches[idx, 1]]
+            plabel = index_to_label[unique_matches[idx, 2]]
+            confusion_matrix[glabel][plabel]["count"] += 1
+            confusion_matrix[glabel][plabel]["examples"].append(
+                {
+                    "datum_id": index_to_datum_id[unique_matches[idx, 0]],
+                    "score": float(scores[unique_match_indices[idx]]),
+                }
+            )
+        if idx < n_unmatched_groundtruths:
+            label = index_to_label[unique_unmatched_groundtruths[idx, 1]]
+            unmatched_ground_truths[label]["count"] += 1
+            unmatched_ground_truths[label]["examples"].append(
                 {
                     "datum_id": index_to_datum_id[
-                        datum_idx(gt_label_idx, example_idx)
-                    ]
+                        unique_unmatched_groundtruths[idx, 0]
+                    ],
                 }
-                for example_idx in range(number_of_examples)
-                if datum_idx(gt_label_idx, example_idx) >= 0
-            ],
-        }
-        for gt_label_idx in range(number_of_labels)
-    }
+            )
+
+    return Metric.confusion_matrix(
+        confusion_matrix=confusion_matrix,
+        unmatched_ground_truths=unmatched_ground_truths,
+        score_threshold=score_threshold,
+    )
 
 
 def unpack_confusion_matrix_into_metric_list(
-    results: tuple[NDArray[np.float64], NDArray[np.int32]],
+    result: NDArray[np.uint8],
+    detailed_pairs: NDArray[np.float64],
     score_thresholds: list[float],
-    number_of_examples: int,
     index_to_datum_id: list[str],
     index_to_label: list[str],
 ) -> list[Metric]:
 
-    (confusion_matrix, unmatched_ground_truths) = results
-    _, n_labels, _, _ = confusion_matrix.shape
+    ids = detailed_pairs[:, :3].astype(np.int32)
+
+    mask_matched = (
+        np.bitwise_and(
+            result, PairClassification.TP | PairClassification.FP_FN_MISCLF
+        )
+        > 0
+    )
+    mask_fn_unmatched = (
+        np.bitwise_and(result, PairClassification.FN_UNMATCHED) > 0
+    )
+
     return [
-        Metric.confusion_matrix(
+        _unpack_confusion_matrix(
+            ids=ids,
+            scores=detailed_pairs[:, 3],
+            mask_matched=mask_matched[score_idx, :],
+            mask_fn_unmatched=mask_fn_unmatched[score_idx, :],
+            index_to_datum_id=index_to_datum_id,
+            index_to_label=index_to_label,
             score_threshold=score_threshold,
-            maximum_number_of_examples=number_of_examples,
-            confusion_matrix=_unpack_confusion_matrix_value(
-                confusion_matrix=confusion_matrix[score_idx, :, :, :],
-                number_of_labels=n_labels,
-                number_of_examples=number_of_examples,
-                index_to_label=index_to_label,
-                index_to_datum_id=index_to_datum_id,
-            ),
-            unmatched_ground_truths=_unpack_unmatched_ground_truths_value(
-                unmatched_ground_truths=unmatched_ground_truths[
-                    score_idx, :, :
-                ],
-                number_of_labels=n_labels,
-                number_of_examples=number_of_examples,
-                index_to_label=index_to_label,
-                index_to_datum_id=index_to_datum_id,
-            ),
         )
         for score_idx, score_threshold in enumerate(score_thresholds)
     ]

valor_lite/exceptions.py

@@ -1,15 +1,15 @@
-class EmptyEvaluatorException(Exception):
+class EmptyEvaluatorError(Exception):
     def __init__(self):
         super().__init__(
             "evaluator cannot be finalized as it contains no data"
         )
 
 
-class EmptyFilterException(Exception):
+class EmptyFilterError(Exception):
     def __init__(self, message: str):
         super().__init__(message)
 
 
-class InternalCacheException(Exception):
+class InternalCacheError(Exception):
     def __init__(self, message: str):
         super().__init__(message)

valor_lite/object_detection/manager.py

@@ -6,9 +6,9 @@ from numpy.typing import NDArray
 from tqdm import tqdm
 
 from valor_lite.exceptions import (
-    EmptyEvaluatorException,
-    EmptyFilterException,
-    InternalCacheException,
+    EmptyEvaluatorError,
+    EmptyFilterError,
+    InternalCacheError,
 )
 from valor_lite.object_detection.annotation import (
     Bitmask,
@@ -102,13 +102,13 @@ class Filter:
     def __post_init__(self):
         # validate datums mask
         if not self.mask_datums.any():
-            raise EmptyFilterException("filter removes all datums")
+            raise EmptyFilterError("filter removes all datums")
 
         # validate annotation masks
         no_gts = self.mask_groundtruths.all()
         no_pds = self.mask_predictions.all()
         if no_gts and no_pds:
-            raise EmptyFilterException("filter removes all annotations")
+            raise EmptyFilterError("filter removes all annotations")
         elif no_gts:
             warnings.warn("filter removes all ground truths")
         elif no_pds:
@@ -173,38 +173,52 @@ class Evaluator:
 
     def create_filter(
         self,
-        datum_ids: list[str] | None = None,
-        groundtruth_ids: list[str] | None = None,
-        prediction_ids: list[str] | None = None,
-        labels: list[str] | None = None,
+        datums: list[str] | NDArray[np.int32] | None = None,
+        groundtruths: list[str] | NDArray[np.int32] | None = None,
+        predictions: list[str] | NDArray[np.int32] | None = None,
+        labels: list[str] | NDArray[np.int32] | None = None,
     ) -> Filter:
         """
         Creates a filter object.
 
         Parameters
         ----------
-        datum_uids : list[str], optional
-            An optional list of string uids representing datums to keep.
-        groundtruth_ids : list[str], optional
-            An optional list of string uids representing ground truth annotations to keep.
-        prediction_ids : list[str], optional
-            An optional list of string uids representing prediction annotations to keep.
-        labels : list[str], optional
-            An optional list of labels to keep.
+        datum : list[str] | NDArray[int32], optional
+            An optional list of string ids or indices representing datums to keep.
+        groundtruth : list[str] | NDArray[int32], optional
+            An optional list of string ids or indices representing ground truth annotations to keep.
+        prediction : list[str] | NDArray[int32], optional
+            An optional list of string ids or indices representing prediction annotations to keep.
+        labels : list[str] | NDArray[int32], optional
+            An optional list of labels or indices to keep.
         """
         mask_datums = np.ones(self._detailed_pairs.shape[0], dtype=np.bool_)
 
         # filter datums
-        if datum_ids is not None:
-            if not datum_ids:
-                raise EmptyFilterException("filter removes all datums")
-            valid_datum_indices = np.array(
-                [self.datum_id_to_index[uid] for uid in datum_ids],
-                dtype=np.int32,
-            )
-            mask_datums = np.isin(
-                self._detailed_pairs[:, 0], valid_datum_indices
-            )
+        if datums is not None:
+            # convert to indices
+            if isinstance(datums, list):
+                datums = np.array(
+                    [self.datum_id_to_index[uid] for uid in datums],
+                    dtype=np.int32,
+                )
+
+            # validate indices
+            if datums.size == 0:
+                raise EmptyFilterError(
+                    "filter removes all datums"
+                )  # validate indices
+            elif datums.min() < 0:
+                raise ValueError(
+                    f"datum index cannot be negative '{datums.min()}'"
+                )
+            elif datums.max() >= len(self.index_to_datum_id):
+                raise ValueError(
+                    f"datum index cannot exceed total number of datums '{datums.max()}'"
+                )
+
+            # apply to mask
+            mask_datums = np.isin(self._detailed_pairs[:, 0], datums)
 
         filtered_detailed_pairs = self._detailed_pairs[mask_datums]
         n_pairs = self._detailed_pairs[mask_datums].shape[0]
@@ -212,43 +226,93 @@ class Evaluator:
         mask_predictions = np.zeros_like(mask_groundtruths)
 
         # filter by ground truth annotation ids
-        if groundtruth_ids is not None:
-            valid_groundtruth_indices = np.array(
-                [self.groundtruth_id_to_index[uid] for uid in groundtruth_ids],
-                dtype=np.int32,
-            )
+        if groundtruths is not None:
+            # convert to indices
+            if isinstance(groundtruths, list):
+                groundtruths = np.array(
+                    [
+                        self.groundtruth_id_to_index[uid]
+                        for uid in groundtruths
+                    ],
+                    dtype=np.int32,
+                )
+
+            # validate indices
+            if groundtruths.size == 0:
+                warnings.warn("filter removes all ground truths")
+            elif groundtruths.min() < 0:
+                raise ValueError(
+                    f"groundtruth annotation index cannot be negative '{groundtruths.min()}'"
+                )
+            elif groundtruths.max() >= len(self.index_to_groundtruth_id):
+                raise ValueError(
+                    f"groundtruth annotation index cannot exceed total number of groundtruths '{groundtruths.max()}'"
+                )
+
+            # apply to mask
             mask_groundtruths[
                 ~np.isin(
                     filtered_detailed_pairs[:, 1],
-                    valid_groundtruth_indices,
+                    groundtruths,
                 )
             ] = True
 
         # filter by prediction annotation ids
-        if prediction_ids is not None:
-            valid_prediction_indices = np.array(
-                [self.prediction_id_to_index[uid] for uid in prediction_ids],
-                dtype=np.int32,
-            )
+        if predictions is not None:
+            # convert to indices
+            if isinstance(predictions, list):
+                predictions = np.array(
+                    [self.prediction_id_to_index[uid] for uid in predictions],
+                    dtype=np.int32,
+                )
+
+            # validate indices
+            if predictions.size == 0:
+                warnings.warn("filter removes all predictions")
+            elif predictions.min() < 0:
+                raise ValueError(
+                    f"prediction annotation index cannot be negative '{predictions.min()}'"
+                )
+            elif predictions.max() >= len(self.index_to_prediction_id):
+                raise ValueError(
+                    f"prediction annotation index cannot exceed total number of predictions '{predictions.max()}'"
+                )
+
+            # apply to mask
             mask_predictions[
                 ~np.isin(
                     filtered_detailed_pairs[:, 2],
-                    valid_prediction_indices,
+                    predictions,
                 )
             ] = True
 
         # filter by labels
         if labels is not None:
-            if not labels:
-                raise EmptyFilterException("filter removes all labels")
-            valid_label_indices = np.array(
-                [self.label_to_index[label] for label in labels] + [-1]
-            )
+            # convert to indices
+            if isinstance(labels, list):
+                labels = np.array(
+                    [self.label_to_index[label] for label in labels]
+                )
+
+            # validate indices
+            if labels.size == 0:
+                raise EmptyFilterError("filter removes all labels")
+            elif labels.min() < 0:
+                raise ValueError(
+                    f"label index cannot be negative '{labels.min()}'"
+                )
+            elif labels.max() >= len(self.index_to_label):
+                raise ValueError(
+                    f"label index cannot exceed total number of labels '{labels.max()}'"
+                )
+
+            # apply to mask
+            labels = np.concatenate([labels, np.array([-1])])  # add null label
             mask_groundtruths[
-                ~np.isin(filtered_detailed_pairs[:, 3], valid_label_indices)
+                ~np.isin(filtered_detailed_pairs[:, 3], labels)
             ] = True
             mask_predictions[
-                ~np.isin(filtered_detailed_pairs[:, 4], valid_label_indices)
+                ~np.isin(filtered_detailed_pairs[:, 4], labels)
             ] = True
 
         filtered_detailed_pairs, _, _ = filter_cache(
@@ -260,8 +324,8 @@ class Evaluator:
         )
 
         number_of_datums = (
-            len(datum_ids)
-            if datum_ids
+            datums.size
+            if datums is not None
             else np.unique(filtered_detailed_pairs[:, 0]).size
         )
 
@@ -467,7 +531,7 @@ class DataLoader:
             if len(self._evaluator.datum_id_to_index) != len(
                 self._evaluator.index_to_datum_id
             ):
-                raise InternalCacheException("datum cache size mismatch")
+                raise InternalCacheError("datum cache size mismatch")
             idx = len(self._evaluator.datum_id_to_index)
             self._evaluator.datum_id_to_index[datum_id] = idx
             self._evaluator.index_to_datum_id.append(datum_id)
@@ -491,9 +555,7 @@ class DataLoader:
             if len(self._evaluator.groundtruth_id_to_index) != len(
                 self._evaluator.index_to_groundtruth_id
             ):
-                raise InternalCacheException(
-                    "ground truth cache size mismatch"
-                )
+                raise InternalCacheError("ground truth cache size mismatch")
             idx = len(self._evaluator.groundtruth_id_to_index)
             self._evaluator.groundtruth_id_to_index[annotation_id] = idx
             self._evaluator.index_to_groundtruth_id.append(annotation_id)
@@ -517,7 +579,7 @@ class DataLoader:
             if len(self._evaluator.prediction_id_to_index) != len(
                 self._evaluator.index_to_prediction_id
             ):
-                raise InternalCacheException("prediction cache size mismatch")
+                raise InternalCacheError("prediction cache size mismatch")
             idx = len(self._evaluator.prediction_id_to_index)
             self._evaluator.prediction_id_to_index[annotation_id] = idx
             self._evaluator.index_to_prediction_id.append(annotation_id)
@@ -542,7 +604,7 @@ class DataLoader:
             if len(self._evaluator.label_to_index) != len(
                 self._evaluator.index_to_label
             ):
-                raise InternalCacheException("label cache size mismatch")
+                raise InternalCacheError("label cache size mismatch")
             self._evaluator.label_to_index[label] = label_id
             self._evaluator.index_to_label.append(label)
             label_id += 1
@@ -768,14 +830,14 @@ class DataLoader:
             A ready-to-use evaluator object.
         """
         if not self.pairs:
-            raise EmptyEvaluatorException()
+            raise EmptyEvaluatorError()
 
         n_labels = len(self._evaluator.index_to_label)
         n_datums = len(self._evaluator.index_to_datum_id)
 
         self._evaluator._detailed_pairs = np.concatenate(self.pairs, axis=0)
         if self._evaluator._detailed_pairs.size == 0:
-            raise EmptyEvaluatorException()
+            raise EmptyEvaluatorError()
 
         # order pairs by descending score, iou
         indices = np.lexsort(

valor_lite/semantic_segmentation/manager.py

@@ -4,7 +4,7 @@ import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
 
-from valor_lite.exceptions import EmptyEvaluatorException, EmptyFilterException
+from valor_lite.exceptions import EmptyEvaluatorError, EmptyFilterError
 from valor_lite.semantic_segmentation.annotation import Segmentation
 from valor_lite.semantic_segmentation.computation import (
     compute_intermediate_confusion_matrices,
@@ -74,11 +74,11 @@ class Filter:
     def __post_init__(self):
         # validate datum mask
         if not self.datum_mask.any():
-            raise EmptyFilterException("filter removes all datums")
+            raise EmptyFilterError("filter removes all datums")
 
         # validate label mask
         if self.label_mask.all():
-            raise EmptyFilterException("filter removes all labels")
+            raise EmptyFilterError("filter removes all labels")
 
 
 class Evaluator:
@@ -127,18 +127,18 @@ class Evaluator:
 
     def create_filter(
         self,
-        datum_ids: list[str] | None = None,
-        labels: list[str] | None = None,
+        datums: list[str] | NDArray[np.int64] | None = None,
+        labels: list[str] | NDArray[np.int64] | None = None,
     ) -> Filter:
         """
         Creates a filter for use with the evaluator.
 
         Parameters
         ----------
-        datum_ids : list[str], optional
-            An optional list of string uids representing datums.
-        labels : list[str], optional
-            An optional list of labels.
+        datums : list[str] | NDArray[int64], optional
+            An optional list of string ids or array of indices representing datums.
+        labels : list[str] | NDArray[int64], optional
+            An optional list of labels or array of indices.
 
         Returns
         -------
@@ -150,38 +150,61 @@ class Evaluator:
             self.metadata.number_of_labels + 1, dtype=np.bool_
         )
 
-        if datum_ids is not None:
-            if not datum_ids:
-                return Filter(
-                    datum_mask=np.zeros_like(datum_mask),
-                    label_mask=label_mask,
-                    metadata=Metadata(),
+        if datums is not None:
+            # convert to indices
+            if isinstance(datums, list):
+                datums = np.array(
+                    [self.datum_id_to_index[uid] for uid in datums],
+                    dtype=np.int64,
                 )
-            datum_id_array = np.array(
-                [self.datum_id_to_index[uid] for uid in datum_ids],
-                dtype=np.int64,
-            )
-            datum_id_array.sort()
+
+            # validate indices
+            if datums.size == 0:
+                raise EmptyFilterError(
+                    "filter removes all datums"
+                )  # validate indices
+            elif datums.min() < 0:
+                raise ValueError(
+                    f"datum index cannot be negative '{datums.min()}'"
+                )
+            elif datums.max() >= len(self.index_to_datum_id):
+                raise ValueError(
+                    f"datum index cannot exceed total number of datums '{datums.max()}'"
+                )
+
+            # apply to mask
+            datums.sort()
             mask_valid_datums = (
                 np.arange(self._confusion_matrices.shape[0]).reshape(-1, 1)
-                == datum_id_array.reshape(1, -1)
+                == datums.reshape(1, -1)
             ).any(axis=1)
             datum_mask[~mask_valid_datums] = False
 
         if labels is not None:
-            if not labels:
-                return Filter(
-                    datum_mask=datum_mask,
-                    label_mask=np.ones_like(label_mask),
-                    metadata=Metadata(),
+            # convert to indices
+            if isinstance(labels, list):
+                labels = np.array(
+                    [self.label_to_index[label] for label in labels],
+                    dtype=np.int64,
                 )
-            labels_id_array = np.array(
-                [self.label_to_index[label] for label in labels] + [-1],
-                dtype=np.int64,
-            )
+
+            # validate indices
+            if labels.size == 0:
+                raise EmptyFilterError("filter removes all labels")
+            elif labels.min() < 0:
+                raise ValueError(
+                    f"label index cannot be negative '{labels.min()}'"
+                )
+            elif labels.max() >= len(self.index_to_label):
+                raise ValueError(
+                    f"label index cannot exceed total number of labels '{labels.max()}'"
+                )
+
+            # apply to mask
+            labels = np.concatenate([labels, np.array([-1])])
             label_range = np.arange(self.metadata.number_of_labels + 1) - 1
             mask_valid_labels = (
-                label_range.reshape(-1, 1) == labels_id_array.reshape(1, -1)
+                label_range.reshape(-1, 1) == labels.reshape(1, -1)
             ).any(axis=1)
             label_mask[~mask_valid_labels] = True
 
@@ -403,7 +426,7 @@ class DataLoader:
         """
 
         if len(self.matrices) == 0:
-            raise EmptyEvaluatorException()
+            raise EmptyEvaluatorError()
 
         n_labels = len(self._evaluator.index_to_label)
         n_datums = len(self._evaluator.index_to_datum_id)

{valor_lite-0.36.2.dist-info → valor_lite-0.36.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: valor-lite
-Version: 0.36.2
+Version: 0.36.4
 Summary: Evaluate machine learning models.
 Project-URL: homepage, https://www.striveworks.com
 Requires-Python: >=3.10

{valor_lite-0.36.2.dist-info → valor_lite-0.36.4.dist-info}/RECORD

@@ -1,26 +1,26 @@
 valor_lite/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
 valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valor_lite/exceptions.py,sha256=S32XtowA3cUtErZMOXFKl73jquzBH-h8l2LdMYaCSnI,397
+valor_lite/exceptions.py,sha256=Q0PLMu0PnCPBx438iEPzpOQyMOcMOA3lOf5xQZP_yYU,385
 valor_lite/profiling.py,sha256=TLIROA1qccFw9NoEkMeQcrvvGGO75c4K5yTIWoCUix8,11746
 valor_lite/schemas.py,sha256=pB0MrPx5qFLbwBWDiOUUm-vmXdWvbJLFCBmKgbcbI5g,198
 valor_lite/classification/__init__.py,sha256=KXaVwyqAbeeeEq7bzNPyt4GTpbxhrABjV7lR58KR6Y4,440
 valor_lite/classification/annotation.py,sha256=0aUOvcwBAZgiNOJuyh-pXyNTG7vP7r8CUfnU3OmpUwQ,1113
-valor_lite/classification/computation.py,sha256=B5Y5K_ksbRnCCvkemYb23PKojKmhvSb2sF5JWpIdgD8,16271
-valor_lite/classification/manager.py,sha256=7yPcjyHLVAimTRwQla0IkRZXP6GxhOyrUjv0TAwvEDo,16267
-valor_lite/classification/metric.py,sha256=BJn82GZ7h-350ugXdRKYNPczidtjW_dvdNE194_i7BM,11905
+valor_lite/classification/computation.py,sha256=kB5n-RHzDsKG75Guvgg25xAOeLEQCq1TgjwHwfwbQ60,12010
+valor_lite/classification/manager.py,sha256=JZwA9sf-OG7p7uK5qIo-D711kSpBDDeTcXsPr1uuIBI,16884
+valor_lite/classification/metric.py,sha256=nSNWjoxQ1ou7gxTPOYxLNoUYf7avKQzJq3NHR9jzM48,11693
 valor_lite/classification/numpy_compatibility.py,sha256=roqtTetsm1_HxuaejrthQdydjsRIy-FpXpGb86cLh_E,365
-valor_lite/classification/utilities.py,sha256=awKz-OdT2y5ydJuwDi5Y9FCFL1gdWyiGBZpqQxWER2A,7063
+valor_lite/classification/utilities.py,sha256=jAcir7dW-o4I2gk_NEmlRr8j8Iniyyq9QT5j3PMxVHk,6435
 valor_lite/object_detection/__init__.py,sha256=eSrVAOpSykk1CfHXIKy1necplonUGxjyVKyDQ5UZoBQ,343
 valor_lite/object_detection/annotation.py,sha256=LVec-rIk408LuFxcOoIkPk0QZMWSSxbmsady4wapC1s,7007
 valor_lite/object_detection/computation.py,sha256=njLN-1_yql56NSVxY4KGKohxJUIStPYczVTpEpj5geA,24478
-valor_lite/object_detection/manager.py,sha256=FrHobYKk9ioYRs42jeeDk1Z64wQTiv4UjCl53d2xU1Y,27534
+valor_lite/object_detection/manager.py,sha256=HfSbq4vfKv2Q3kBRIqpBbq7VCrOxCl7_Pd80yUl6TKQ,30053
 valor_lite/object_detection/metric.py,sha256=sUYSZwXYfIyfmXG6_7Tje1_ZL_QwvecPq85jrGmwOWE,22739
 valor_lite/object_detection/utilities.py,sha256=tNdv5dL7JhzOamGQkZ8x3ocZoTwPI6K8rcRAGMhp2nc,11217
 valor_lite/semantic_segmentation/__init__.py,sha256=3YdItCThY_tW23IChCBm-R0zahnbZ06JDVjs-gQLVes,293
 valor_lite/semantic_segmentation/annotation.py,sha256=XRMV32Sx9A1bAVMFQdBGc3tN5Xz2RfmlyKGXCzdee7A,3705
 valor_lite/semantic_segmentation/benchmark.py,sha256=uxd0SiDY3npsgU5pdeT4HvNP_au9GVRWzoqT6br9DtM,5961
 valor_lite/semantic_segmentation/computation.py,sha256=ZO0qAFmq8lN73UjCyiynSv18qQDtn35FNOmvuXY4rOw,7380
-valor_lite/semantic_segmentation/manager.py,sha256=QmKRCy2_dJUD3h1seReVEJ1ImOfcBcmFF6kOZPO-gN0,13319
+valor_lite/semantic_segmentation/manager.py,sha256=h5w8Xl-O9gZxAzqT-ESofVE2th7d3cYahx4hHBic3pw,14256
 valor_lite/semantic_segmentation/metric.py,sha256=T9RfPJf4WgqGQTXYvSy08vJG5bjXXJnyYZeW0mlxMa8,7132
 valor_lite/semantic_segmentation/utilities.py,sha256=zgVmV8nyKWQK-T4Ov8cZFQzOmTKc5EL7errKFvc2H0g,2957
 valor_lite/text_generation/__init__.py,sha256=pGhpWCSZjLM0pPHCtPykAfos55B8ie3mi9EzbNxfj-U,356
@@ -35,7 +35,7 @@ valor_lite/text_generation/llm/instructions.py,sha256=fz2onBZZWcl5W8iy7zEWkPGU9N
 valor_lite/text_generation/llm/integrations.py,sha256=-rTfdAjq1zH-4ixwYuMQEOQ80pIFzMTe0BYfroVx3Pg,6974
 valor_lite/text_generation/llm/utilities.py,sha256=bjqatGgtVTcl1PrMwiDKTYPGJXKrBrx7PDtzIblGSys,1178
 valor_lite/text_generation/llm/validators.py,sha256=Wzr5RlfF58_2wOU-uTw7C8skan_fYdhy4Gfn0jSJ8HM,2700
-valor_lite-0.36.2.dist-info/METADATA,sha256=2nQpZZRFB9dRTwbffGRvTADwPFmUSVBSFj1L418y2sE,5071
-valor_lite-0.36.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-valor_lite-0.36.2.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
-valor_lite-0.36.2.dist-info/RECORD,,
+valor_lite-0.36.4.dist-info/METADATA,sha256=2UmPknazuM-lpHiYGqEDGl_JBr7dX-HJPCGFJKY5kck,5071
+valor_lite-0.36.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+valor_lite-0.36.4.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.36.4.dist-info/RECORD,,