valor-lite 0.33.7__py3-none-any.whl → 0.33.9__py3-none-any.whl

valor_lite/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Striveworks
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

valor_lite/classification/annotation.py

@@ -3,11 +3,39 @@ from dataclasses import dataclass
 
 @dataclass
 class Classification:
+    """
+    Classification data structure containing a ground truth label and a list of predictions.
+
+    Parameters
+    ----------
+    uid : str
+        Unique identifier for the instance.
+    groundtruth : str
+        The true label for the instance.
+    predictions : list of str
+        List of predicted labels.
+    scores : list of float
+        Confidence scores corresponding to each predicted label.
+
+    Examples
+    --------
+    >>> classification = Classification(
+    ...     uid='123',
+    ...     groundtruth='cat',
+    ...     predictions=['cat', 'dog', 'bird'],
+    ...     scores=[0.9, 0.05, 0.05]
+    ... )
+    """
+
     uid: str
-    groundtruths: list[tuple[str, str]]
-    predictions: list[tuple[str, str]]
+    groundtruth: str
+    predictions: list[str]
     scores: list[float]
 
     def __post_init__(self):
+        if not isinstance(self.groundtruth, str):
+            raise ValueError(
+                "A classification must contain a single groundtruth."
+            )
         if len(self.predictions) != len(self.scores):
             raise ValueError("There must be a score per prediction label.")

valor_lite/classification/computation.py

@@ -3,28 +3,18 @@ from numpy.typing import NDArray
 
 
 def _compute_rocauc(
-    data: NDArray[np.floating],
+    data: NDArray[np.float64],
     label_metadata: NDArray[np.int32],
     n_datums: int,
     n_labels: int,
-    n_label_keys: int,
     mask_matching_labels: NDArray[np.bool_],
     pd_labels: NDArray[np.int32],
 ):
     """
     Compute ROCAUC and mean ROCAUC.
     """
-    count_labels_per_key = np.bincount(label_metadata[:, 2])
-    count_groundtruths_per_key = np.bincount(
-        label_metadata[:, 2],
-        weights=label_metadata[:, 0],
-        minlength=n_label_keys,
-    )
-
     positive_count = label_metadata[:, 0]
-    negative_count = (
-        count_groundtruths_per_key[label_metadata[:, 2]] - label_metadata[:, 0]
-    )
+    negative_count = label_metadata[:, 1] - label_metadata[:, 0]
 
     true_positives = np.zeros((n_labels, n_datums), dtype=np.int32)
     false_positives = np.zeros_like(true_positives)
@@ -35,7 +25,6 @@ def _compute_rocauc(
             continue
 
         mask_pds = pd_labels == label_idx
-
         true_positives[label_idx] = mask_matching_labels[mask_pds]
         false_positives[label_idx] = ~mask_matching_labels[mask_pds]
         scores[label_idx] = data[mask_pds, 3]
@@ -70,32 +59,25 @@ def _compute_rocauc(
     rocauc = np.trapz(x=fpr, y=tpr, axis=1)  # type: ignore - numpy will be switching to `trapezoid` in the future.
 
     # compute mean rocauc
-    summed_rocauc = np.bincount(label_metadata[:, 2], weights=rocauc)
-    mean_rocauc = np.zeros(n_label_keys, dtype=np.float64)
-    np.divide(
-        summed_rocauc,
-        count_labels_per_key,
-        where=count_labels_per_key > 1e-9,
-        out=mean_rocauc,
-    )
+    mean_rocauc = rocauc.mean()
 
     return rocauc, mean_rocauc
 
 
 def compute_metrics(
-    data: NDArray[np.floating],
+    data: NDArray[np.float64],
     label_metadata: NDArray[np.int32],
-    score_thresholds: NDArray[np.floating],
+    score_thresholds: NDArray[np.float64],
     hardmax: bool,
     n_datums: int,
 ) -> tuple[
     NDArray[np.int32],
-    NDArray[np.floating],
-    NDArray[np.floating],
-    NDArray[np.floating],
-    NDArray[np.floating],
-    NDArray[np.floating],
-    NDArray[np.floating],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    float,
 ]:
     """
     Computes classification metrics.
@@ -110,14 +92,14 @@ def compute_metrics(
 
     Parameters
     ----------
-    data : NDArray[np.floating]
+    data : NDArray[np.float64]
         A sorted array of classification pairs.
     label_metadata : NDArray[np.int32]
         An array containing metadata related to labels.
-    score_thresholds : NDArray[np.floating]
+    score_thresholds : NDArray[np.float64]
         A 1-D array contains score thresholds to compute metrics over.
     hardmax : bool
-        Option to only allow a single positive prediction per label key.
+        Option to only allow a single positive prediction.
     n_datums : int
         The number of datums being operated over.
 
@@ -125,22 +107,21 @@ def compute_metrics(
     -------
     NDArray[np.int32]
         TP, FP, FN, TN counts.
-    NDArray[np.floating]
+    NDArray[np.float64]
         Precision.
-    NDArray[np.floating]
+    NDArray[np.float64]
         Recall.
-    NDArray[np.floating]
+    NDArray[np.float64]
         Accuracy
-    NDArray[np.floating]
+    NDArray[np.float64]
         F1 Score
-    NDArray[np.floating]
+    NDArray[np.float64]
         ROCAUC.
-    NDArray[np.floating]
+    float
         mROCAUC.
     """
 
     n_labels = label_metadata.shape[0]
-    n_label_keys = np.unique(label_metadata[:, 2]).size
     n_scores = score_thresholds.shape[0]
 
     pd_labels = data[:, 2].astype(int)
@@ -155,7 +136,6 @@ def compute_metrics(
         label_metadata=label_metadata,
         n_datums=n_datums,
         n_labels=n_labels,
-        n_label_keys=n_label_keys,
         mask_matching_labels=mask_matching_labels,
         pd_labels=pd_labels,
     )
@@ -229,16 +209,16 @@ def compute_metrics(
 
 
 def _count_with_examples(
-    data: NDArray[np.floating],
+    data: NDArray[np.float64],
     unique_idx: int | list[int],
     label_idx: int | list[int],
-) -> tuple[NDArray[np.floating], NDArray[np.int32], NDArray[np.int32]]:
+) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.int32]]:
     """
     Helper function for counting occurences of unique detailed pairs.
 
     Parameters
     ----------
-    data : NDArray[np.floating]
+    data : NDArray[np.float64]
         A masked portion of a detailed pairs array.
     unique_idx : int | list[int]
         The index or indices upon which uniqueness is constrained.
@@ -247,7 +227,7 @@ def _count_with_examples(
 
     Returns
     -------
-    NDArray[np.floating]
+    NDArray[np.float64]
         Examples drawn from the data input.
     NDArray[np.int32]
         Unique label indices.
@@ -267,13 +247,12 @@ def _count_with_examples(
 
 
 def compute_confusion_matrix(
-    data: NDArray[np.floating],
+    data: NDArray[np.float64],
     label_metadata: NDArray[np.int32],
-    score_thresholds: NDArray[np.floating],
+    score_thresholds: NDArray[np.float64],
     hardmax: bool,
     n_examples: int,
-) -> tuple[NDArray[np.floating], NDArray[np.int32]]:
-
+) -> tuple[NDArray[np.float64], NDArray[np.int32]]:
     """
     Compute detailed confusion matrix.
 
@@ -287,20 +266,20 @@ def compute_confusion_matrix(
 
     Parameters
     ----------
-    data : NDArray[np.floating]
+    data : NDArray[np.float64]
         A sorted array summarizing the IOU calculations of one or more pairs.
     label_metadata : NDArray[np.int32]
         An array containing metadata related to labels.
-    iou_thresholds : NDArray[np.floating]
+    iou_thresholds : NDArray[np.float64]
         A 1-D array containing IoU thresholds.
-    score_thresholds : NDArray[np.floating]
+    score_thresholds : NDArray[np.float64]
         A 1-D array containing score thresholds.
     n_examples : int
         The maximum number of examples to return per count.
 
     Returns
     -------
-    NDArray[np.floating]
+    NDArray[np.float64]
         Confusion matrix.
     NDArray[np.int32]
         Ground truths with missing predictions.