valor-lite 0.33.8__py3-none-any.whl → 0.33.10__py3-none-any.whl

valor_lite/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Striveworks
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

valor_lite/classification/annotation.py

@@ -3,6 +3,30 @@ from dataclasses import dataclass
 
 @dataclass
 class Classification:
+    """
+    Classification data structure containing a ground truth label and a list of predictions.
+
+    Parameters
+    ----------
+    uid : str
+        Unique identifier for the instance.
+    groundtruth : str
+        The true label for the instance.
+    predictions : list of str
+        List of predicted labels.
+    scores : list of float
+        Confidence scores corresponding to each predicted label.
+
+    Examples
+    --------
+    >>> classification = Classification(
+    ...     uid='123',
+    ...     groundtruth='cat',
+    ...     predictions=['cat', 'dog', 'bird'],
+    ...     scores=[0.9, 0.05, 0.05]
+    ... )
+    """
+
     uid: str
     groundtruth: str
     predictions: list[str]

valor_lite/classification/manager.py

@@ -191,12 +191,117 @@ class Evaluator:
             n_datums=n_datums,
         )
 
-    def evaluate(
+    def _unpack_confusion_matrix(
+        self,
+        confusion_matrix: NDArray[np.float64],
+        number_of_labels: int,
+        number_of_examples: int,
+    ) -> dict[
+        str,
+        dict[
+            str,
+            dict[
+                str,
+                int
+                | list[
+                    dict[
+                        str,
+                        str | float,
+                    ]
+                ],
+            ],
+        ],
+    ]:
+        """
+        Unpacks a numpy array of confusion matrix counts and examples.
+        """
+
+        datum_idx = lambda gt_label_idx, pd_label_idx, example_idx: int(  # noqa: E731 - lambda fn
+            confusion_matrix[
+                gt_label_idx,
+                pd_label_idx,
+                example_idx * 2 + 1,
+            ]
+        )
+
+        score_idx = lambda gt_label_idx, pd_label_idx, example_idx: float(  # noqa: E731 - lambda fn
+            confusion_matrix[
+                gt_label_idx,
+                pd_label_idx,
+                example_idx * 2 + 2,
+            ]
+        )
+
+        return {
+            self.index_to_label[gt_label_idx]: {
+                self.index_to_label[pd_label_idx]: {
+                    "count": max(
+                        int(confusion_matrix[gt_label_idx, pd_label_idx, 0]),
+                        0,
+                    ),
+                    "examples": [
+                        {
+                            "datum": self.index_to_uid[
+                                datum_idx(
+                                    gt_label_idx, pd_label_idx, example_idx
+                                )
+                            ],
+                            "score": score_idx(
+                                gt_label_idx, pd_label_idx, example_idx
+                            ),
+                        }
+                        for example_idx in range(number_of_examples)
+                        if datum_idx(gt_label_idx, pd_label_idx, example_idx)
+                        >= 0
+                    ],
+                }
+                for pd_label_idx in range(number_of_labels)
+            }
+            for gt_label_idx in range(number_of_labels)
+        }
+
+    def _unpack_missing_predictions(
+        self,
+        missing_predictions: NDArray[np.int32],
+        number_of_labels: int,
+        number_of_examples: int,
+    ) -> dict[str, dict[str, int | list[dict[str, str]]]]:
+        """
+        Unpacks a numpy array of missing prediction counts and examples.
+        """
+
+        datum_idx = (
+            lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
+                missing_predictions[
+                    gt_label_idx,
+                    example_idx + 1,
+                ]
+            )
+        )
+
+        return {
+            self.index_to_label[gt_label_idx]: {
+                "count": max(
+                    int(missing_predictions[gt_label_idx, 0]),
+                    0,
+                ),
+                "examples": [
+                    {
+                        "datum": self.index_to_uid[
+                            datum_idx(gt_label_idx, example_idx)
+                        ]
+                    }
+                    for example_idx in range(number_of_examples)
+                    if datum_idx(gt_label_idx, example_idx) >= 0
+                ],
+            }
+            for gt_label_idx in range(number_of_labels)
+        }
+
+    def compute_precision_recall(
         self,
-        metrics_to_return: list[MetricType] = MetricType.base(),
         score_thresholds: list[float] = [0.0],
         hardmax: bool = True,
-        number_of_examples: int = 0,
         filter_: Filter | None = None,
         as_dict: bool = False,
     ) -> dict[MetricType, list]:
@@ -205,14 +310,10 @@ class Evaluator:
 
         Parameters
         ----------
-        metrics_to_return : list[MetricType]
-            A list of metrics to return in the results.
         score_thresholds : list[float]
             A list of score thresholds to compute metrics over.
         hardmax : bool
             Toggles whether a hardmax is applied to predictions.
-        number_of_examples : int, default=0
-            Maximum number of annotation examples to return in ConfusionMatrix.
         filter_ : Filter, optional
             An optional filter object.
         as_dict : bool, default=False
@@ -253,7 +354,7 @@ class Evaluator:
 
         metrics[MetricType.ROCAUC] = [
             ROCAUC(
-                value=rocauc[label_idx],
+                value=float(rocauc[label_idx]),
                 label=self.index_to_label[label_idx],
             )
             for label_idx in range(label_metadata.shape[0])
@@ -262,7 +363,7 @@ class Evaluator:
 
         metrics[MetricType.mROCAUC] = [
             mROCAUC(
-                value=mean_rocauc,
+                value=float(mean_rocauc),
             )
         ]
 
@@ -276,10 +377,10 @@ class Evaluator:
             row = counts[:, label_idx]
             metrics[MetricType.Counts].append(
                 Counts(
-                    tp=row[:, 0].tolist(),
-                    fp=row[:, 1].tolist(),
-                    fn=row[:, 2].tolist(),
-                    tn=row[:, 3].tolist(),
+                    tp=row[:, 0].astype(int).tolist(),
+                    fp=row[:, 1].astype(int).tolist(),
+                    fn=row[:, 2].astype(int).tolist(),
+                    tn=row[:, 3].astype(int).tolist(),
                     **kwargs,
                 )
             )
@@ -290,44 +391,29 @@ class Evaluator:
 
             metrics[MetricType.Precision].append(
                 Precision(
-                    value=precision[:, label_idx].tolist(),
+                    value=precision[:, label_idx].astype(float).tolist(),
                     **kwargs,
                 )
             )
             metrics[MetricType.Recall].append(
                 Recall(
-                    value=recall[:, label_idx].tolist(),
+                    value=recall[:, label_idx].astype(float).tolist(),
                     **kwargs,
                 )
             )
             metrics[MetricType.Accuracy].append(
                 Accuracy(
-                    value=accuracy[:, label_idx].tolist(),
+                    value=accuracy[:, label_idx].astype(float).tolist(),
                     **kwargs,
                 )
             )
             metrics[MetricType.F1].append(
                 F1(
-                    value=f1_score[:, label_idx].tolist(),
+                    value=f1_score[:, label_idx].astype(float).tolist(),
                     **kwargs,
                 )
             )
 
-        if MetricType.ConfusionMatrix in metrics_to_return:
-            metrics[
-                MetricType.ConfusionMatrix
-            ] = self._compute_confusion_matrix(
-                data=data,
-                label_metadata=label_metadata,
-                score_thresholds=score_thresholds,
-                hardmax=hardmax,
-                number_of_examples=number_of_examples,
-            )
-
-        for metric in set(metrics.keys()):
-            if metric not in metrics_to_return:
-                del metrics[metric]
-
         if as_dict:
             return {
                 mtype: [metric.to_dict() for metric in mvalues]
@@ -336,143 +422,43 @@ class Evaluator:
 
         return metrics
 
-    def _unpack_confusion_matrix(
+    def compute_confusion_matrix(
         self,
-        confusion_matrix: NDArray[np.float64],
-        number_of_labels: int,
-        number_of_examples: int,
-    ) -> dict[
-        str,
-        dict[
-            str,
-            dict[
-                str,
-                int
-                | list[
-                    dict[
-                        str,
-                        str | float,
-                    ]
-                ],
-            ],
-        ],
-    ]:
-        """
-        Unpacks a numpy array of confusion matrix counts and examples.
-        """
-
-        datum_idx = lambda gt_label_idx, pd_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            confusion_matrix[
-                gt_label_idx,
-                pd_label_idx,
-                example_idx * 2 + 1,
-            ]
-        )
-
-        score_idx = lambda gt_label_idx, pd_label_idx, example_idx: float(  # noqa: E731 - lambda fn
-            confusion_matrix[
-                gt_label_idx,
-                pd_label_idx,
-                example_idx * 2 + 2,
-            ]
-        )
-
-        return {
-            self.index_to_label[gt_label_idx]: {
-                self.index_to_label[pd_label_idx]: {
-                    "count": max(
-                        int(confusion_matrix[gt_label_idx, pd_label_idx, 0]),
-                        0,
-                    ),
-                    "examples": [
-                        {
-                            "datum": self.index_to_uid[
-                                datum_idx(
-                                    gt_label_idx, pd_label_idx, example_idx
-                                )
-                            ],
-                            "score": score_idx(
-                                gt_label_idx, pd_label_idx, example_idx
-                            ),
-                        }
-                        for example_idx in range(number_of_examples)
-                        if datum_idx(gt_label_idx, pd_label_idx, example_idx)
-                        >= 0
-                    ],
-                }
-                for pd_label_idx in range(number_of_labels)
-            }
-            for gt_label_idx in range(number_of_labels)
-        }
-
-    def _unpack_missing_predictions(
-        self,
-        missing_predictions: NDArray[np.int32],
-        number_of_labels: int,
-        number_of_examples: int,
-    ) -> dict[str, dict[str, int | list[dict[str, str]]]]:
-        """
-        Unpacks a numpy array of missing prediction counts and examples.
-        """
-
-        datum_idx = (
-            lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-                missing_predictions[
-                    gt_label_idx,
-                    example_idx + 1,
-                ]
-            )
-        )
-
-        return {
-            self.index_to_label[gt_label_idx]: {
-                "count": max(
-                    int(missing_predictions[gt_label_idx, 0]),
-                    0,
-                ),
-                "examples": [
-                    {
-                        "datum": self.index_to_uid[
-                            datum_idx(gt_label_idx, example_idx)
-                        ]
-                    }
-                    for example_idx in range(number_of_examples)
-                    if datum_idx(gt_label_idx, example_idx) >= 0
-                ],
-            }
-            for gt_label_idx in range(number_of_labels)
-        }
-
-    def _compute_confusion_matrix(
-        self,
-        data: NDArray[np.float64],
-        label_metadata: NDArray[np.int32],
-        score_thresholds: list[float],
-        hardmax: bool,
-        number_of_examples: int,
-    ) -> list[ConfusionMatrix]:
+        score_thresholds: list[float] = [0.0],
+        hardmax: bool = True,
+        number_of_examples: int = 0,
+        filter_: Filter | None = None,
+        as_dict: bool = False,
+    ) -> list:
         """
         Computes a detailed confusion matrix..
 
         Parameters
         ----------
-        data : NDArray[np.float64]
-            A data array containing classification pairs.
-        label_metadata : NDArray[np.int32]
-            An integer array containing label metadata.
         score_thresholds : list[float]
             A list of score thresholds to compute metrics over.
         hardmax : bool
             Toggles whether a hardmax is applied to predictions.
         number_of_examples : int, default=0
             The number of examples to return per count.
+        filter_ : Filter, optional
+            An optional filter object.
+        as_dict : bool, default=False
+            An option to return metrics as dictionaries.
 
         Returns
         -------
-        list[ConfusionMatrix]
-            A list of ConfusionMatrix objects.
+        list[ConfusionMatrix] | list[dict]
+            A list of confusion matrices.
         """
 
+        # apply filters
+        data = self._detailed_pairs
+        label_metadata = self._label_metadata
+        if filter_ is not None:
+            data = data[filter_.indices]
+            label_metadata = filter_.label_metadata
+
         if data.size == 0:
             return list()
 
@@ -485,7 +471,7 @@ class Evaluator:
         )
 
         n_scores, n_labels, _, _ = confusion_matrix.shape
-        return [
+        results = [
             ConfusionMatrix(
                 score_threshold=score_thresholds[score_idx],
                 number_of_examples=number_of_examples,
@@ -503,6 +489,56 @@ class Evaluator:
             for score_idx in range(n_scores)
         ]
 
+        if as_dict:
+            return [m.to_dict() for m in results]
+
+        return results
+
+    def evaluate(
+        self,
+        score_thresholds: list[float] = [0.0],
+        hardmax: bool = True,
+        number_of_examples: int = 0,
+        filter_: Filter | None = None,
+        as_dict: bool = False,
+    ) -> dict[MetricType, list]:
+        """
+        Computes a detailed confusion matrix..
+
+        Parameters
+        ----------
+        score_thresholds : list[float]
+            A list of score thresholds to compute metrics over.
+        hardmax : bool
+            Toggles whether a hardmax is applied to predictions.
+        number_of_examples : int, default=0
+            The number of examples to return per count.
+        filter_ : Filter, optional
+            An optional filter object.
+        as_dict : bool, default=False
+            An option to return metrics as dictionaries.
+
+        Returns
+        -------
+        list[ConfusionMatrix] | list[dict]
+            A list of confusion matrices.
+        """
+
+        results = self.compute_precision_recall(
+            score_thresholds=score_thresholds,
+            hardmax=hardmax,
+            filter_=filter_,
+            as_dict=as_dict,
+        )
+        results[MetricType.ConfusionMatrix] = self.compute_confusion_matrix(
+            score_thresholds=score_thresholds,
+            hardmax=hardmax,
+            number_of_examples=number_of_examples,
+            filter_=filter_,
+            as_dict=as_dict,
+        )
+        return results
+
 
 class DataLoader:
     """
@@ -644,70 +680,6 @@ class DataLoader:
                 predictions=predictions,
             )
 
-    def add_data_from_valor_dict(
-        self,
-        classifications: list[tuple[dict, dict]],
-        show_progress: bool = False,
-    ):
-        """
-        Adds Valor-format classifications to the cache.
-
-        Parameters
-        ----------
-        classifications : list[tuple[dict, dict]]
-            A list of groundtruth, prediction pairs in Valor-format dictionaries.
-        show_progress : bool, default=False
-            Toggle for tqdm progress bar.
-        """
-
-        disable_tqdm = not show_progress
-        for groundtruth, prediction in tqdm(
-            classifications, disable=disable_tqdm
-        ):
-
-            # update metadata
-            self._evaluator.n_datums += 1
-            self._evaluator.n_groundtruths += len(groundtruth["annotations"])
-            self._evaluator.n_predictions += len(prediction["annotations"])
-
-            # update datum uid index
-            uid_index = self._add_datum(uid=groundtruth["datum"]["uid"])
-
-            # cache labels and annotations
-            predictions = list()
-            groundtruths = None
-            for gann in groundtruth["annotations"]:
-                for valor_label in gann["labels"]:
-                    glabel = f'{valor_label["key"]}_{valor_label["value"]}'
-                    label_idx = self._add_label(glabel)
-                    self.groundtruth_count[label_idx][uid_index] += 1
-                    groundtruths = label_idx
-            for pann in prediction["annotations"]:
-                for valor_label in pann["labels"]:
-                    plabel = f'{valor_label["key"]}_{valor_label["value"]}'
-                    pscore = valor_label["score"]
-                    label_idx = self._add_label(plabel)
-                    self.prediction_count[label_idx][uid_index] += 1
-                    predictions.append(
-                        (
-                            label_idx,
-                            pscore,
-                        )
-                    )
-
-            # fix type error where groundtruths can possibly be unbound now that it's a float
-            # in practice, this error should never be hit since groundtruths can't be empty without throwing a ValueError earlier in the flow
-            if groundtruths is None:
-                raise ValueError(
-                    "Expected a value for groundtruths, but got None."
-                )
-
-            self._add_data(
-                uid_index=uid_index,
-                groundtruth=groundtruths,
-                predictions=predictions,
-            )
-
     def finalize(self) -> Evaluator:
         """
         Performs data finalization and some preprocessing steps.