valor-lite 0.33.17__py3-none-any.whl → 0.33.19__py3-none-any.whl

valor_lite/classification/computation.py

@@ -282,7 +282,7 @@ def compute_confusion_matrix(
     NDArray[np.float64]
         Confusion matrix.
     NDArray[np.int32]
-        Ground truths with missing predictions.
+        Unmatched Ground Truths.
     """
 
     n_labels = label_metadata.shape[0]
@@ -292,7 +292,7 @@ def compute_confusion_matrix(
         (n_scores, n_labels, n_labels, 2 * n_examples + 1),
         dtype=np.float32,
     )
-    missing_predictions = -1 * np.ones(
+    unmatched_ground_truths = -1 * np.ones(
         (n_scores, n_labels, n_examples + 1),
         dtype=np.int32,
     )
@@ -339,7 +339,7 @@ def compute_confusion_matrix(
             score_idx, misclf_labels[:, 0], misclf_labels[:, 1], 0
         ] = misclf_counts
 
-        missing_predictions[score_idx, misprd_labels, 0] = misprd_counts
+        unmatched_ground_truths[score_idx, misprd_labels, 0] = misprd_counts
 
         if n_examples > 0:
             for label_idx in range(n_labels):
@@ -375,16 +375,16 @@ def compute_confusion_matrix(
                                 1 : 2 * misclf_label_examples.shape[0] + 1,
                             ] = misclf_label_examples[:, [0, 3]].flatten()
 
-                # missing prediction examples
+                # unmatched ground truth examples
                 mask_misprd_label = misprd_examples[:, 1] == label_idx
                 if misprd_examples.size > 0:
                     misprd_label_examples = misprd_examples[mask_misprd_label][
                         :n_examples
                     ]
-                    missing_predictions[
+                    unmatched_ground_truths[
                         score_idx,
                         label_idx,
                         1 : misprd_label_examples.shape[0] + 1,
                     ] = misprd_label_examples[:, 0].flatten()
 
-    return confusion_matrix, missing_predictions
+    return confusion_matrix, unmatched_ground_truths

valor_lite/classification/metric.py

@@ -321,7 +321,7 @@ class Metric(BaseMetric):
                 ],
             ],
         ],
-        missing_predictions: dict[
+        unmatched_ground_truths: dict[
             str,  # ground truth label value
             dict[
                 str,  # either `count` or `examples`
@@ -335,8 +335,8 @@ class Metric(BaseMetric):
         The confusion matrix and related metrics for the classification task.
 
         This class encapsulates detailed information about the model's performance, including correct
-        predictions, misclassifications, hallucinations (false positives), and missing predictions
-        (false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
+        predictions, misclassifications, unmatched predictions (subset of false positives), and unmatched ground truths
+        (subset of false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
 
         Confusion Matrix Structure:
         {
@@ -358,7 +358,7 @@ class Metric(BaseMetric):
             ...
         }
 
-        Missing Prediction Structure:
+        Unmatched Ground Truths Structure:
         {
             ground_truth_label: {
                 'count': int,
@@ -379,7 +379,7 @@ class Metric(BaseMetric):
             A nested dictionary where the first key is the ground truth label value, the second key
             is the prediction label value, and the innermost dictionary contains either a `count`
             or a list of `examples`. Each example includes the datum UID and prediction score.
-        missing_predictions : dict
+        unmatched_ground_truths : dict
             A dictionary where each key is a ground truth label value for which the model failed to predict
             (false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
             Each example includes the datum UID.
@@ -396,7 +396,7 @@ class Metric(BaseMetric):
             type=MetricType.ConfusionMatrix.value,
             value={
                 "confusion_matrix": confusion_matrix,
-                "missing_predictions": missing_predictions,
+                "unmatched_ground_truths": unmatched_ground_truths,
             },
             parameters={
                 "score_threshold": score_threshold,

valor_lite/classification/utilities.py

@@ -153,20 +153,20 @@ def _unpack_confusion_matrix_value(
     }
 
 
-def _unpack_missing_predictions_value(
-    missing_predictions: NDArray[np.int32],
+def _unpack_unmatched_ground_truths_value(
+    unmatched_ground_truths: NDArray[np.int32],
     number_of_labels: int,
     number_of_examples: int,
     index_to_uid: dict[int, str],
     index_to_label: dict[int, str],
 ) -> dict[str, dict[str, int | list[dict[str, str]]]]:
     """
-    Unpacks a numpy array of missing prediction counts and examples.
+    Unpacks a numpy array of unmatched ground truth counts and examples.
     """
 
     datum_idx = (
         lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            missing_predictions[
+            unmatched_ground_truths[
                 gt_label_idx,
                 example_idx + 1,
             ]
@@ -176,7 +176,7 @@ def _unpack_missing_predictions_value(
     return {
         index_to_label[gt_label_idx]: {
             "count": max(
-                int(missing_predictions[gt_label_idx, 0]),
+                int(unmatched_ground_truths[gt_label_idx, 0]),
                 0,
             ),
             "examples": [
@@ -197,7 +197,7 @@ def unpack_confusion_matrix_into_metric_list(
     index_to_label: dict[int, str],
 ) -> list[Metric]:
 
-    (confusion_matrix, missing_predictions) = results
+    (confusion_matrix, unmatched_ground_truths) = results
     n_scores, n_labels, _, _ = confusion_matrix.shape
     return [
         Metric.confusion_matrix(
@@ -210,8 +210,10 @@ def unpack_confusion_matrix_into_metric_list(
                 index_to_label=index_to_label,
                 index_to_uid=index_to_uid,
             ),
-            missing_predictions=_unpack_missing_predictions_value(
-                missing_predictions=missing_predictions[score_idx, :, :],
+            unmatched_ground_truths=_unpack_unmatched_ground_truths_value(
+                unmatched_ground_truths=unmatched_ground_truths[
+                    score_idx, :, :
+                ],
                 number_of_labels=n_labels,
                 number_of_examples=number_of_examples,
                 index_to_label=index_to_label,

valor_lite/object_detection/computation.py

@@ -669,9 +669,9 @@ def compute_confusion_matrix(
     NDArray[np.float64]
         Confusion matrix.
     NDArray[np.float64]
-        Hallucinations.
+        Unmatched Predictions.
     NDArray[np.int32]
-        Missing Predictions.
+        Unmatched Ground Truths.
     """
 
     n_labels = label_metadata.shape[0]
@@ -683,12 +683,12 @@ def compute_confusion_matrix(
         (n_ious, n_scores, n_labels, n_labels, 4 * n_examples + 1),
         dtype=np.float32,
     )
-    hallucinations = -1 * np.ones(
+    unmatched_predictions = -1 * np.ones(
         # (datum idx, pd idx, pd score) * n_examples + count
         (n_ious, n_scores, n_labels, 3 * n_examples + 1),
         dtype=np.float32,
     )
-    missing_predictions = -1 * np.ones(
+    unmatched_ground_truths = -1 * np.ones(
         # (datum idx, gt idx) * n_examples + count
         (n_ious, n_scores, n_labels, 2 * n_examples + 1),
         dtype=np.int32,
@@ -793,7 +793,7 @@ def compute_confusion_matrix(
                 data[mask_misclf], unique_idx=[0, 1, 2, 4, 5], label_idx=[3, 4]
             )
 
-            # count hallucinations
+            # count unmatched predictions
             (
                 halluc_examples,
                 halluc_labels,
@@ -802,7 +802,7 @@ def compute_confusion_matrix(
                 data[mask_halluc], unique_idx=[0, 2, 5], label_idx=2
             )
 
-            # count missing predictions
+            # count unmatched ground truths
             (
                 misprd_examples,
                 misprd_labels,
@@ -822,13 +822,13 @@ def compute_confusion_matrix(
                 misclf_labels[:, 1],
                 0,
             ] = misclf_counts
-            hallucinations[
+            unmatched_predictions[
                 iou_idx,
                 score_idx,
                 halluc_labels,
                 0,
             ] = halluc_counts
-            missing_predictions[
+            unmatched_ground_truths[
                 iou_idx,
                 score_idx,
                 misprd_labels,
@@ -877,26 +877,26 @@ def compute_confusion_matrix(
                                     :, [0, 1, 2, 6]
                                 ].flatten()
 
-                    # hallucination examples
+                    # unmatched prediction examples
                     mask_halluc_label = halluc_examples[:, 5] == label_idx
                     if mask_halluc_label.sum() > 0:
                         halluc_label_examples = halluc_examples[
                             mask_halluc_label
                         ][:n_examples]
-                        hallucinations[
+                        unmatched_predictions[
                             iou_idx,
                             score_idx,
                             label_idx,
                             1 : 3 * halluc_label_examples.shape[0] + 1,
                         ] = halluc_label_examples[:, [0, 2, 6]].flatten()
 
-                    # missing prediction examples
+                    # unmatched ground truth examples
                     mask_misprd_label = misprd_examples[:, 4] == label_idx
                     if misprd_examples.size > 0:
                         misprd_label_examples = misprd_examples[
                             mask_misprd_label
                         ][:n_examples]
-                        missing_predictions[
+                        unmatched_ground_truths[
                             iou_idx,
                             score_idx,
                             label_idx,
@@ -905,6 +905,6 @@ def compute_confusion_matrix(
 
     return (
         confusion_matrix,
-        hallucinations,
-        missing_predictions,
+        unmatched_predictions,
+        unmatched_ground_truths,
     )

valor_lite/object_detection/manager.py

@@ -334,6 +334,10 @@ class Evaluator:
         return metrics
 
 
+def defaultdict_int():
+    return defaultdict(int)
+
+
 class DataLoader:
     """
     Object Detection DataLoader
@@ -342,8 +346,8 @@ class DataLoader:
     def __init__(self):
         self._evaluator = Evaluator()
         self.pairs: list[NDArray[np.float64]] = list()
-        self.groundtruth_count = defaultdict(lambda: defaultdict(int))
-        self.prediction_count = defaultdict(lambda: defaultdict(int))
+        self.groundtruth_count = defaultdict(defaultdict_int)
+        self.prediction_count = defaultdict(defaultdict_int)
 
     def _add_datum(self, uid: str) -> int:
         """

valor_lite/object_detection/metric.py

@@ -619,7 +619,7 @@ class Metric(BaseMetric):
                 ],
             ],
         ],
-        hallucinations: dict[
+        unmatched_predictions: dict[
             str,  # prediction label value
             dict[
                 str,  # either `count` or `examples`
@@ -636,7 +636,7 @@ class Metric(BaseMetric):
                 ],
             ],
         ],
-        missing_predictions: dict[
+        unmatched_ground_truths: dict[
             str,  # ground truth label value
             dict[
                 str,  # either `count` or `examples`
@@ -660,8 +660,8 @@ class Metric(BaseMetric):
         Confusion matrix for object detection tasks.
 
         This class encapsulates detailed information about the model's performance, including correct
-        predictions, misclassifications, hallucinations (false positives), and missing predictions
-        (false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
+        predictions, misclassifications, unmatched_predictions (subset of false positives), and unmatched ground truths
+        (subset of false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
 
         Confusion Matrix Format:
         {
@@ -683,7 +683,7 @@ class Metric(BaseMetric):
             ...
         }
 
-        Hallucinations Format:
+        Unmatched Predictions Format:
         {
             <prediction label>: {
                 'count': int,
@@ -699,7 +699,7 @@ class Metric(BaseMetric):
             ...
         }
 
-        Missing Prediction Format:
+        Unmatched Ground Truths Format:
         {
             <ground truth label>: {
                 'count': int,
@@ -721,13 +721,13 @@ class Metric(BaseMetric):
             is the prediction label value, and the innermost dictionary contains either a `count`
             or a list of `examples`. Each example includes the datum UID, ground truth bounding box,
             predicted bounding box, and prediction scores.
-        hallucinations : dict
+        unmatched_predictions : dict
             A dictionary where each key is a prediction label value with no corresponding ground truth
-            (false positives). The value is a dictionary containing either a `count` or a list of
+            (subset of false positives). The value is a dictionary containing either a `count` or a list of
             `examples`. Each example includes the datum UID, predicted bounding box, and prediction score.
-        missing_predictions : dict
+        unmatched_ground_truths : dict
             A dictionary where each key is a ground truth label value for which the model failed to predict
-            (false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
+            (subset of false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
             Each example includes the datum UID and ground truth bounding box.
         score_threshold : float
             The confidence score threshold used to filter predictions.
@@ -744,8 +744,8 @@ class Metric(BaseMetric):
             type=MetricType.ConfusionMatrix.value,
             value={
                 "confusion_matrix": confusion_matrix,
-                "hallucinations": hallucinations,
-                "missing_predictions": missing_predictions,
+                "unmatched_predictions": unmatched_predictions,
+                "unmatched_ground_truths": unmatched_ground_truths,
             },
             parameters={
                 "score_threshold": score_threshold,

valor_lite/object_detection/utilities.py

@@ -321,8 +321,8 @@ def _unpack_confusion_matrix_value(
     }
 
 
-def _unpack_hallucinations_value(
-    hallucinations: NDArray[np.float64],
+def _unpack_unmatched_predictions_value(
+    unmatched_predictions: NDArray[np.float64],
     number_of_labels: int,
     number_of_examples: int,
     index_to_uid: dict[int, str],
@@ -336,12 +336,12 @@ def _unpack_hallucinations_value(
     ],
 ]:
     """
-    Unpacks a numpy array of hallucination counts and examples.
+    Unpacks a numpy array of unmatched_prediction counts and examples.
     """
 
     datum_idx = (
         lambda pd_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            hallucinations[
+            unmatched_predictions[
                 pd_label_idx,
                 example_idx * 3 + 1,
             ]
@@ -350,7 +350,7 @@ def _unpack_hallucinations_value(
 
     prediction_idx = (
         lambda pd_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            hallucinations[
+            unmatched_predictions[
                 pd_label_idx,
                 example_idx * 3 + 2,
             ]
@@ -359,7 +359,7 @@ def _unpack_hallucinations_value(
 
     score_idx = (
         lambda pd_label_idx, example_idx: float(  # noqa: E731 - lambda fn
-            hallucinations[
+            unmatched_predictions[
                 pd_label_idx,
                 example_idx * 3 + 3,
             ]
@@ -369,7 +369,7 @@ def _unpack_hallucinations_value(
     return {
         index_to_label[pd_label_idx]: {
             "count": max(
-                int(hallucinations[pd_label_idx, 0]),
+                int(unmatched_predictions[pd_label_idx, 0]),
                 0,
             ),
             "examples": [
@@ -392,8 +392,8 @@ def _unpack_hallucinations_value(
     }
 
 
-def _unpack_missing_predictions_value(
-    missing_predictions: NDArray[np.int32],
+def _unpack_unmatched_ground_truths_value(
+    unmatched_ground_truths: NDArray[np.int32],
     number_of_labels: int,
     number_of_examples: int,
     index_to_uid: dict[int, str],
@@ -401,12 +401,12 @@ def _unpack_missing_predictions_value(
     groundtruth_examples: dict[int, NDArray[np.float16]],
 ) -> dict[str, dict[str, int | list[dict[str, str | dict[str, float]]]]]:
     """
-    Unpacks a numpy array of missing prediction counts and examples.
+    Unpacks a numpy array of unmatched ground truth counts and examples.
     """
 
     datum_idx = (
         lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            missing_predictions[
+            unmatched_ground_truths[
                 gt_label_idx,
                 example_idx * 2 + 1,
             ]
@@ -415,7 +415,7 @@ def _unpack_missing_predictions_value(
 
     groundtruth_idx = (
         lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            missing_predictions[
+            unmatched_ground_truths[
                 gt_label_idx,
                 example_idx * 2 + 2,
             ]
@@ -425,7 +425,7 @@ def _unpack_missing_predictions_value(
     return {
         index_to_label[gt_label_idx]: {
             "count": max(
-                int(missing_predictions[gt_label_idx, 0]),
+                int(unmatched_ground_truths[gt_label_idx, 0]),
                 0,
             ),
             "examples": [
@@ -463,8 +463,8 @@ def unpack_confusion_matrix_into_metric_list(
 ) -> list[Metric]:
     (
         confusion_matrix,
-        hallucinations,
-        missing_predictions,
+        unmatched_predictions,
+        unmatched_ground_truths,
     ) = results
     n_labels = len(index_to_label)
     return [
@@ -481,16 +481,18 @@ def unpack_confusion_matrix_into_metric_list(
                 groundtruth_examples=groundtruth_examples,
                 prediction_examples=prediction_examples,
             ),
-            hallucinations=_unpack_hallucinations_value(
-                hallucinations=hallucinations[iou_idx, score_idx, :, :],
+            unmatched_predictions=_unpack_unmatched_predictions_value(
+                unmatched_predictions=unmatched_predictions[
+                    iou_idx, score_idx, :, :
+                ],
                 number_of_labels=n_labels,
                 number_of_examples=number_of_examples,
                 index_to_label=index_to_label,
                 index_to_uid=index_to_uid,
                 prediction_examples=prediction_examples,
             ),
-            missing_predictions=_unpack_missing_predictions_value(
-                missing_predictions=missing_predictions[
+            unmatched_ground_truths=_unpack_unmatched_ground_truths_value(
+                unmatched_ground_truths=unmatched_ground_truths[
                     iou_idx, score_idx, :, :
                 ],
                 number_of_labels=n_labels,