valor-lite 0.33.18__py3-none-any.whl → 0.34.0__py3-none-any.whl

valor_lite/classification/computation.py

@@ -212,7 +212,7 @@ def _count_with_examples(
     data: NDArray[np.float64],
     unique_idx: int | list[int],
     label_idx: int | list[int],
-) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.int32]]:
+) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.intp]]:
     """
     Helper function for counting occurences of unique detailed pairs.
 
@@ -231,7 +231,7 @@ def _count_with_examples(
         Examples drawn from the data input.
     NDArray[np.int32]
         Unique label indices.
-    NDArray[np.int32]
+    NDArray[np.intp]
         Counts for each unique label index.
     """
     unique_rows, indices = np.unique(
@@ -282,18 +282,20 @@ def compute_confusion_matrix(
     NDArray[np.float64]
         Confusion matrix.
     NDArray[np.int32]
-        Ground truths with missing predictions.
+        Unmatched Ground Truths.
     """
 
     n_labels = label_metadata.shape[0]
     n_scores = score_thresholds.shape[0]
 
-    confusion_matrix = -1 * np.ones(
+    confusion_matrix = np.full(
         (n_scores, n_labels, n_labels, 2 * n_examples + 1),
+        fill_value=-1.0,
         dtype=np.float32,
     )
-    missing_predictions = -1 * np.ones(
+    unmatched_ground_truths = np.full(
         (n_scores, n_labels, n_examples + 1),
+        fill_value=-1,
         dtype=np.int32,
     )
 
@@ -339,7 +341,7 @@ def compute_confusion_matrix(
             score_idx, misclf_labels[:, 0], misclf_labels[:, 1], 0
         ] = misclf_counts
 
-        missing_predictions[score_idx, misprd_labels, 0] = misprd_counts
+        unmatched_ground_truths[score_idx, misprd_labels, 0] = misprd_counts
 
         if n_examples > 0:
             for label_idx in range(n_labels):
@@ -375,16 +377,16 @@ def compute_confusion_matrix(
                                 1 : 2 * misclf_label_examples.shape[0] + 1,
                             ] = misclf_label_examples[:, [0, 3]].flatten()
 
-                # missing prediction examples
+                # unmatched ground truth examples
                 mask_misprd_label = misprd_examples[:, 1] == label_idx
                 if misprd_examples.size > 0:
                     misprd_label_examples = misprd_examples[mask_misprd_label][
                         :n_examples
                     ]
-                    missing_predictions[
+                    unmatched_ground_truths[
                         score_idx,
                         label_idx,
                         1 : misprd_label_examples.shape[0] + 1,
                     ] = misprd_label_examples[:, 0].flatten()
 
-    return confusion_matrix, missing_predictions
+    return confusion_matrix, unmatched_ground_truths  # type: ignore[reportReturnType]

valor_lite/classification/manager.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
+
 from valor_lite.classification.annotation import Classification
 from valor_lite.classification.computation import (
     compute_confusion_matrix,
@@ -38,7 +39,7 @@ filtered_metrics = evaluator.evaluate(filter_mask=filter_mask)
 
 @dataclass
 class Filter:
-    indices: NDArray[np.int32]
+    indices: NDArray[np.intp]
     label_metadata: NDArray[np.int32]
     n_datums: int
 
@@ -169,8 +170,7 @@ class Evaluator:
         label_metadata_per_datum = self._label_metadata_per_datum.copy()
         label_metadata_per_datum[:, ~mask] = 0
 
-        label_metadata = np.zeros_like(self._label_metadata, dtype=np.int32)
-        label_metadata = np.transpose(
+        label_metadata: NDArray[np.int32] = np.transpose(
             np.sum(
                 label_metadata_per_datum,
                 axis=1,

valor_lite/classification/metric.py

@@ -321,7 +321,7 @@ class Metric(BaseMetric):
                 ],
             ],
         ],
-        missing_predictions: dict[
+        unmatched_ground_truths: dict[
             str,  # ground truth label value
             dict[
                 str,  # either `count` or `examples`
@@ -335,8 +335,8 @@ class Metric(BaseMetric):
         The confusion matrix and related metrics for the classification task.
 
         This class encapsulates detailed information about the model's performance, including correct
-        predictions, misclassifications, hallucinations (false positives), and missing predictions
-        (false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
+        predictions, misclassifications, unmatched predictions (subset of false positives), and unmatched ground truths
+        (subset of false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
 
         Confusion Matrix Structure:
         {
@@ -358,7 +358,7 @@ class Metric(BaseMetric):
             ...
         }
 
-        Missing Prediction Structure:
+        Unmatched Ground Truths Structure:
         {
             ground_truth_label: {
                 'count': int,
@@ -379,7 +379,7 @@ class Metric(BaseMetric):
             A nested dictionary where the first key is the ground truth label value, the second key
             is the prediction label value, and the innermost dictionary contains either a `count`
             or a list of `examples`. Each example includes the datum UID and prediction score.
-        missing_predictions : dict
+        unmatched_ground_truths : dict
             A dictionary where each key is a ground truth label value for which the model failed to predict
             (false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
             Each example includes the datum UID.
@@ -396,7 +396,7 @@ class Metric(BaseMetric):
             type=MetricType.ConfusionMatrix.value,
             value={
                 "confusion_matrix": confusion_matrix,
-                "missing_predictions": missing_predictions,
+                "unmatched_ground_truths": unmatched_ground_truths,
             },
             parameters={
                 "score_threshold": score_threshold,

valor_lite/classification/utilities.py

@@ -2,6 +2,7 @@ from collections import defaultdict
 
 import numpy as np
 from numpy.typing import NDArray
+
 from valor_lite.classification.metric import Metric, MetricType
 
 
@@ -153,20 +154,20 @@ def _unpack_confusion_matrix_value(
     }
 
 
-def _unpack_missing_predictions_value(
-    missing_predictions: NDArray[np.int32],
+def _unpack_unmatched_ground_truths_value(
+    unmatched_ground_truths: NDArray[np.int32],
     number_of_labels: int,
     number_of_examples: int,
     index_to_uid: dict[int, str],
     index_to_label: dict[int, str],
 ) -> dict[str, dict[str, int | list[dict[str, str]]]]:
     """
-    Unpacks a numpy array of missing prediction counts and examples.
+    Unpacks a numpy array of unmatched ground truth counts and examples.
     """
 
     datum_idx = (
         lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            missing_predictions[
+            unmatched_ground_truths[
                 gt_label_idx,
                 example_idx + 1,
             ]
@@ -176,7 +177,7 @@ def _unpack_missing_predictions_value(
     return {
         index_to_label[gt_label_idx]: {
             "count": max(
-                int(missing_predictions[gt_label_idx, 0]),
+                int(unmatched_ground_truths[gt_label_idx, 0]),
                 0,
             ),
             "examples": [
@@ -197,7 +198,7 @@ def unpack_confusion_matrix_into_metric_list(
     index_to_label: dict[int, str],
 ) -> list[Metric]:
 
-    (confusion_matrix, missing_predictions) = results
+    (confusion_matrix, unmatched_ground_truths) = results
     n_scores, n_labels, _, _ = confusion_matrix.shape
     return [
         Metric.confusion_matrix(
@@ -210,8 +211,10 @@ def unpack_confusion_matrix_into_metric_list(
                 index_to_label=index_to_label,
                 index_to_uid=index_to_uid,
             ),
-            missing_predictions=_unpack_missing_predictions_value(
-                missing_predictions=missing_predictions[score_idx, :, :],
+            unmatched_ground_truths=_unpack_unmatched_ground_truths_value(
+                unmatched_ground_truths=unmatched_ground_truths[
+                    score_idx, :, :
+                ],
                 number_of_labels=n_labels,
                 number_of_examples=number_of_examples,
                 index_to_label=index_to_label,

valor_lite/object_detection/computation.py

@@ -381,9 +381,9 @@ def compute_precion_recall(
             _, indices_gt_unique = np.unique(
                 tp_candidates[:, [0, 1, 4]], axis=0, return_index=True
             )
-            mask_gt_unique = np.zeros(tp_candidates.shape[0], dtype=bool)
+            mask_gt_unique = np.zeros(tp_candidates.shape[0], dtype=np.bool_)
             mask_gt_unique[indices_gt_unique] = True
-            true_positives_mask = np.zeros(n_rows, dtype=bool)
+            true_positives_mask = np.zeros(n_rows, dtype=np.bool_)
             true_positives_mask[mask_tp_inner] = mask_gt_unique
 
             # calculate intermediates
@@ -452,9 +452,9 @@ def compute_precion_recall(
         _, indices_gt_unique = np.unique(
             tp_candidates[:, [0, 1, 4]], axis=0, return_index=True
         )
-        mask_gt_unique = np.zeros(tp_candidates.shape[0], dtype=bool)
+        mask_gt_unique = np.zeros(tp_candidates.shape[0], dtype=np.bool_)
         mask_gt_unique[indices_gt_unique] = True
-        true_positives_mask = np.zeros(n_rows, dtype=bool)
+        true_positives_mask = np.zeros(n_rows, dtype=np.bool_)
         true_positives_mask[mask_tp_outer] = mask_gt_unique
 
         # count running tp and total for AP
@@ -501,8 +501,8 @@ def compute_precion_recall(
         )
 
     # calculate average precision
-    running_max_precision = np.zeros((n_ious, n_labels))
-    running_max_score = np.zeros((n_labels))
+    running_max_precision = np.zeros((n_ious, n_labels), dtype=np.float64)
+    running_max_score = np.zeros((n_labels), dtype=np.float64)
     for recall in range(100, -1, -1):
 
         # running max precision
@@ -528,8 +528,12 @@ def compute_precion_recall(
 
     # calculate mAP and mAR
     if unique_pd_labels.size > 0:
-        mAP = average_precision[:, unique_pd_labels].mean(axis=1)
-        mAR = average_recall[:, unique_pd_labels].mean(axis=1)
+        mAP: NDArray[np.float64] = average_precision[:, unique_pd_labels].mean(
+            axis=1
+        )
+        mAR: NDArray[np.float64] = average_recall[:, unique_pd_labels].mean(
+            axis=1
+        )
     else:
         mAP = np.zeros(n_ious, dtype=np.float64)
         mAR = np.zeros(n_scores, dtype=np.float64)
@@ -561,14 +565,14 @@ def compute_precion_recall(
         accuracy,
         counts,
         pr_curve,
-    )
+    )  # type: ignore[reportReturnType]
 
 
 def _count_with_examples(
     data: NDArray[np.float64],
     unique_idx: int | list[int],
     label_idx: int | list[int],
-) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.int32]]:
+) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.intp]]:
     """
     Helper function for counting occurences of unique detailed pairs.
 
@@ -587,7 +591,7 @@ def _count_with_examples(
         Examples drawn from the data input.
     NDArray[np.int32]
         Unique label indices.
-    NDArray[np.int32]
+    NDArray[np.intp]
         Counts for each unique label index.
     """
     unique_rows, indices = np.unique(
@@ -669,9 +673,9 @@ def compute_confusion_matrix(
     NDArray[np.float64]
         Confusion matrix.
     NDArray[np.float64]
-        Hallucinations.
+        Unmatched Predictions.
     NDArray[np.int32]
-        Missing Predictions.
+        Unmatched Ground Truths.
     """
 
     n_labels = label_metadata.shape[0]
@@ -683,12 +687,12 @@ def compute_confusion_matrix(
         (n_ious, n_scores, n_labels, n_labels, 4 * n_examples + 1),
         dtype=np.float32,
     )
-    hallucinations = -1 * np.ones(
+    unmatched_predictions = -1 * np.ones(
         # (datum idx, pd idx, pd score) * n_examples + count
         (n_ious, n_scores, n_labels, 3 * n_examples + 1),
         dtype=np.float32,
     )
-    missing_predictions = -1 * np.ones(
+    unmatched_ground_truths = -1 * np.ones(
         # (datum idx, gt idx) * n_examples + count
         (n_ious, n_scores, n_labels, 2 * n_examples + 1),
         dtype=np.int32,
@@ -793,7 +797,7 @@ def compute_confusion_matrix(
                 data[mask_misclf], unique_idx=[0, 1, 2, 4, 5], label_idx=[3, 4]
             )
 
-            # count hallucinations
+            # count unmatched predictions
             (
                 halluc_examples,
                 halluc_labels,
@@ -802,7 +806,7 @@ def compute_confusion_matrix(
                 data[mask_halluc], unique_idx=[0, 2, 5], label_idx=2
             )
 
-            # count missing predictions
+            # count unmatched ground truths
             (
                 misprd_examples,
                 misprd_labels,
@@ -822,13 +826,13 @@ def compute_confusion_matrix(
                 misclf_labels[:, 1],
                 0,
             ] = misclf_counts
-            hallucinations[
+            unmatched_predictions[
                 iou_idx,
                 score_idx,
                 halluc_labels,
                 0,
             ] = halluc_counts
-            missing_predictions[
+            unmatched_ground_truths[
                 iou_idx,
                 score_idx,
                 misprd_labels,
@@ -877,26 +881,26 @@ def compute_confusion_matrix(
                                     :, [0, 1, 2, 6]
                                 ].flatten()
 
-                    # hallucination examples
+                    # unmatched prediction examples
                     mask_halluc_label = halluc_examples[:, 5] == label_idx
                     if mask_halluc_label.sum() > 0:
                         halluc_label_examples = halluc_examples[
                             mask_halluc_label
                         ][:n_examples]
-                        hallucinations[
+                        unmatched_predictions[
                             iou_idx,
                             score_idx,
                             label_idx,
                             1 : 3 * halluc_label_examples.shape[0] + 1,
                         ] = halluc_label_examples[:, [0, 2, 6]].flatten()
 
-                    # missing prediction examples
+                    # unmatched ground truth examples
                     mask_misprd_label = misprd_examples[:, 4] == label_idx
                     if misprd_examples.size > 0:
                         misprd_label_examples = misprd_examples[
                             mask_misprd_label
                         ][:n_examples]
-                        missing_predictions[
+                        unmatched_ground_truths[
                             iou_idx,
                             score_idx,
                             label_idx,
@@ -905,6 +909,6 @@ def compute_confusion_matrix(
 
     return (
         confusion_matrix,
-        hallucinations,
-        missing_predictions,
-    )
+        unmatched_predictions,
+        unmatched_ground_truths,
+    )  # type: ignore[reportReturnType]

valor_lite/object_detection/manager.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
+
 from valor_lite.object_detection.annotation import Detection
 from valor_lite.object_detection.computation import (
     compute_bbox_iou,
@@ -42,8 +43,8 @@ filtered_metrics = evaluator.evaluate(iou_thresholds=[0.5], filter_mask=filter_m
 
 @dataclass
 class Filter:
-    ranked_indices: NDArray[np.int32]
-    detailed_indices: NDArray[np.int32]
+    ranked_indices: NDArray[np.intp]
+    detailed_indices: NDArray[np.intp]
     label_metadata: NDArray[np.int32]
 
 
@@ -569,7 +570,8 @@ class DataLoader:
                         [gt.extrema, pd.extrema]
                         for pd in detection.predictions
                         for gt in detection.groundtruths
-                    ]
+                    ],
+                    dtype=np.float64,
                 )
             ).reshape(len(detection.predictions), len(detection.groundtruths))
             for detection in detections

valor_lite/object_detection/metric.py

@@ -619,7 +619,7 @@ class Metric(BaseMetric):
                 ],
             ],
         ],
-        hallucinations: dict[
+        unmatched_predictions: dict[
             str,  # prediction label value
             dict[
                 str,  # either `count` or `examples`
@@ -636,7 +636,7 @@ class Metric(BaseMetric):
                 ],
             ],
         ],
-        missing_predictions: dict[
+        unmatched_ground_truths: dict[
             str,  # ground truth label value
             dict[
                 str,  # either `count` or `examples`
@@ -660,8 +660,8 @@ class Metric(BaseMetric):
         Confusion matrix for object detection tasks.
 
         This class encapsulates detailed information about the model's performance, including correct
-        predictions, misclassifications, hallucinations (false positives), and missing predictions
-        (false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
+        predictions, misclassifications, unmatched_predictions (subset of false positives), and unmatched ground truths
+        (subset of false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
 
         Confusion Matrix Format:
         {
@@ -683,7 +683,7 @@ class Metric(BaseMetric):
             ...
         }
 
-        Hallucinations Format:
+        Unmatched Predictions Format:
         {
             <prediction label>: {
                 'count': int,
@@ -699,7 +699,7 @@ class Metric(BaseMetric):
             ...
         }
 
-        Missing Prediction Format:
+        Unmatched Ground Truths Format:
         {
             <ground truth label>: {
                 'count': int,
@@ -721,13 +721,13 @@ class Metric(BaseMetric):
             is the prediction label value, and the innermost dictionary contains either a `count`
             or a list of `examples`. Each example includes the datum UID, ground truth bounding box,
             predicted bounding box, and prediction scores.
-        hallucinations : dict
+        unmatched_predictions : dict
             A dictionary where each key is a prediction label value with no corresponding ground truth
-            (false positives). The value is a dictionary containing either a `count` or a list of
+            (subset of false positives). The value is a dictionary containing either a `count` or a list of
             `examples`. Each example includes the datum UID, predicted bounding box, and prediction score.
-        missing_predictions : dict
+        unmatched_ground_truths : dict
             A dictionary where each key is a ground truth label value for which the model failed to predict
-            (false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
+            (subset of false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
             Each example includes the datum UID and ground truth bounding box.
         score_threshold : float
             The confidence score threshold used to filter predictions.
@@ -744,8 +744,8 @@ class Metric(BaseMetric):
             type=MetricType.ConfusionMatrix.value,
             value={
                 "confusion_matrix": confusion_matrix,
-                "hallucinations": hallucinations,
-                "missing_predictions": missing_predictions,
+                "unmatched_predictions": unmatched_predictions,
+                "unmatched_ground_truths": unmatched_ground_truths,
             },
             parameters={
                 "score_threshold": score_threshold,

valor_lite/object_detection/utilities.py

@@ -2,6 +2,7 @@ from collections import defaultdict
 
 import numpy as np
 from numpy.typing import NDArray
+
 from valor_lite.object_detection.metric import Metric, MetricType
 
 
@@ -136,10 +137,8 @@ def unpack_precision_recall_into_metric_lists(
 
     metrics[MetricType.PrecisionRecallCurve] = [
         Metric.precision_recall_curve(
-            precisions=pr_curves[iou_idx, label_idx, :, 0]
-            .astype(float)
-            .tolist(),
-            scores=pr_curves[iou_idx, label_idx, :, 1].astype(float).tolist(),
+            precisions=pr_curves[iou_idx, label_idx, :, 0].tolist(),  # type: ignore[reportArgumentType]
+            scores=pr_curves[iou_idx, label_idx, :, 1].tolist(),  # type: ignore[reportArgumentType]
             iou_threshold=iou_threshold,
             label=label,
         )
@@ -321,8 +320,8 @@ def _unpack_confusion_matrix_value(
     }
 
 
-def _unpack_hallucinations_value(
-    hallucinations: NDArray[np.float64],
+def _unpack_unmatched_predictions_value(
+    unmatched_predictions: NDArray[np.float64],
     number_of_labels: int,
     number_of_examples: int,
     index_to_uid: dict[int, str],
@@ -336,12 +335,12 @@ def _unpack_hallucinations_value(
     ],
 ]:
     """
-    Unpacks a numpy array of hallucination counts and examples.
+    Unpacks a numpy array of unmatched_prediction counts and examples.
     """
 
     datum_idx = (
         lambda pd_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            hallucinations[
+            unmatched_predictions[
                 pd_label_idx,
                 example_idx * 3 + 1,
             ]
@@ -350,7 +349,7 @@ def _unpack_hallucinations_value(
 
     prediction_idx = (
         lambda pd_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            hallucinations[
+            unmatched_predictions[
                 pd_label_idx,
                 example_idx * 3 + 2,
             ]
@@ -359,7 +358,7 @@ def _unpack_hallucinations_value(
 
     score_idx = (
         lambda pd_label_idx, example_idx: float(  # noqa: E731 - lambda fn
-            hallucinations[
+            unmatched_predictions[
                 pd_label_idx,
                 example_idx * 3 + 3,
             ]
@@ -369,7 +368,7 @@ def _unpack_hallucinations_value(
     return {
         index_to_label[pd_label_idx]: {
             "count": max(
-                int(hallucinations[pd_label_idx, 0]),
+                int(unmatched_predictions[pd_label_idx, 0]),
                 0,
             ),
             "examples": [
@@ -392,8 +391,8 @@ def _unpack_hallucinations_value(
     }
 
 
-def _unpack_missing_predictions_value(
-    missing_predictions: NDArray[np.int32],
+def _unpack_unmatched_ground_truths_value(
+    unmatched_ground_truths: NDArray[np.int32],
     number_of_labels: int,
     number_of_examples: int,
     index_to_uid: dict[int, str],
@@ -401,12 +400,12 @@ def _unpack_missing_predictions_value(
     groundtruth_examples: dict[int, NDArray[np.float16]],
 ) -> dict[str, dict[str, int | list[dict[str, str | dict[str, float]]]]]:
     """
-    Unpacks a numpy array of missing prediction counts and examples.
+    Unpacks a numpy array of unmatched ground truth counts and examples.
     """
 
     datum_idx = (
         lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            missing_predictions[
+            unmatched_ground_truths[
                 gt_label_idx,
                 example_idx * 2 + 1,
             ]
@@ -415,7 +414,7 @@ def _unpack_missing_predictions_value(
 
     groundtruth_idx = (
         lambda gt_label_idx, example_idx: int(  # noqa: E731 - lambda fn
-            missing_predictions[
+            unmatched_ground_truths[
                 gt_label_idx,
                 example_idx * 2 + 2,
             ]
@@ -425,7 +424,7 @@ def _unpack_missing_predictions_value(
     return {
         index_to_label[gt_label_idx]: {
             "count": max(
-                int(missing_predictions[gt_label_idx, 0]),
+                int(unmatched_ground_truths[gt_label_idx, 0]),
                 0,
             ),
             "examples": [
@@ -463,8 +462,8 @@ def unpack_confusion_matrix_into_metric_list(
 ) -> list[Metric]:
     (
         confusion_matrix,
-        hallucinations,
-        missing_predictions,
+        unmatched_predictions,
+        unmatched_ground_truths,
     ) = results
     n_labels = len(index_to_label)
     return [
@@ -481,16 +480,18 @@ def unpack_confusion_matrix_into_metric_list(
                 groundtruth_examples=groundtruth_examples,
                 prediction_examples=prediction_examples,
             ),
-            hallucinations=_unpack_hallucinations_value(
-                hallucinations=hallucinations[iou_idx, score_idx, :, :],
+            unmatched_predictions=_unpack_unmatched_predictions_value(
+                unmatched_predictions=unmatched_predictions[
+                    iou_idx, score_idx, :, :
+                ],
                 number_of_labels=n_labels,
                 number_of_examples=number_of_examples,
                 index_to_label=index_to_label,
                 index_to_uid=index_to_uid,
                 prediction_examples=prediction_examples,
             ),
-            missing_predictions=_unpack_missing_predictions_value(
-                missing_predictions=missing_predictions[
+            unmatched_ground_truths=_unpack_unmatched_ground_truths_value(
+                unmatched_ground_truths=unmatched_ground_truths[
                     iou_idx, score_idx, :, :
                 ],
                 number_of_labels=n_labels,

valor_lite/semantic_segmentation/computation.py

@@ -98,9 +98,9 @@ def compute_metrics(
     NDArray[np.float64]
         Confusion matrix containing IOU values.
     NDArray[np.float64]
-        Hallucination ratios.
+        Unmatched prediction ratios.
     NDArray[np.float64]
-        Missing prediction ratios.
+        Unmatched ground truth ratios.
     """
     n_labels = label_metadata.shape[0]
     gt_counts = label_metadata[:, 0]
@@ -108,7 +108,7 @@ def compute_metrics(
 
     counts = data.sum(axis=0)
 
-    # compute iou, missing_predictions and hallucinations
+    # compute iou, unmatched_ground_truth and unmatched predictions
     intersection_ = counts[1:, 1:]
     union_ = (
         gt_counts[:, np.newaxis] + pd_counts[np.newaxis, :] - intersection_
@@ -122,20 +122,20 @@ def compute_metrics(
         out=ious,
     )
 
-    hallucination_ratio = np.zeros((n_labels), dtype=np.float64)
+    unmatched_prediction_ratio = np.zeros((n_labels), dtype=np.float64)
     np.divide(
         counts[0, 1:],
         pd_counts,
         where=pd_counts > 1e-9,
-        out=hallucination_ratio,
+        out=unmatched_prediction_ratio,
     )
 
-    missing_prediction_ratio = np.zeros((n_labels), dtype=np.float64)
+    unmatched_ground_truth_ratio = np.zeros((n_labels), dtype=np.float64)
     np.divide(
         counts[1:, 0],
         gt_counts,
         where=gt_counts > 1e-9,
-        out=missing_prediction_ratio,
+        out=unmatched_ground_truth_ratio,
     )
 
     # compute precision, recall, f1
@@ -168,6 +168,6 @@ def compute_metrics(
         f1_score,
         accuracy,
         ious,
-        hallucination_ratio,
-        missing_prediction_ratio,
+        unmatched_prediction_ratio,
+        unmatched_ground_truth_ratio,
     )

valor_lite/semantic_segmentation/manager.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
+
 from valor_lite.semantic_segmentation.annotation import Segmentation
 from valor_lite.semantic_segmentation.computation import (
     compute_intermediate_confusion_matrices,
@@ -37,7 +38,7 @@ filtered_metrics = evaluator.evaluate(filter_mask=filter_mask)
 
 @dataclass
 class Filter:
-    indices: NDArray[np.int32]
+    indices: NDArray[np.intp]
     label_metadata: NDArray[np.int32]
     n_pixels: int
 

valor_lite/semantic_segmentation/metric.py

@@ -209,11 +209,11 @@ class Metric(BaseMetric):
                 dict[str, float],  # iou
             ],
         ],
-        hallucinations: dict[
+        unmatched_predictions: dict[
             str,  # prediction label value
             dict[str, float],  # pixel ratio
         ],
-        missing_predictions: dict[
+        unmatched_ground_truths: dict[
             str,  # ground truth label value
             dict[str, float],  # pixel ratio
         ],
@@ -222,8 +222,8 @@ class Metric(BaseMetric):
         The confusion matrix and related metrics for semantic segmentation tasks.
 
         This class encapsulates detailed information about the model's performance, including correct
-        predictions, misclassifications, hallucinations (false positives), and missing predictions
-        (false negatives). It provides counts for each category to facilitate in-depth analysis.
+        predictions, misclassifications, unmatched_predictions (subset of false positives), and unmatched ground truths
+        (subset of false negatives). It provides counts for each category to facilitate in-depth analysis.
 
         Confusion Matrix Format:
         {
@@ -234,14 +234,14 @@ class Metric(BaseMetric):
             },
         }
 
-        Hallucinations Format:
+        Unmatched Predictions Format:
         {
             <prediction label>: {
                 'iou': <float>,
             },
         }
 
-        Missing Predictions Format:
+        Unmatched Ground Truths Format:
         {
             <ground truth label>: {
                 'iou': <float>,
@@ -253,10 +253,10 @@ class Metric(BaseMetric):
         confusion_matrix : dict
             Nested dictionaries representing the Intersection over Union (IOU) scores for each
             ground truth label and prediction label pair.
-        hallucinations : dict
+        unmatched_predictions : dict
             Dictionary representing the pixel ratios for predicted labels that do not correspond
             to any ground truth labels (false positives).
-        missing_predictions : dict
+        unmatched_ground_truths : dict
             Dictionary representing the pixel ratios for ground truth labels that were not predicted
             (false negatives).
 
@@ -268,8 +268,8 @@ class Metric(BaseMetric):
             type=MetricType.ConfusionMatrix.value,
             value={
                 "confusion_matrix": confusion_matrix,
-                "hallucinations": hallucinations,
-                "missing_predictions": missing_predictions,
+                "unmatched_predictions": unmatched_predictions,
+                "unmatched_ground_truths": unmatched_ground_truths,
             },
             parameters={},
         )

valor_lite/semantic_segmentation/utilities.py

@@ -2,6 +2,7 @@ from collections import defaultdict
 
 import numpy as np
 from numpy.typing import NDArray
+
 from valor_lite.semantic_segmentation.metric import Metric, MetricType
 
 
@@ -18,8 +19,8 @@ def unpack_precision_recall_iou_into_metric_lists(
         f1_score,
         accuracy,
         ious,
-        hallucination_ratios,
-        missing_prediction_ratios,
+        unmatched_prediction_ratios,
+        unmatched_ground_truth_ratios,
     ) = results
 
     metrics = defaultdict(list)
@@ -43,16 +44,16 @@ def unpack_precision_recall_iou_into_metric_lists(
                 for gt_label_idx in range(n_labels)
                 if label_metadata[gt_label_idx, 0] > 0
             },
-            hallucinations={
+            unmatched_predictions={
                 index_to_label[pd_label_idx]: {
-                    "ratio": float(hallucination_ratios[pd_label_idx])
+                    "ratio": float(unmatched_prediction_ratios[pd_label_idx])
                 }
                 for pd_label_idx in range(n_labels)
                 if label_metadata[pd_label_idx, 0] > 0
             },
-            missing_predictions={
+            unmatched_ground_truths={
                 index_to_label[gt_label_idx]: {
-                    "ratio": float(missing_prediction_ratios[gt_label_idx])
+                    "ratio": float(unmatched_ground_truth_ratios[gt_label_idx])
                 }
                 for gt_label_idx in range(n_labels)
                 if label_metadata[gt_label_idx, 0] > 0

valor_lite/text_generation/computation.py

@@ -1,6 +1,7 @@
 import evaluate
 from nltk.tokenize import RegexpTokenizer
 from nltk.translate import bleu_score
+
 from valor_lite.text_generation.llm.generation import (
     generate_answer_correctness_verdicts,
     generate_answer_relevance_verdicts,

{valor_lite-0.33.18.dist-info → valor_lite-0.34.0.dist-info}/METADATA

@@ -1,50 +1,33 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.18
-Summary: Compute valor metrics locally.
-License: MIT License
-        
-        Copyright (c) 2023 Striveworks
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
-        
+Version: 0.34.0
+Summary: Evaluate machine learning models.
 Project-URL: homepage, https://www.striveworks.com
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: evaluate
-Requires-Dist: nltk
 Requires-Dist: numpy
-Requires-Dist: Pillow >=9.1.0
-Requires-Dist: requests
-Requires-Dist: rouge-score
-Requires-Dist: shapely
 Requires-Dist: tqdm
-Requires-Dist: importlib-metadata ; python_version < "3.8"
+Requires-Dist: shapely
+Requires-Dist: evaluate
+Requires-Dist: nltk
+Requires-Dist: rouge_score
 Provides-Extra: mistral
-Requires-Dist: mistralai >=1.0 ; extra == 'mistral'
+Requires-Dist: mistralai>=1.0; extra == "mistral"
 Provides-Extra: openai
-Requires-Dist: openai ; extra == 'openai'
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: test
-Requires-Dist: pytest ; extra == 'test'
-Requires-Dist: coverage ; extra == 'test'
-Requires-Dist: pre-commit ; extra == 'test'
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: coverage; extra == "test"
+Requires-Dist: pre-commit; extra == "test"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: mkdocstrings; extra == "docs"
+Requires-Dist: mkdocstrings-python; extra == "docs"
+Requires-Dist: mkdocs-include-dir-to-nav; extra == "docs"
+Requires-Dist: mkdocs-swagger-ui-tag; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: valor-lite[docs,mistral,openai,test]; extra == "dev"
 
 # valor-lite: Fast, local machine learning evaluation.
 

{valor_lite-0.33.18.dist-info → valor_lite-0.34.0.dist-info}/RECORD

@@ -4,26 +4,26 @@ valor_lite/profiling.py,sha256=TLIROA1qccFw9NoEkMeQcrvvGGO75c4K5yTIWoCUix8,11746
 valor_lite/schemas.py,sha256=pB0MrPx5qFLbwBWDiOUUm-vmXdWvbJLFCBmKgbcbI5g,198
 valor_lite/classification/__init__.py,sha256=8MI8bGwCxYGqRP7KxG7ezhYv4qQ5947XGvvlF8WPM5g,392
 valor_lite/classification/annotation.py,sha256=0aUOvcwBAZgiNOJuyh-pXyNTG7vP7r8CUfnU3OmpUwQ,1113
-valor_lite/classification/computation.py,sha256=qfBhhuDYCiY8h2RdBG3shzgJbHLXDVNujkYFg9xZa6U,12116
-valor_lite/classification/manager.py,sha256=8GXZECSx4CBbG5NfPrA19BPENqmrjo-wZBmaulWHY20,16676
-valor_lite/classification/metric.py,sha256=fkAo-_3s4EIRSkyn3owBSf4_Gp6lBK9xdToDYMWmT8A,12236
-valor_lite/classification/utilities.py,sha256=PmQar06Vt-ew4Jvnn0IM63mq730QVTsdRtFdVu1HMFU,6885
+valor_lite/classification/computation.py,sha256=dOcq1xxarwwwtHya1Wl8KZXE-PoGafOAYK0LQqxGphY,12191
+valor_lite/classification/manager.py,sha256=cZ6-DKao59QqF0JF_U26tBoydpCElAAH8rRyX_Kc6bc,16618
+valor_lite/classification/metric.py,sha256=_mW3zynmpW8jUIhK2OeX4usdftHgHM9_l7EAbEe2N3w,12288
+valor_lite/classification/utilities.py,sha256=eG-Qhd213uf2GXuuqsPxCgBRBFV-z_ADbzneF1kE368,6964
 valor_lite/object_detection/__init__.py,sha256=Ql8rju2q7y0Zd9zFvtBJDRhgQFDm1RSYkTsyH3ZE6pA,648
 valor_lite/object_detection/annotation.py,sha256=x9bsl8b75yvkMByXXiIYI9d9T03olDqtykSvKJc3aFw,7729
-valor_lite/object_detection/computation.py,sha256=P5ijxEBuZ3mxYjBQy24TiQpGxRmPuS40Gwn44uv0J7M,28064
-valor_lite/object_detection/manager.py,sha256=utdILUUCx04EWC0_bHGpEPaxcCOhmsOx5lxT9qU1a9s,23033
-valor_lite/object_detection/metric.py,sha256=8QhdauuaRrzE39idetkFYTPxA12wrBalQDIR4IUzEbg,24794
-valor_lite/object_detection/utilities.py,sha256=98VSW-g8EYI8Cdd9KHLHdm6F4fI89jaX5I4z99zny4s,16271
+valor_lite/object_detection/computation.py,sha256=Q9sFMQhRupseZmGZzA7uEpqpeo9XpmSIkeKGMNEZfP4,28345
+valor_lite/object_detection/manager.py,sha256=wjjMvzCoWs5oavtSj1NT-o4QFT5Sqf79WIb_OO64y1o,23071
+valor_lite/object_detection/metric.py,sha256=npK2sxiwCUTKlRlFym1AlZTvP9herf9lakbsBDwljGM,24901
+valor_lite/object_detection/utilities.py,sha256=42RRyP6L3eWtDY_f7qs7f0WTjhcibmUBu2I4yAwupF0,16456
 valor_lite/semantic_segmentation/__init__.py,sha256=BhTUbwbdJa1FdS4ZA3QSIZ8TuJmdGGLGCd5hX6SzKa4,297
 valor_lite/semantic_segmentation/annotation.py,sha256=xd2qJyIeTW8CT_Goyu3Kvl_51b9b6D3WvUfqwShR0Sk,4990
 valor_lite/semantic_segmentation/benchmark.py,sha256=iVdxUo9LgDbbXUa6eRhZ49LOYw-yyr2W4p9FP3KHg0k,3848
-valor_lite/semantic_segmentation/computation.py,sha256=myHjJZ70f2Xc-PGHx3DcLWvXXRu_H8w9z20n7qV-Abo,4687
-valor_lite/semantic_segmentation/manager.py,sha256=TtwJI7Bsn3zHL2ECOqCmymG-JqREo7I6qxYtycbz54Y,14322
-valor_lite/semantic_segmentation/metric.py,sha256=aJv3wPEl6USLhZ3c4yz6prnBU-EaG4Kz16f0BXcodd4,7046
-valor_lite/semantic_segmentation/utilities.py,sha256=vZM66YNMz9VJclhuKvcWp74nF65s6bscnnD5U9iDW7Q,2925
+valor_lite/semantic_segmentation/computation.py,sha256=l98h8s9RTWQOB_eg2rconqGL1ZbTS4GMtz69vbyEdQ0,4741
+valor_lite/semantic_segmentation/manager.py,sha256=p0RuV27S1NTBeYZD6G9dSdOcl3yuRLrjL_SMUjEgRXE,14322
+valor_lite/semantic_segmentation/metric.py,sha256=T9RfPJf4WgqGQTXYvSy08vJG5bjXXJnyYZeW0mlxMa8,7132
+valor_lite/semantic_segmentation/utilities.py,sha256=UD0X-iCWMR8Rmw2YaP4HM3lxwhYwo-yNGzF-taAJ8RA,2959
 valor_lite/text_generation/__init__.py,sha256=pGhpWCSZjLM0pPHCtPykAfos55B8ie3mi9EzbNxfj-U,356
 valor_lite/text_generation/annotation.py,sha256=O5aXiwCS4WjA-fqn4ly-O0MsTHoIOmqxqCaAp9IeI3M,1270
-valor_lite/text_generation/computation.py,sha256=cG35qMpxNPEYHXN2fz8wcanESriSHoWMl1idpm9-ous,18638
+valor_lite/text_generation/computation.py,sha256=vdZTajB0OuGEwoKWxKkq3lWxlPaUsrp_X5-ZgjkYLhg,18639
 valor_lite/text_generation/manager.py,sha256=C4QwvronGHXmYSkaRmUGy7TN0C0aeyDx9Hb-ClNYXK4,24810
 valor_lite/text_generation/metric.py,sha256=C9gbWejjOJ23JVLecuUhYW5rkx30NUCfRtgsM46uMds,10409
 valor_lite/text_generation/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -33,8 +33,7 @@ valor_lite/text_generation/llm/instructions.py,sha256=fz2onBZZWcl5W8iy7zEWkPGU9N
 valor_lite/text_generation/llm/integrations.py,sha256=-rTfdAjq1zH-4ixwYuMQEOQ80pIFzMTe0BYfroVx3Pg,6974
 valor_lite/text_generation/llm/utilities.py,sha256=bjqatGgtVTcl1PrMwiDKTYPGJXKrBrx7PDtzIblGSys,1178
 valor_lite/text_generation/llm/validators.py,sha256=Wzr5RlfF58_2wOU-uTw7C8skan_fYdhy4Gfn0jSJ8HM,2700
-valor_lite-0.33.18.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
-valor_lite-0.33.18.dist-info/METADATA,sha256=oo3sEQQvJJvAIelgFRB1Me2Jmkk-nb_dkphL2k4wo7Y,5888
-valor_lite-0.33.18.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-valor_lite-0.33.18.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
-valor_lite-0.33.18.dist-info/RECORD,,
+valor_lite-0.34.0.dist-info/METADATA,sha256=2l5zKt0RkVHbdYmC1kLF71rG_dKZnZQ-XyuAQGf7dTg,4908
+valor_lite-0.34.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+valor_lite-0.34.0.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.34.0.dist-info/RECORD,,

{valor_lite-0.33.18.dist-info → valor_lite-0.34.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.5.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

valor_lite-0.33.18.dist-info/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2023 Striveworks
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.