valor-lite 0.33.1__tar.gz → 0.33.3__tar.gz

{valor_lite-0.33.1 → valor_lite-0.33.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.1
+Version: 0.33.3
 Summary: Compute valor metrics directly in your client.
 License: MIT License
         

{valor_lite-0.33.1 → valor_lite-0.33.3}/benchmarks/benchmark_objdet.py

@@ -8,7 +8,7 @@ from time import time
 
 import requests
 from tqdm import tqdm
-from valor_lite.detection import DataLoader
+from valor_lite.detection import DataLoader, MetricType
 
 
 class AnnotationType(str, Enum):
@@ -258,24 +258,48 @@ def run_benchmarking_analysis(
                     f"Base precomputation timed out with limit of {limit}."
                 )
 
-            # test detailed counts with no samples
-            detailed_counts_time_no_samples, _ = time_it(
-                evaluator.compute_detailed_counts
-            )()
-
-            # test detailed counts with 3 samples
-            detailed_counts_time_three_samples, _ = time_it(
-                evaluator.compute_detailed_counts
-            )(n_samples=3)
-
-            # evaluate
+            # evaluate - base metrics only
             eval_time, metrics = time_it(evaluator.evaluate)()
-            # print(metrics)
             if eval_time > evaluation_timeout and evaluation_timeout != -1:
                 raise TimeoutError(
                     f"Base evaluation timed out with {evaluator.n_datums} datums."
                 )
 
+            # evaluate - base metrics + detailed counts with no samples
+            detailed_counts_time_no_samples, metrics = time_it(
+                evaluator.evaluate
+            )(
+                [
+                    MetricType.DetailedCounts,
+                    *MetricType.base_metrics(),
+                ]
+            )
+            if (
+                detailed_counts_time_no_samples > evaluation_timeout
+                and evaluation_timeout != -1
+            ):
+                raise TimeoutError(
+                    f"Detailed evaluation w/ no samples timed out with {evaluator.n_datums} datums."
+                )
+
+            # evaluate - base metrics + detailed counts with 3 samples
+            detailed_counts_time_three_samples, metrics = time_it(
+                evaluator.evaluate
+            )(
+                [
+                    MetricType.DetailedCounts,
+                    *MetricType.base_metrics(),
+                ],
+                number_of_examples=3,
+            )
+            if (
+                detailed_counts_time_three_samples > evaluation_timeout
+                and evaluation_timeout != -1
+            ):
+                raise TimeoutError(
+                    f"Detailed w/ 3 samples evaluation timed out with {evaluator.n_datums} datums."
+                )
+
             results.append(
                 Benchmark(
                     limit=limit,

{valor_lite-0.33.1 → valor_lite-0.33.3}/tests/detection/test_average_precision.py

@@ -629,6 +629,17 @@ def test_ap_ranked_pair_ordering(detection_ranked_pair_ordering: Detection):
     loader.add_data(detections=[detection_ranked_pair_ordering])
     evaluator = loader.finalize()
 
+    assert evaluator.metadata == {
+        "ignored_prediction_labels": [
+            ("class", "label4"),
+        ],
+        "missing_prediction_labels": [],
+        "n_datums": 1,
+        "n_groundtruths": 3,
+        "n_labels": 4,
+        "n_predictions": 4,
+    }
+
     metrics = evaluator.evaluate(iou_thresholds=[0.5, 0.75])
 
     actual_metrics = [m.to_dict() for m in metrics[MetricType.AP]]

{valor_lite-0.33.1 → valor_lite-0.33.3}/tests/detection/test_average_recall.py

@@ -432,3 +432,77 @@ def test_ar_true_positive_deassignment(
         assert m in expected_metrics
     for m in expected_metrics:
         assert m in actual_metrics
+
+
+def test_ar_ranked_pair_ordering(detection_ranked_pair_ordering: Detection):
+
+    loader = DataLoader()
+    loader.add_data(detections=[detection_ranked_pair_ordering])
+    evaluator = loader.finalize()
+
+    assert evaluator.metadata == {
+        "ignored_prediction_labels": [
+            ("class", "label4"),
+        ],
+        "missing_prediction_labels": [],
+        "n_datums": 1,
+        "n_groundtruths": 3,
+        "n_labels": 4,
+        "n_predictions": 4,
+    }
+
+    metrics = evaluator.evaluate(
+        iou_thresholds=[0.5, 0.75], score_thresholds=[0.0]
+    )
+
+    actual_metrics = [m.to_dict() for m in metrics[MetricType.AR]]
+    expected_metrics = expected_metrics = [
+        {
+            "type": "AR",
+            "value": 1.0,
+            "parameters": {
+                "score_threshold": 0.0,
+                "iou_thresholds": [0.5, 0.75],
+                "label": {"key": "class", "value": "label1"},
+            },
+        },
+        {
+            "type": "AR",
+            "value": 1.0,
+            "parameters": {
+                "score_threshold": 0.0,
+                "iou_thresholds": [0.5, 0.75],
+                "label": {"key": "class", "value": "label2"},
+            },
+        },
+        {
+            "type": "AR",
+            "value": 0.0,
+            "parameters": {
+                "score_threshold": 0.0,
+                "iou_thresholds": [0.5, 0.75],
+                "label": {"key": "class", "value": "label3"},
+            },
+        },
+    ]
+    for m in actual_metrics:
+        assert m in expected_metrics
+    for m in expected_metrics:
+        assert m in actual_metrics
+
+    actual_metrics = [m.to_dict() for m in metrics[MetricType.mAR]]
+    expected_metrics = expected_metrics = [
+        {
+            "type": "mAR",
+            "value": 0.6666666666666666,
+            "parameters": {
+                "score_threshold": 0.0,
+                "iou_thresholds": [0.5, 0.75],
+                "label_key": "class",
+            },
+        },
+    ]
+    for m in actual_metrics:
+        assert m in expected_metrics
+    for m in expected_metrics:
+        assert m in actual_metrics

{valor_lite-0.33.1 → valor_lite-0.33.3}/tests/detection/test_counts.py

@@ -41,52 +41,52 @@ def test_counts_metrics(basic_detections: list[Detection]):
         {
             "type": "Counts",
             "value": {
-                "tp": 0,
-                "fp": 1,
+                "tp": 1,
+                "fp": 0,
                 "fn": 1,
             },
             "parameters": {
                 "iou_threshold": 0.1,
                 "score_threshold": 0.0,
-                "label": {"key": "k2", "value": "v2"},
+                "label": {"key": "k1", "value": "v1"},
             },
         },
         {
             "type": "Counts",
             "value": {
-                "tp": 0,
-                "fp": 1,
+                "tp": 1,
+                "fp": 0,
                 "fn": 1,
             },
             "parameters": {
                 "iou_threshold": 0.6,
                 "score_threshold": 0.0,
-                "label": {"key": "k2", "value": "v2"},
+                "label": {"key": "k1", "value": "v1"},
             },
         },
         {
             "type": "Counts",
             "value": {
-                "tp": 1,
+                "tp": 0,
                 "fp": 0,
-                "fn": 1,
+                "fn": 2,
             },
             "parameters": {
                 "iou_threshold": 0.1,
-                "score_threshold": 0.0,
+                "score_threshold": 0.5,
                 "label": {"key": "k1", "value": "v1"},
             },
         },
         {
             "type": "Counts",
             "value": {
-                "tp": 1,
+                "tp": 0,
                 "fp": 0,
-                "fn": 1,
+                "fn": 2,
             },
             "parameters": {
                 "iou_threshold": 0.6,
-                "score_threshold": 0.0,
+                "score_threshold": 0.5,
                 "label": {"key": "k1", "value": "v1"},
             },
         },
@@ -99,7 +99,7 @@ def test_counts_metrics(basic_detections: list[Detection]):
             },
             "parameters": {
                 "iou_threshold": 0.1,
-                "score_threshold": 0.5,
+                "score_threshold": 0.0,
                 "label": {"key": "k2", "value": "v2"},
             },
         },
@@ -112,7 +112,7 @@ def test_counts_metrics(basic_detections: list[Detection]):
             },
             "parameters": {
                 "iou_threshold": 0.6,
-                "score_threshold": 0.5,
+                "score_threshold": 0.0,
                 "label": {"key": "k2", "value": "v2"},
             },
         },
@@ -120,26 +120,26 @@ def test_counts_metrics(basic_detections: list[Detection]):
             "type": "Counts",
             "value": {
                 "tp": 0,
-                "fp": 0,
-                "fn": 2,
+                "fp": 1,
+                "fn": 1,
             },
             "parameters": {
                 "iou_threshold": 0.1,
                 "score_threshold": 0.5,
-                "label": {"key": "k1", "value": "v1"},
+                "label": {"key": "k2", "value": "v2"},
             },
         },
         {
             "type": "Counts",
             "value": {
                 "tp": 0,
-                "fp": 0,
-                "fn": 2,
+                "fp": 1,
+                "fn": 1,
             },
             "parameters": {
                 "iou_threshold": 0.6,
                 "score_threshold": 0.5,
-                "label": {"key": "k1", "value": "v1"},
+                "label": {"key": "k2", "value": "v2"},
             },
         },
     ]
@@ -455,3 +455,89 @@ def test_counts_false_negatives_two_datums_one_only_with_different_class_high_co
         assert m in actual_metrics
     for m in actual_metrics:
         assert m in expected_metrics
+
+
+def test_counts_ranked_pair_ordering(
+    detection_ranked_pair_ordering: Detection,
+):
+
+    loader = DataLoader()
+    loader.add_data(detections=[detection_ranked_pair_ordering])
+    evaluator = loader.finalize()
+
+    assert evaluator.metadata == {
+        "ignored_prediction_labels": [
+            ("class", "label4"),
+        ],
+        "missing_prediction_labels": [],
+        "n_datums": 1,
+        "n_groundtruths": 3,
+        "n_labels": 4,
+        "n_predictions": 4,
+    }
+
+    metrics = evaluator.evaluate(
+        iou_thresholds=[0.5, 0.75], score_thresholds=[0.0]
+    )
+
+    actual_metrics = [m.to_dict() for m in metrics[MetricType.Counts]]
+    expected_metrics = [
+        {
+            "type": "Counts",
+            "value": {"tp": 1, "fp": 0, "fn": 0},
+            "parameters": {
+                "iou_threshold": 0.5,
+                "score_threshold": 0.0,
+                "label": {"key": "class", "value": "label1"},
+            },
+        },
+        {
+            "type": "Counts",
+            "value": {"tp": 1, "fp": 0, "fn": 0},
+            "parameters": {
+                "iou_threshold": 0.75,
+                "score_threshold": 0.0,
+                "label": {"key": "class", "value": "label1"},
+            },
+        },
+        {
+            "type": "Counts",
+            "value": {"tp": 1, "fp": 0, "fn": 0},
+            "parameters": {
+                "iou_threshold": 0.5,
+                "score_threshold": 0.0,
+                "label": {"key": "class", "value": "label2"},
+            },
+        },
+        {
+            "type": "Counts",
+            "value": {"tp": 1, "fp": 0, "fn": 0},
+            "parameters": {
+                "iou_threshold": 0.75,
+                "score_threshold": 0.0,
+                "label": {"key": "class", "value": "label2"},
+            },
+        },
+        {
+            "type": "Counts",
+            "value": {"tp": 0, "fp": 1, "fn": 1},
+            "parameters": {
+                "iou_threshold": 0.5,
+                "score_threshold": 0.0,
+                "label": {"key": "class", "value": "label3"},
+            },
+        },
+        {
+            "type": "Counts",
+            "value": {"tp": 0, "fp": 1, "fn": 1},
+            "parameters": {
+                "iou_threshold": 0.75,
+                "score_threshold": 0.0,
+                "label": {"key": "class", "value": "label3"},
+            },
+        },
+    ]
+    for m in actual_metrics:
+        assert m in expected_metrics
+    for m in expected_metrics:
+        assert m in actual_metrics

{valor_lite-0.33.1 → valor_lite-0.33.3}/tests/detection/test_dataloader.py

@@ -22,7 +22,7 @@ def test_valor_integration():
     loader.add_data_from_valor_dict([(gt, pd)])
 
     assert len(loader.pairs) == 1
-    assert loader.pairs[0].shape == (281, 7)
+    assert loader.pairs[0].shape == (71, 7)
 
     assert set(loader._evaluator.label_key_to_index.keys()) == {
         "iscrowd",