valor-lite 0.33.2__tar.gz → 0.33.3__tar.gz

{valor_lite-0.33.2 → valor_lite-0.33.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.2
+Version: 0.33.3
 Summary: Compute valor metrics directly in your client.
 License: MIT License
         

{valor_lite-0.33.2 → valor_lite-0.33.3}/benchmarks/benchmark_objdet.py

@@ -8,7 +8,7 @@ from time import time
 
 import requests
 from tqdm import tqdm
-from valor_lite.detection import DataLoader
+from valor_lite.detection import DataLoader, MetricType
 
 
 class AnnotationType(str, Enum):
@@ -258,24 +258,48 @@ def run_benchmarking_analysis(
                     f"Base precomputation timed out with limit of {limit}."
                 )
 
-            # test detailed counts with no samples
-            detailed_counts_time_no_samples, _ = time_it(
-                evaluator.compute_detailed_counts
-            )()
-
-            # test detailed counts with 3 samples
-            detailed_counts_time_three_samples, _ = time_it(
-                evaluator.compute_detailed_counts
-            )(n_samples=3)
-
-            # evaluate
+            # evaluate - base metrics only
             eval_time, metrics = time_it(evaluator.evaluate)()
-            # print(metrics)
             if eval_time > evaluation_timeout and evaluation_timeout != -1:
                 raise TimeoutError(
                     f"Base evaluation timed out with {evaluator.n_datums} datums."
                 )
 
+            # evaluate - base metrics + detailed counts with no samples
+            detailed_counts_time_no_samples, metrics = time_it(
+                evaluator.evaluate
+            )(
+                [
+                    MetricType.DetailedCounts,
+                    *MetricType.base_metrics(),
+                ]
+            )
+            if (
+                detailed_counts_time_no_samples > evaluation_timeout
+                and evaluation_timeout != -1
+            ):
+                raise TimeoutError(
+                    f"Detailed evaluation w/ no samples timed out with {evaluator.n_datums} datums."
+                )
+
+            # evaluate - base metrics + detailed counts with 3 samples
+            detailed_counts_time_three_samples, metrics = time_it(
+                evaluator.evaluate
+            )(
+                [
+                    MetricType.DetailedCounts,
+                    *MetricType.base_metrics(),
+                ],
+                number_of_examples=3,
+            )
+            if (
+                detailed_counts_time_three_samples > evaluation_timeout
+                and evaluation_timeout != -1
+            ):
+                raise TimeoutError(
+                    f"Detailed w/ 3 samples evaluation timed out with {evaluator.n_datums} datums."
+                )
+
             results.append(
                 Benchmark(
                     limit=limit,

{valor_lite-0.33.2 → valor_lite-0.33.3}/tests/detection/test_dataloader.py

@@ -22,7 +22,7 @@ def test_valor_integration():
     loader.add_data_from_valor_dict([(gt, pd)])
 
     assert len(loader.pairs) == 1
-    assert loader.pairs[0].shape == (281, 7)
+    assert loader.pairs[0].shape == (71, 7)
 
     assert set(loader._evaluator.label_key_to_index.keys()) == {
         "iscrowd",