valor-lite 0.33.17__tar.gz → 0.33.19__tar.gz

{valor_lite-0.33.17/valor_lite.egg-info → valor_lite-0.33.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.17
+Version: 0.33.19
 Summary: Compute valor metrics locally.
 License: MIT License
         

valor_lite-0.33.19/benchmarks/synthetic/benchmark_semantic_segmentation.py

@@ -0,0 +1,94 @@
+from valor_lite.profiling import Benchmark, BenchmarkError
+from valor_lite.semantic_segmentation.benchmark import (
+    benchmark_add_data,
+    benchmark_evaluate,
+    benchmark_finalize,
+)
+
+
+def benchmark(
+    bitmask_shape: tuple[int, int],
+    number_of_unique_labels: int,
+    number_of_images: int,
+    *_,
+    memory_limit: float = 4.0,
+    time_limit: float = 10.0,
+    repeat: int = 1,
+    verbose: bool = False,
+):
+    """
+    Runs a single benchmark.
+
+    Parameters
+    ----------
+    bitmask_shape : tuple[int, int]
+        The size (h, w) of the bitmask array.
+    number_of_unique_labels : int
+        The number of unique labels used in the synthetic example.
+    number_of_images : int
+        The number of distinct datums that are created.
+    memory_limit : float
+        The maximum amount of system memory allowed in gigabytes (GB).
+    time_limit : float
+        The maximum amount of time permitted before killing the benchmark.
+    repeat : int
+        The number of times to run a benchmark to produce an average runtime.
+    verbose : bool, default=False
+        Toggles terminal output of benchmark results.
+    """
+
+    b = Benchmark(
+        time_limit=time_limit,
+        memory_limit=int(memory_limit * (1024**3)),
+        repeat=repeat,
+        verbose=verbose,
+    )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_add_data,
+        n_labels=[number_of_unique_labels],
+        shape=[bitmask_shape],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_finalize,
+        n_datums=[number_of_images],
+        n_labels=[number_of_unique_labels],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_evaluate,
+        n_datums=[number_of_images],
+        n_labels=[number_of_unique_labels],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+
+if __name__ == "__main__":
+
+    benchmark(
+        bitmask_shape=(4000, 4000),
+        number_of_images=1000,
+        number_of_unique_labels=10,
+        memory_limit=4.0,
+        time_limit=10.0,
+        repeat=1,
+        verbose=True,
+    )

valor_lite-0.33.19/examples/benchmarking.ipynb

@@ -0,0 +1,279 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from valor_lite.profiling import Benchmark\n",
+    "\n",
+    "b = Benchmark(\n",
+    "    time_limit=5.0,  # 5s\n",
+    "    memory_limit=8 * (1024 ** 3),  # 8 GB\n",
+    "    repeat=1,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Semantic Segmentation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from valor_lite.semantic_segmentation.benchmark import (\n",
+    "    benchmark_add_data as semseg_add_data,\n",
+    "    benchmark_finalize as semseg_finalize,\n",
+    "    benchmark_evaluate as semseg_evaluate,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "n_datums = [\n",
+    "    10000,\n",
+    "    1000,\n",
+    "    100,\n",
+    "    10,\n",
+    "    1,\n",
+    "]\n",
+    "\n",
+    "n_labels = [\n",
+    "    1000,\n",
+    "    100,\n",
+    "    10,\n",
+    "    3,\n",
+    "]\n",
+    "\n",
+    "shapes = [\n",
+    "    (10000, 10000),\n",
+    "    (2500, 2500),\n",
+    "    (1000, 1000),\n",
+    "    (100, 100),\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 69%|██████▉   | 11/16 [00:46<00:21,  4.26s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=====================================================================\n",
+      "Details\n",
+      "{\n",
+      "    \"benchmark\": \"benchmark_add_data\",\n",
+      "    \"limits\": {\n",
+      "        \"memory_limit\": \"8.0 GB\",\n",
+      "        \"time_limit\": \"5.0 seconds\",\n",
+      "        \"repeat\": 1\n",
+      "    },\n",
+      "    \"passed\": 8,\n",
+      "    \"failed\": 8,\n",
+      "    \"total\": 16\n",
+      "}\n",
+      "\n",
+      "Passed\n",
+      "  complexity    |     runtime     |    n_labels     |      shape     \n",
+      "---------------------------------------------------------------------\n",
+      "   300000000    |     1.5151      |        3        | (10000, 10000) \n",
+      "   62500000     |     0.5952      |       10        |  (2500, 2500)  \n",
+      "   10000000     |     0.0911      |       10        |  (1000, 1000)  \n",
+      "    1000000     |     0.0582      |       100       |   (100, 100)   \n",
+      "\n",
+      "Failed\n",
+      "  complexity    |      error      |    n_labels     |      shape      |       msg      \n",
+      "---------------------------------------------------------------------------------------\n",
+      " 100000000000   |   MemoryError   |      1000       | (10000, 10000)  | Unable to allocate 186. GiB for an array with shape (1001, 20000, 10000) and data type bool\n",
+      "  10000000000   |   MemoryError   |       100       | (10000, 10000)  | Unable to allocate 18.8 GiB for an array with shape (101, 20000, 10000) and data type bool\n",
+      "  6250000000    |   MemoryError   |      1000       |  (2500, 2500)   | Unable to allocate 11.7 GiB for an array with shape (1001, 5000, 2500) and data type bool\n",
+      "  1000000000    |   MemoryError   |       10        | (10000, 10000)  | Unable to allocate 9.31 GiB for an array with shape (10, 10, 100000000) and data type bool\n",
+      "  1000000000    |   MemoryError   |      1000       |  (1000, 1000)   | Unable to allocate 931. GiB for an array with shape (1000, 1000, 1000000) and data type bool\n",
+      "   625000000    |   MemoryError   |       100       |  (2500, 2500)   | Unable to allocate 58.2 GiB for an array with shape (100, 100, 6250000) and data type bool\n",
+      "   100000000    |   MemoryError   |       100       |  (1000, 1000)   | Unable to allocate 9.31 GiB for an array with shape (100, 100, 1000000) and data type bool\n",
+      "   10000000     |   MemoryError   |      1000       |   (100, 100)    | Unable to allocate 9.31 GiB for an array with shape (1000, 1000, 10000) and data type bool\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "_ = b.run(\n",
+    "    benchmark=semseg_add_data,\n",
+    "    n_labels=n_labels,\n",
+    "    shape=shapes,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 20%|██        | 4/20 [02:35<10:22, 38.92s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=====================================================================\n",
+      "Details\n",
+      "{\n",
+      "    \"benchmark\": \"benchmark_finalize\",\n",
+      "    \"limits\": {\n",
+      "        \"memory_limit\": \"8.0 GB\",\n",
+      "        \"time_limit\": \"5.0 seconds\",\n",
+      "        \"repeat\": 1\n",
+      "    },\n",
+      "    \"passed\": 18,\n",
+      "    \"failed\": 2,\n",
+      "    \"total\": 20\n",
+      "}\n",
+      "\n",
+      "Passed\n",
+      "  complexity    |     runtime     |    n_datums     |    n_labels    \n",
+      "---------------------------------------------------------------------\n",
+      "    1000000     |     1.1142      |      10000      |       100      \n",
+      "    100000      |     0.1748      |       100       |      1000      \n",
+      "    100000      |     0.1086      |      1000       |       100      \n",
+      "\n",
+      "Failed\n",
+      "  complexity    |      error      |    n_datums     |    n_labels     |       msg      \n",
+      "---------------------------------------------------------------------------------------\n",
+      "   10000000     |   MemoryError   |      10000      |      1000       | Unable to allocate 7.63 MiB for an array with shape (1000, 1000) and data type int64\n",
+      "    1000000     |   MemoryError   |      1000       |      1000       |                \n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "_ = b.run(\n",
+    "    benchmark=semseg_finalize,\n",
+    "    n_datums=n_datums,\n",
+    "    n_labels=n_labels,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 20%|██        | 4/20 [02:25<09:40, 36.28s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=====================================================================\n",
+      "Details\n",
+      "{\n",
+      "    \"benchmark\": \"benchmark_evaluate\",\n",
+      "    \"limits\": {\n",
+      "        \"memory_limit\": \"8.0 GB\",\n",
+      "        \"time_limit\": \"5.0 seconds\",\n",
+      "        \"repeat\": 1\n",
+      "    },\n",
+      "    \"passed\": 18,\n",
+      "    \"failed\": 2,\n",
+      "    \"total\": 20\n",
+      "}\n",
+      "\n",
+      "Passed\n",
+      "  complexity    |     runtime     |    n_datums     |    n_labels    \n",
+      "---------------------------------------------------------------------\n",
+      "    1000000     |     0.0537      |      10000      |       100      \n",
+      "    100000      |     0.0815      |       100       |      1000      \n",
+      "    100000      |     0.0137      |      1000       |       100      \n",
+      "\n",
+      "Failed\n",
+      "  complexity    |      error      |    n_datums     |    n_labels     |       msg      \n",
+      "---------------------------------------------------------------------------------------\n",
+      "   10000000     |   MemoryError   |      10000      |      1000       | Unable to allocate 23.8 MiB for an array with shape (1000, 1000, 25) and data type bool\n",
+      "    1000000     |   MemoryError   |      1000       |      1000       | Unable to allocate 3.73 GiB for an array with shape (1000, 1001, 1001) and data type int32\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "_ = b.run(\n",
+    "    benchmark=semseg_evaluate,\n",
+    "    n_datums=n_datums,\n",
+    "    n_labels=n_labels,\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".env-valor",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

{valor_lite-0.33.17 → valor_lite-0.33.19}/examples/object-detection.ipynb

@@ -959,7 +959,7 @@
       "id": "98edc4dd",
       "metadata": {},
       "source": [
-        "### Hallucinations"
+        "### Unmatched Predictions"
       ]
     },
     {
@@ -1764,7 +1764,7 @@
         }
       ],
       "source": [
-        "metric.value[\"hallucinations\"]"
+        "metric.value[\"unmatched_predictions\"]"
       ]
     },
     {
@@ -1773,7 +1773,7 @@
       "id": "415335e4",
       "metadata": {},
       "source": [
-        "### Ground Truths Missing Predictions"
+        "### Unmatched Ground Truths"
       ]
     },
     {
@@ -2995,7 +2995,7 @@
         }
       ],
       "source": [
-        "metric.value[\"missing_predictions\"]"
+        "metric.value[\"unmatched_ground_truths\"]"
       ]
     }
   ],

{valor_lite-0.33.17 → valor_lite-0.33.19}/examples/tabular_classification.ipynb

@@ -605,7 +605,7 @@
     }
    ],
    "source": [
-    "cm.value[\"missing_predictions\"]"
+    "cm.value[\"unmatched_ground_truths\"]"
    ]
   }
  ],

{valor_lite-0.33.17 → valor_lite-0.33.19}/tests/classification/test_confusion_matrix.py

@@ -40,7 +40,7 @@ def test_compute_confusion_matrix():
 
     score_thresholds = np.array([0.25, 0.75], dtype=np.float64)
 
-    confusion_matrix, missing_predictions = compute_confusion_matrix(
+    confusion_matrix, unmatched_ground_truths = compute_confusion_matrix(
         data=data,
         label_metadata=label_metadata,
         score_thresholds=score_thresholds,
@@ -74,15 +74,15 @@ def test_compute_confusion_matrix():
         )
     ).all()
 
-    assert missing_predictions.shape == (2, 4, 1)
+    assert unmatched_ground_truths.shape == (2, 4, 1)
     assert (
         # score >= 0.25
-        missing_predictions[0, :, 0]
+        unmatched_ground_truths[0, :, 0]
         == np.array([-1.0, -1.0, -1.0, -1.0])
     ).all()
     assert (
         # score >= 0.75
-        missing_predictions[1, :, 0]
+        unmatched_ground_truths[1, :, 0]
         == np.array([-1.0, -1.0, -1.0, 1.0])
     ).all()
 
@@ -144,7 +144,7 @@ def test_confusion_matrix_basic(basic_classifications: list[Classification]):
                         }
                     },
                 },
-                "missing_predictions": {},
+                "unmatched_ground_truths": {},
             },
             "parameters": {
                 "score_threshold": 0.25,
@@ -166,7 +166,7 @@ def test_confusion_matrix_basic(basic_classifications: list[Classification]):
                         },
                     }
                 },
-                "missing_predictions": {
+                "unmatched_ground_truths": {
                     "3": {"count": 1, "examples": [{"datum": "uid2"}]}
                 },
             },
@@ -179,7 +179,7 @@ def test_confusion_matrix_basic(basic_classifications: list[Classification]):
     for m in actual_metrics:
         _filter_elements_with_zero_count(
             cm=m["value"]["confusion_matrix"],
-            mp=m["value"]["missing_predictions"],
+            mp=m["value"]["unmatched_ground_truths"],
         )
         assert m in expected_metrics
     for m in expected_metrics:
@@ -212,7 +212,7 @@ def test_confusion_matrix_unit(
                     "1": {"1": {"count": 1, "examples": []}},
                     "2": {"1": {"count": 2, "examples": []}},
                 },
-                "missing_predictions": {},
+                "unmatched_ground_truths": {},
             },
             "parameters": {
                 "score_threshold": 0.5,
@@ -223,7 +223,7 @@ def test_confusion_matrix_unit(
     for m in actual_metrics:
         _filter_elements_with_zero_count(
             cm=m["value"]["confusion_matrix"],
-            mp=m["value"]["missing_predictions"],
+            mp=m["value"]["unmatched_ground_truths"],
         )
         assert m in expected_metrics
     for m in expected_metrics:
@@ -282,7 +282,7 @@ def test_confusion_matrix_with_animal_example(
                         }
                     },
                 },
-                "missing_predictions": {
+                "unmatched_ground_truths": {
                     "dog": {"count": 1, "examples": [{"datum": "uid5"}]}
                 },
             },
@@ -295,7 +295,7 @@ def test_confusion_matrix_with_animal_example(
     for m in actual_metrics:
         _filter_elements_with_zero_count(
             cm=m["value"]["confusion_matrix"],
-            mp=m["value"]["missing_predictions"],
+            mp=m["value"]["unmatched_ground_truths"],
         )
         assert m in expected_metrics
     for m in expected_metrics:
@@ -356,7 +356,7 @@ def test_confusion_matrix_with_color_example(
                         }
                     },
                 },
-                "missing_predictions": {
+                "unmatched_ground_truths": {
                     "red": {"count": 1, "examples": [{"datum": "uid2"}]}
                 },
             },
@@ -369,7 +369,7 @@ def test_confusion_matrix_with_color_example(
     for m in actual_metrics:
         _filter_elements_with_zero_count(
             cm=m["value"]["confusion_matrix"],
-            mp=m["value"]["missing_predictions"],
+            mp=m["value"]["unmatched_ground_truths"],
         )
         assert m in expected_metrics
     for m in expected_metrics:
@@ -438,7 +438,7 @@ def test_confusion_matrix_multiclass(
                         }
                     },
                 },
-                "missing_predictions": {},
+                "unmatched_ground_truths": {},
             },
             "parameters": {
                 "score_threshold": 0.05,
@@ -466,7 +466,7 @@ def test_confusion_matrix_multiclass(
                         }
                     },
                 },
-                "missing_predictions": {
+                "unmatched_ground_truths": {
                     "cat": {
                         "count": 2,
                         "examples": [{"datum": "uid0"}, {"datum": "uid2"}],
@@ -483,7 +483,7 @@ def test_confusion_matrix_multiclass(
             "type": "ConfusionMatrix",
             "value": {
                 "confusion_matrix": {},
-                "missing_predictions": {
+                "unmatched_ground_truths": {
                     "cat": {
                         "count": 2,
                         "examples": [{"datum": "uid0"}, {"datum": "uid2"}],
@@ -504,7 +504,7 @@ def test_confusion_matrix_multiclass(
     for m in actual_metrics:
         _filter_elements_with_zero_count(
             cm=m["value"]["confusion_matrix"],
-            mp=m["value"]["missing_predictions"],
+            mp=m["value"]["unmatched_ground_truths"],
         )
         assert m in expected_metrics
     for m in expected_metrics:
@@ -560,7 +560,7 @@ def test_confusion_matrix_without_hardmax_animal_example(
                         },
                     }
                 },
-                "missing_predictions": {},
+                "unmatched_ground_truths": {},
             },
             "parameters": {
                 "score_threshold": 0.05,
@@ -580,7 +580,7 @@ def test_confusion_matrix_without_hardmax_animal_example(
                         }
                     }
                 },
-                "missing_predictions": {},
+                "unmatched_ground_truths": {},
             },
             "parameters": {
                 "score_threshold": 0.4,
@@ -591,7 +591,7 @@ def test_confusion_matrix_without_hardmax_animal_example(
             "type": "ConfusionMatrix",
             "value": {
                 "confusion_matrix": {},
-                "missing_predictions": {
+                "unmatched_ground_truths": {
                     "ant": {
                         "count": 1,
                         "examples": [
@@ -611,7 +611,7 @@ def test_confusion_matrix_without_hardmax_animal_example(
     for m in actual_metrics:
         _filter_elements_with_zero_count(
             cm=m["value"]["confusion_matrix"],
-            mp=m["value"]["missing_predictions"],
+            mp=m["value"]["unmatched_ground_truths"],
         )
         assert m in expected_metrics
     for m in expected_metrics:

{valor_lite-0.33.17 → valor_lite-0.33.19}/tests/classification/test_dataloader.py

@@ -8,7 +8,7 @@ def test_no_data():
         loader.finalize()
 
 
-def test_missing_predictions(
+def test_unmatched_ground_truths(
     classifications_no_predictions: list[Classification],
 ):
     loader = DataLoader()

{valor_lite-0.33.17 → valor_lite-0.33.19}/tests/object_detection/conftest.py

@@ -704,7 +704,7 @@ def false_negatives_two_images_one_only_with_different_class_high_confidence_of_
 
 
 @pytest.fixture
-def detections_fp_hallucination_edge_case() -> list[Detection]:
+def detections_fp_unmatched_prediction_edge_case() -> list[Detection]:
     return [
         Detection(
             uid="uid1",
@@ -1093,7 +1093,7 @@ def detections_for_detailed_counting(
                     xmax=rect4[1],
                     ymin=rect4[2],
                     ymax=rect4[3],
-                    labels=["hallucination"],
+                    labels=["no_overlap"],
                     scores=[0.1],
                 ),
             ],

{valor_lite-0.33.17 → valor_lite-0.33.19}/tests/object_detection/test_accuracy.py

@@ -95,9 +95,9 @@ def test_accuracy_metrics_first_class(
     groundtruths
         datum uid1
             box 1 - label v1 - tp
-            box 3 - label v2 - fn missing prediction
+            box 3 - label v2 - fn unmatched ground truths
         datum uid2
-            box 2 - label v1 - fn missing prediction
+            box 2 - label v1 - fn unmatched ground truths
 
     predictions
         datum uid1
@@ -176,7 +176,7 @@ def test_accuracy_metrics_second_class(
 
     groundtruths
         datum uid1
-            box 3 - label v2 - fn missing prediction
+            box 3 - label v2 - fn unmatched ground truths
         datum uid2
            none
     predictions

{valor_lite-0.33.17 → valor_lite-0.33.19}/tests/object_detection/test_average_precision.py

@@ -72,7 +72,7 @@ def test_ap_metrics_first_class(
         datum uid1
             box 1 - label v1 - tp
         datum uid2
-            box 2 - label v1 - fn missing prediction
+            box 2 - label v1 - fn unmatched ground truths
 
     predictions
         datum uid1
@@ -192,7 +192,7 @@ def test_ap_metrics_second_class(
 
     groundtruths
         datum uid1
-            box 3 - label v2 - fn missing prediction
+            box 3 - label v2 - fn unmatched ground truths
         datum uid2
            none
     predictions