valor-lite 0.33.17__tar.gz → 0.33.18__tar.gz

{valor_lite-0.33.17/valor_lite.egg-info → valor_lite-0.33.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.17
+Version: 0.33.18
 Summary: Compute valor metrics locally.
 License: MIT License
         

valor_lite-0.33.18/benchmarks/synthetic/benchmark_semantic_segmentation.py

@@ -0,0 +1,94 @@
+from valor_lite.profiling import Benchmark, BenchmarkError
+from valor_lite.semantic_segmentation.benchmark import (
+    benchmark_add_data,
+    benchmark_evaluate,
+    benchmark_finalize,
+)
+
+
+def benchmark(
+    bitmask_shape: tuple[int, int],
+    number_of_unique_labels: int,
+    number_of_images: int,
+    *_,
+    memory_limit: float = 4.0,
+    time_limit: float = 10.0,
+    repeat: int = 1,
+    verbose: bool = False,
+):
+    """
+    Runs a single benchmark.
+
+    Parameters
+    ----------
+    bitmask_shape : tuple[int, int]
+        The size (h, w) of the bitmask array.
+    number_of_unique_labels : int
+        The number of unique labels used in the synthetic example.
+    number_of_images : int
+        The number of distinct datums that are created.
+    memory_limit : float
+        The maximum amount of system memory allowed in gigabytes (GB).
+    time_limit : float
+        The maximum amount of time permitted before killing the benchmark.
+    repeat : int
+        The number of times to run a benchmark to produce an average runtime.
+    verbose : bool, default=False
+        Toggles terminal output of benchmark results.
+    """
+
+    b = Benchmark(
+        time_limit=time_limit,
+        memory_limit=int(memory_limit * (1024**3)),
+        repeat=repeat,
+        verbose=verbose,
+    )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_add_data,
+        n_labels=[number_of_unique_labels],
+        shape=[bitmask_shape],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_finalize,
+        n_datums=[number_of_images],
+        n_labels=[number_of_unique_labels],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_evaluate,
+        n_datums=[number_of_images],
+        n_labels=[number_of_unique_labels],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+
+if __name__ == "__main__":
+
+    benchmark(
+        bitmask_shape=(4000, 4000),
+        number_of_images=1000,
+        number_of_unique_labels=10,
+        memory_limit=4.0,
+        time_limit=10.0,
+        repeat=1,
+        verbose=True,
+    )

valor_lite-0.33.18/examples/benchmarking.ipynb

@@ -0,0 +1,279 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from valor_lite.profiling import Benchmark\n",
+    "\n",
+    "b = Benchmark(\n",
+    "    time_limit=5.0,  # 5s\n",
+    "    memory_limit=8 * (1024 ** 3),  # 8 GB\n",
+    "    repeat=1,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Semantic Segmentation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from valor_lite.semantic_segmentation.benchmark import (\n",
+    "    benchmark_add_data as semseg_add_data,\n",
+    "    benchmark_finalize as semseg_finalize,\n",
+    "    benchmark_evaluate as semseg_evaluate,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "n_datums = [\n",
+    "    10000,\n",
+    "    1000,\n",
+    "    100,\n",
+    "    10,\n",
+    "    1,\n",
+    "]\n",
+    "\n",
+    "n_labels = [\n",
+    "    1000,\n",
+    "    100,\n",
+    "    10,\n",
+    "    3,\n",
+    "]\n",
+    "\n",
+    "shapes = [\n",
+    "    (10000, 10000),\n",
+    "    (2500, 2500),\n",
+    "    (1000, 1000),\n",
+    "    (100, 100),\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 69%|██████▉   | 11/16 [00:46<00:21,  4.26s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=====================================================================\n",
+      "Details\n",
+      "{\n",
+      "    \"benchmark\": \"benchmark_add_data\",\n",
+      "    \"limits\": {\n",
+      "        \"memory_limit\": \"8.0 GB\",\n",
+      "        \"time_limit\": \"5.0 seconds\",\n",
+      "        \"repeat\": 1\n",
+      "    },\n",
+      "    \"passed\": 8,\n",
+      "    \"failed\": 8,\n",
+      "    \"total\": 16\n",
+      "}\n",
+      "\n",
+      "Passed\n",
+      "  complexity    |     runtime     |    n_labels     |      shape     \n",
+      "---------------------------------------------------------------------\n",
+      "   300000000    |     1.5151      |        3        | (10000, 10000) \n",
+      "   62500000     |     0.5952      |       10        |  (2500, 2500)  \n",
+      "   10000000     |     0.0911      |       10        |  (1000, 1000)  \n",
+      "    1000000     |     0.0582      |       100       |   (100, 100)   \n",
+      "\n",
+      "Failed\n",
+      "  complexity    |      error      |    n_labels     |      shape      |       msg      \n",
+      "---------------------------------------------------------------------------------------\n",
+      " 100000000000   |   MemoryError   |      1000       | (10000, 10000)  | Unable to allocate 186. GiB for an array with shape (1001, 20000, 10000) and data type bool\n",
+      "  10000000000   |   MemoryError   |       100       | (10000, 10000)  | Unable to allocate 18.8 GiB for an array with shape (101, 20000, 10000) and data type bool\n",
+      "  6250000000    |   MemoryError   |      1000       |  (2500, 2500)   | Unable to allocate 11.7 GiB for an array with shape (1001, 5000, 2500) and data type bool\n",
+      "  1000000000    |   MemoryError   |       10        | (10000, 10000)  | Unable to allocate 9.31 GiB for an array with shape (10, 10, 100000000) and data type bool\n",
+      "  1000000000    |   MemoryError   |      1000       |  (1000, 1000)   | Unable to allocate 931. GiB for an array with shape (1000, 1000, 1000000) and data type bool\n",
+      "   625000000    |   MemoryError   |       100       |  (2500, 2500)   | Unable to allocate 58.2 GiB for an array with shape (100, 100, 6250000) and data type bool\n",
+      "   100000000    |   MemoryError   |       100       |  (1000, 1000)   | Unable to allocate 9.31 GiB for an array with shape (100, 100, 1000000) and data type bool\n",
+      "   10000000     |   MemoryError   |      1000       |   (100, 100)    | Unable to allocate 9.31 GiB for an array with shape (1000, 1000, 10000) and data type bool\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "_ = b.run(\n",
+    "    benchmark=semseg_add_data,\n",
+    "    n_labels=n_labels,\n",
+    "    shape=shapes,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 20%|██        | 4/20 [02:35<10:22, 38.92s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=====================================================================\n",
+      "Details\n",
+      "{\n",
+      "    \"benchmark\": \"benchmark_finalize\",\n",
+      "    \"limits\": {\n",
+      "        \"memory_limit\": \"8.0 GB\",\n",
+      "        \"time_limit\": \"5.0 seconds\",\n",
+      "        \"repeat\": 1\n",
+      "    },\n",
+      "    \"passed\": 18,\n",
+      "    \"failed\": 2,\n",
+      "    \"total\": 20\n",
+      "}\n",
+      "\n",
+      "Passed\n",
+      "  complexity    |     runtime     |    n_datums     |    n_labels    \n",
+      "---------------------------------------------------------------------\n",
+      "    1000000     |     1.1142      |      10000      |       100      \n",
+      "    100000      |     0.1748      |       100       |      1000      \n",
+      "    100000      |     0.1086      |      1000       |       100      \n",
+      "\n",
+      "Failed\n",
+      "  complexity    |      error      |    n_datums     |    n_labels     |       msg      \n",
+      "---------------------------------------------------------------------------------------\n",
+      "   10000000     |   MemoryError   |      10000      |      1000       | Unable to allocate 7.63 MiB for an array with shape (1000, 1000) and data type int64\n",
+      "    1000000     |   MemoryError   |      1000       |      1000       |                \n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "_ = b.run(\n",
+    "    benchmark=semseg_finalize,\n",
+    "    n_datums=n_datums,\n",
+    "    n_labels=n_labels,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 20%|██        | 4/20 [02:25<09:40, 36.28s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=====================================================================\n",
+      "Details\n",
+      "{\n",
+      "    \"benchmark\": \"benchmark_evaluate\",\n",
+      "    \"limits\": {\n",
+      "        \"memory_limit\": \"8.0 GB\",\n",
+      "        \"time_limit\": \"5.0 seconds\",\n",
+      "        \"repeat\": 1\n",
+      "    },\n",
+      "    \"passed\": 18,\n",
+      "    \"failed\": 2,\n",
+      "    \"total\": 20\n",
+      "}\n",
+      "\n",
+      "Passed\n",
+      "  complexity    |     runtime     |    n_datums     |    n_labels    \n",
+      "---------------------------------------------------------------------\n",
+      "    1000000     |     0.0537      |      10000      |       100      \n",
+      "    100000      |     0.0815      |       100       |      1000      \n",
+      "    100000      |     0.0137      |      1000       |       100      \n",
+      "\n",
+      "Failed\n",
+      "  complexity    |      error      |    n_datums     |    n_labels     |       msg      \n",
+      "---------------------------------------------------------------------------------------\n",
+      "   10000000     |   MemoryError   |      10000      |      1000       | Unable to allocate 23.8 MiB for an array with shape (1000, 1000, 25) and data type bool\n",
+      "    1000000     |   MemoryError   |      1000       |      1000       | Unable to allocate 3.73 GiB for an array with shape (1000, 1001, 1001) and data type int32\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "_ = b.run(\n",
+    "    benchmark=semseg_evaluate,\n",
+    "    n_datums=n_datums,\n",
+    "    n_labels=n_labels,\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".env-valor",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

{valor_lite-0.33.17 → valor_lite-0.33.18}/tests/semantic_segmentation/test_annotation.py

@@ -1,6 +1,10 @@
 import numpy as np
 import pytest
-from valor_lite.semantic_segmentation import Bitmask, Segmentation
+from valor_lite.semantic_segmentation import (
+    Bitmask,
+    Segmentation,
+    generate_segmentation,
+)
 
 
 def test_bitmask():
@@ -78,3 +82,55 @@ def test_segmentation():
             predictions=[],
         )
     assert "missing predictions" in str(e)
+
+
+def test_generate_segmentation():
+
+    # N labels > 1
+    segmentation = generate_segmentation(
+        datum_uid="uid1",
+        number_of_unique_labels=3,
+        mask_height=2,
+        mask_width=3,
+    )
+
+    assert segmentation.uid == "uid1"
+    assert segmentation.shape == (2, 3)
+    assert segmentation.size == 6
+
+    assert len(segmentation.groundtruths) == 3
+    assert all(gt.mask.dtype == np.bool_ for gt in segmentation.groundtruths)
+    assert all(gt.mask.shape == (2, 3) for gt in segmentation.groundtruths)
+
+    assert len(segmentation.predictions) == 3
+    assert all(pd.mask.dtype == np.bool_ for pd in segmentation.predictions)
+    assert all(pd.mask.shape == (2, 3) for pd in segmentation.predictions)
+
+    # N labels = 1
+    segmentation = generate_segmentation(
+        datum_uid="uid1",
+        number_of_unique_labels=1,
+        mask_height=2,
+        mask_width=3,
+    )
+
+    assert segmentation.uid == "uid1"
+    assert segmentation.shape == (2, 3)
+    assert segmentation.size == 6
+
+    assert len(segmentation.groundtruths) == 1
+    assert all(gt.mask.dtype == np.bool_ for gt in segmentation.groundtruths)
+    assert all(gt.mask.shape == (2, 3) for gt in segmentation.groundtruths)
+
+    assert len(segmentation.predictions) == 1
+    assert all(pd.mask.dtype == np.bool_ for pd in segmentation.predictions)
+    assert all(pd.mask.shape == (2, 3) for pd in segmentation.predictions)
+
+    # N labels = 0
+    with pytest.raises(ValueError):
+        generate_segmentation(
+            datum_uid="uid1",
+            number_of_unique_labels=0,
+            mask_height=2,
+            mask_width=3,
+        )

{valor_lite-0.33.17 → valor_lite-0.33.18}/valor_lite/object_detection/manager.py

@@ -334,6 +334,10 @@ class Evaluator:
         return metrics
 
 
+def defaultdict_int():
+    return defaultdict(int)
+
+
 class DataLoader:
     """
     Object Detection DataLoader
@@ -342,8 +346,8 @@ class DataLoader:
     def __init__(self):
         self._evaluator = Evaluator()
         self.pairs: list[NDArray[np.float64]] = list()
-        self.groundtruth_count = defaultdict(lambda: defaultdict(int))
-        self.prediction_count = defaultdict(lambda: defaultdict(int))
+        self.groundtruth_count = defaultdict(defaultdict_int)
+        self.prediction_count = defaultdict(defaultdict_int)
 
     def _add_datum(self, uid: str) -> int:
         """