valor-lite 0.33.16__tar.gz → 0.33.18__tar.gz

{valor_lite-0.33.16 → valor_lite-0.33.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.16
+Version: 0.33.18
 Summary: Compute valor metrics locally.
 License: MIT License
         

valor_lite-0.33.18/benchmarks/synthetic/benchmark_semantic_segmentation.py

@@ -0,0 +1,94 @@
+from valor_lite.profiling import Benchmark, BenchmarkError
+from valor_lite.semantic_segmentation.benchmark import (
+    benchmark_add_data,
+    benchmark_evaluate,
+    benchmark_finalize,
+)
+
+
+def benchmark(
+    bitmask_shape: tuple[int, int],
+    number_of_unique_labels: int,
+    number_of_images: int,
+    *_,
+    memory_limit: float = 4.0,
+    time_limit: float = 10.0,
+    repeat: int = 1,
+    verbose: bool = False,
+):
+    """
+    Runs a single benchmark.
+
+    Parameters
+    ----------
+    bitmask_shape : tuple[int, int]
+        The size (h, w) of the bitmask array.
+    number_of_unique_labels : int
+        The number of unique labels used in the synthetic example.
+    number_of_images : int
+        The number of distinct datums that are created.
+    memory_limit : float
+        The maximum amount of system memory allowed in gigabytes (GB).
+    time_limit : float
+        The maximum amount of time permitted before killing the benchmark.
+    repeat : int
+        The number of times to run a benchmark to produce an average runtime.
+    verbose : bool, default=False
+        Toggles terminal output of benchmark results.
+    """
+
+    b = Benchmark(
+        time_limit=time_limit,
+        memory_limit=int(memory_limit * (1024**3)),
+        repeat=repeat,
+        verbose=verbose,
+    )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_add_data,
+        n_labels=[number_of_unique_labels],
+        shape=[bitmask_shape],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_finalize,
+        n_datums=[number_of_images],
+        n_labels=[number_of_unique_labels],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_evaluate,
+        n_datums=[number_of_images],
+        n_labels=[number_of_unique_labels],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+
+if __name__ == "__main__":
+
+    benchmark(
+        bitmask_shape=(4000, 4000),
+        number_of_images=1000,
+        number_of_unique_labels=10,
+        memory_limit=4.0,
+        time_limit=10.0,
+        repeat=1,
+        verbose=True,
+    )

valor_lite-0.33.18/examples/benchmarking.ipynb

@@ -0,0 +1,279 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from valor_lite.profiling import Benchmark\n",
+    "\n",
+    "b = Benchmark(\n",
+    "    time_limit=5.0,  # 5s\n",
+    "    memory_limit=8 * (1024 ** 3),  # 8 GB\n",
+    "    repeat=1,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Semantic Segmentation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from valor_lite.semantic_segmentation.benchmark import (\n",
+    "    benchmark_add_data as semseg_add_data,\n",
+    "    benchmark_finalize as semseg_finalize,\n",
+    "    benchmark_evaluate as semseg_evaluate,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "n_datums = [\n",
+    "    10000,\n",
+    "    1000,\n",
+    "    100,\n",
+    "    10,\n",
+    "    1,\n",
+    "]\n",
+    "\n",
+    "n_labels = [\n",
+    "    1000,\n",
+    "    100,\n",
+    "    10,\n",
+    "    3,\n",
+    "]\n",
+    "\n",
+    "shapes = [\n",
+    "    (10000, 10000),\n",
+    "    (2500, 2500),\n",
+    "    (1000, 1000),\n",
+    "    (100, 100),\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 69%|██████▉   | 11/16 [00:46<00:21,  4.26s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=====================================================================\n",
+      "Details\n",
+      "{\n",
+      "    \"benchmark\": \"benchmark_add_data\",\n",
+      "    \"limits\": {\n",
+      "        \"memory_limit\": \"8.0 GB\",\n",
+      "        \"time_limit\": \"5.0 seconds\",\n",
+      "        \"repeat\": 1\n",
+      "    },\n",
+      "    \"passed\": 8,\n",
+      "    \"failed\": 8,\n",
+      "    \"total\": 16\n",
+      "}\n",
+      "\n",
+      "Passed\n",
+      "  complexity    |     runtime     |    n_labels     |      shape     \n",
+      "---------------------------------------------------------------------\n",
+      "   300000000    |     1.5151      |        3        | (10000, 10000) \n",
+      "   62500000     |     0.5952      |       10        |  (2500, 2500)  \n",
+      "   10000000     |     0.0911      |       10        |  (1000, 1000)  \n",
+      "    1000000     |     0.0582      |       100       |   (100, 100)   \n",
+      "\n",
+      "Failed\n",
+      "  complexity    |      error      |    n_labels     |      shape      |       msg      \n",
+      "---------------------------------------------------------------------------------------\n",
+      " 100000000000   |   MemoryError   |      1000       | (10000, 10000)  | Unable to allocate 186. GiB for an array with shape (1001, 20000, 10000) and data type bool\n",
+      "  10000000000   |   MemoryError   |       100       | (10000, 10000)  | Unable to allocate 18.8 GiB for an array with shape (101, 20000, 10000) and data type bool\n",
+      "  6250000000    |   MemoryError   |      1000       |  (2500, 2500)   | Unable to allocate 11.7 GiB for an array with shape (1001, 5000, 2500) and data type bool\n",
+      "  1000000000    |   MemoryError   |       10        | (10000, 10000)  | Unable to allocate 9.31 GiB for an array with shape (10, 10, 100000000) and data type bool\n",
+      "  1000000000    |   MemoryError   |      1000       |  (1000, 1000)   | Unable to allocate 931. GiB for an array with shape (1000, 1000, 1000000) and data type bool\n",
+      "   625000000    |   MemoryError   |       100       |  (2500, 2500)   | Unable to allocate 58.2 GiB for an array with shape (100, 100, 6250000) and data type bool\n",
+      "   100000000    |   MemoryError   |       100       |  (1000, 1000)   | Unable to allocate 9.31 GiB for an array with shape (100, 100, 1000000) and data type bool\n",
+      "   10000000     |   MemoryError   |      1000       |   (100, 100)    | Unable to allocate 9.31 GiB for an array with shape (1000, 1000, 10000) and data type bool\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "_ = b.run(\n",
+    "    benchmark=semseg_add_data,\n",
+    "    n_labels=n_labels,\n",
+    "    shape=shapes,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 20%|██        | 4/20 [02:35<10:22, 38.92s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=====================================================================\n",
+      "Details\n",
+      "{\n",
+      "    \"benchmark\": \"benchmark_finalize\",\n",
+      "    \"limits\": {\n",
+      "        \"memory_limit\": \"8.0 GB\",\n",
+      "        \"time_limit\": \"5.0 seconds\",\n",
+      "        \"repeat\": 1\n",
+      "    },\n",
+      "    \"passed\": 18,\n",
+      "    \"failed\": 2,\n",
+      "    \"total\": 20\n",
+      "}\n",
+      "\n",
+      "Passed\n",
+      "  complexity    |     runtime     |    n_datums     |    n_labels    \n",
+      "---------------------------------------------------------------------\n",
+      "    1000000     |     1.1142      |      10000      |       100      \n",
+      "    100000      |     0.1748      |       100       |      1000      \n",
+      "    100000      |     0.1086      |      1000       |       100      \n",
+      "\n",
+      "Failed\n",
+      "  complexity    |      error      |    n_datums     |    n_labels     |       msg      \n",
+      "---------------------------------------------------------------------------------------\n",
+      "   10000000     |   MemoryError   |      10000      |      1000       | Unable to allocate 7.63 MiB for an array with shape (1000, 1000) and data type int64\n",
+      "    1000000     |   MemoryError   |      1000       |      1000       |                \n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "_ = b.run(\n",
+    "    benchmark=semseg_finalize,\n",
+    "    n_datums=n_datums,\n",
+    "    n_labels=n_labels,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 20%|██        | 4/20 [02:25<09:40, 36.28s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=====================================================================\n",
+      "Details\n",
+      "{\n",
+      "    \"benchmark\": \"benchmark_evaluate\",\n",
+      "    \"limits\": {\n",
+      "        \"memory_limit\": \"8.0 GB\",\n",
+      "        \"time_limit\": \"5.0 seconds\",\n",
+      "        \"repeat\": 1\n",
+      "    },\n",
+      "    \"passed\": 18,\n",
+      "    \"failed\": 2,\n",
+      "    \"total\": 20\n",
+      "}\n",
+      "\n",
+      "Passed\n",
+      "  complexity    |     runtime     |    n_datums     |    n_labels    \n",
+      "---------------------------------------------------------------------\n",
+      "    1000000     |     0.0537      |      10000      |       100      \n",
+      "    100000      |     0.0815      |       100       |      1000      \n",
+      "    100000      |     0.0137      |      1000       |       100      \n",
+      "\n",
+      "Failed\n",
+      "  complexity    |      error      |    n_datums     |    n_labels     |       msg      \n",
+      "---------------------------------------------------------------------------------------\n",
+      "   10000000     |   MemoryError   |      10000      |      1000       | Unable to allocate 23.8 MiB for an array with shape (1000, 1000, 25) and data type bool\n",
+      "    1000000     |   MemoryError   |      1000       |      1000       | Unable to allocate 3.73 GiB for an array with shape (1000, 1001, 1001) and data type int32\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "_ = b.run(\n",
+    "    benchmark=semseg_evaluate,\n",
+    "    n_datums=n_datums,\n",
+    "    n_labels=n_labels,\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".env-valor",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

{valor_lite-0.33.16 → valor_lite-0.33.18}/tests/object_detection/test_dataloader.py

@@ -16,6 +16,49 @@ def test_no_data():
         loader.finalize()
 
 
+def test_iou_computation():
+
+    detection = Detection(
+        uid="uid",
+        groundtruths=[
+            BoundingBox(xmin=0, xmax=10, ymin=0, ymax=10, labels=["0"]),
+            BoundingBox(xmin=100, xmax=110, ymin=100, ymax=110, labels=["0"]),
+            BoundingBox(
+                xmin=1000, xmax=1100, ymin=1000, ymax=1100, labels=["0"]
+            ),
+        ],
+        predictions=[
+            BoundingBox(
+                xmin=1,
+                xmax=11,
+                ymin=1,
+                ymax=11,
+                labels=["0", "1", "2"],
+                scores=[0.5, 0.25, 0.25],
+            ),
+            BoundingBox(
+                xmin=105,
+                xmax=116,
+                ymin=105,
+                ymax=116,
+                labels=["0", "1", "2"],
+                scores=[0.5, 0.25, 0.25],
+            ),
+        ],
+    )
+
+    loader = DataLoader()
+    loader.add_bounding_boxes([detection])
+
+    assert len(loader.pairs) == 1
+
+    # show that three unique IOUs exist
+    unique_ious = np.unique(loader.pairs[0][:, 3])
+    assert np.isclose(
+        unique_ious, np.array([0.0, 0.12755102, 0.68067227])
+    ).all()
+
+
 def test_mixed_annotations(
     rect1: tuple[float, float, float, float],
     rect1_rotated_5_degrees_around_origin: tuple[float, float, float, float],
@@ -87,7 +130,15 @@ def test_mixed_annotations(
 
     loader = DataLoader()
 
-    for input_ in mixed_detections:
-        with pytest.raises(ValueError) as e:
-            loader.add_bounding_boxes([input_])
-        assert "but annotation is of type" in str(e)
+    for detection in mixed_detections:
+
+        # anything can be converted to a bbox
+        loader.add_bounding_boxes([detection])
+
+        with pytest.raises(AttributeError) as e:
+            loader.add_polygons([detection])
+        assert "no attribute 'shape'" in str(e)
+
+        with pytest.raises(AttributeError) as e:
+            loader.add_bitmasks([detection])
+        assert "no attribute 'mask'" in str(e)

{valor_lite-0.33.16 → valor_lite-0.33.18}/tests/semantic_segmentation/test_annotation.py

@@ -1,6 +1,10 @@
 import numpy as np
 import pytest
-from valor_lite.semantic_segmentation import Bitmask, Segmentation
+from valor_lite.semantic_segmentation import (
+    Bitmask,
+    Segmentation,
+    generate_segmentation,
+)
 
 
 def test_bitmask():
@@ -78,3 +82,55 @@ def test_segmentation():
             predictions=[],
         )
     assert "missing predictions" in str(e)
+
+
+def test_generate_segmentation():
+
+    # N labels > 1
+    segmentation = generate_segmentation(
+        datum_uid="uid1",
+        number_of_unique_labels=3,
+        mask_height=2,
+        mask_width=3,
+    )
+
+    assert segmentation.uid == "uid1"
+    assert segmentation.shape == (2, 3)
+    assert segmentation.size == 6
+
+    assert len(segmentation.groundtruths) == 3
+    assert all(gt.mask.dtype == np.bool_ for gt in segmentation.groundtruths)
+    assert all(gt.mask.shape == (2, 3) for gt in segmentation.groundtruths)
+
+    assert len(segmentation.predictions) == 3
+    assert all(pd.mask.dtype == np.bool_ for pd in segmentation.predictions)
+    assert all(pd.mask.shape == (2, 3) for pd in segmentation.predictions)
+
+    # N labels = 1
+    segmentation = generate_segmentation(
+        datum_uid="uid1",
+        number_of_unique_labels=1,
+        mask_height=2,
+        mask_width=3,
+    )
+
+    assert segmentation.uid == "uid1"
+    assert segmentation.shape == (2, 3)
+    assert segmentation.size == 6
+
+    assert len(segmentation.groundtruths) == 1
+    assert all(gt.mask.dtype == np.bool_ for gt in segmentation.groundtruths)
+    assert all(gt.mask.shape == (2, 3) for gt in segmentation.groundtruths)
+
+    assert len(segmentation.predictions) == 1
+    assert all(pd.mask.dtype == np.bool_ for pd in segmentation.predictions)
+    assert all(pd.mask.shape == (2, 3) for pd in segmentation.predictions)
+
+    # N labels = 0
+    with pytest.raises(ValueError):
+        generate_segmentation(
+            datum_uid="uid1",
+            number_of_unique_labels=0,
+            mask_height=2,
+            mask_width=3,
+        )

{valor_lite-0.33.16 → valor_lite-0.33.18}/tests/semantic_segmentation/test_confusion_matrix.py

@@ -1,4 +1,6 @@
+import numpy as np
 from valor_lite.semantic_segmentation import (
+    Bitmask,
     DataLoader,
     MetricType,
     Segmentation,
@@ -89,3 +91,63 @@ def test_confusion_matrix_segmentations_from_boxes(
         assert m in expected_metrics
     for m in expected_metrics:
         assert m in actual_metrics
+
+
+def test_confusion_matrix_intermediate_counting():
+
+    segmentation = Segmentation(
+        uid="uid1",
+        groundtruths=[
+            Bitmask(
+                mask=np.array([[False, False], [True, False]]),
+                label="a",
+            ),
+            Bitmask(
+                mask=np.array([[False, False], [False, True]]),
+                label="b",
+            ),
+            Bitmask(
+                mask=np.array([[True, False], [False, False]]),
+                label="c",
+            ),
+            Bitmask(
+                mask=np.array([[False, True], [False, False]]),
+                label="d",
+            ),
+        ],
+        predictions=[
+            Bitmask(
+                mask=np.array([[False, False], [False, False]]),
+                label="a",
+            ),
+            Bitmask(
+                mask=np.array([[False, False], [False, False]]),
+                label="b",
+            ),
+            Bitmask(
+                mask=np.array([[True, True], [True, True]]),
+                label="c",
+            ),
+            Bitmask(
+                mask=np.array([[False, False], [False, False]]),
+                label="d",
+            ),
+        ],
+    )
+
+    loader = DataLoader()
+    loader.add_data([segmentation])
+
+    assert len(loader.matrices) == 1
+    assert (
+        loader.matrices[0]
+        == np.array(
+            [
+                [0, 0, 0, 0, 0],
+                [0, 0, 0, 1, 0],
+                [0, 0, 0, 1, 0],
+                [0, 0, 0, 1, 0],
+                [0, 0, 0, 1, 0],
+            ]
+        )
+    ).all()

{valor_lite-0.33.16 → valor_lite-0.33.18}/valor_lite/object_detection/annotation.py

@@ -142,18 +142,6 @@ class Polygon:
         xmin, ymin, xmax, ymax = self.shape.bounds
         return (xmin, xmax, ymin, ymax)
 
-    @property
-    def annotation(self) -> ShapelyPolygon:
-        """
-        Returns the annotation's data representation.
-
-        Returns
-        -------
-        shapely.geometry.Polygon
-            The polygon shape.
-        """
-        return self.shape
-
 
 @dataclass
 class Bitmask:
@@ -222,18 +210,6 @@ class Bitmask:
         rows, cols = np.nonzero(self.mask)
         return (cols.min(), cols.max(), rows.min(), rows.max())
 
-    @property
-    def annotation(self) -> NDArray[np.bool_]:
-        """
-        Returns the annotation's data representation.
-
-        Returns
-        -------
-        NDArray[np.bool_]
-            The binary mask array.
-        """
-        return self.mask
-
 
 @dataclass
 class Detection: