valor-lite 0.33.16__py3-none-any.whl → 0.33.18__py3-none-any.whl

valor_lite/object_detection/annotation.py

@@ -142,18 +142,6 @@ class Polygon:
         xmin, ymin, xmax, ymax = self.shape.bounds
         return (xmin, xmax, ymin, ymax)
 
-    @property
-    def annotation(self) -> ShapelyPolygon:
-        """
-        Returns the annotation's data representation.
-
-        Returns
-        -------
-        shapely.geometry.Polygon
-            The polygon shape.
-        """
-        return self.shape
-
 
 @dataclass
 class Bitmask:
@@ -222,18 +210,6 @@ class Bitmask:
         rows, cols = np.nonzero(self.mask)
         return (cols.min(), cols.max(), rows.min(), rows.max())
 
-    @property
-    def annotation(self) -> NDArray[np.bool_]:
-        """
-        Returns the annotation's data representation.
-
-        Returns
-        -------
-        NDArray[np.bool_]
-            The binary mask array.
-        """
-        return self.mask
-
 
 @dataclass
 class Detection:

valor_lite/object_detection/manager.py

@@ -1,17 +1,10 @@
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Type
 
 import numpy as np
-import valor_lite.object_detection.annotation as annotation
 from numpy.typing import NDArray
 from tqdm import tqdm
-from valor_lite.object_detection.annotation import (
-    Bitmask,
-    BoundingBox,
-    Detection,
-    Polygon,
-)
+from valor_lite.object_detection.annotation import Detection
 from valor_lite.object_detection.computation import (
     compute_bbox_iou,
     compute_bitmask_iou,
@@ -341,6 +334,10 @@ class Evaluator:
         return metrics
 
 
+def defaultdict_int():
+    return defaultdict(int)
+
+
 class DataLoader:
     """
     Object Detection DataLoader
@@ -349,8 +346,8 @@ class DataLoader:
     def __init__(self):
         self._evaluator = Evaluator()
         self.pairs: list[NDArray[np.float64]] = list()
-        self.groundtruth_count = defaultdict(lambda: defaultdict(int))
-        self.prediction_count = defaultdict(lambda: defaultdict(int))
+        self.groundtruth_count = defaultdict(defaultdict_int)
+        self.prediction_count = defaultdict(defaultdict_int)
 
     def _add_datum(self, uid: str) -> int:
         """
@@ -396,74 +393,47 @@ class DataLoader:
 
         return self._evaluator.label_to_index[label]
 
-    def _compute_ious_and_cache_pairs(
+    def _cache_pairs(
         self,
         uid_index: int,
         groundtruths: list,
         predictions: list,
-        annotation_type: Type[BoundingBox] | Type[Polygon] | Type[Bitmask],
+        ious: NDArray[np.float64],
     ) -> None:
         """
         Compute IOUs between groundtruths and preditions before storing as pairs.
 
         Parameters
         ----------
-        uid_index: int
+        uid_index : int
             The index of the detection.
-        groundtruths: list
+        groundtruths : list
             A list of groundtruths.
-        predictions: list
+        predictions : list
             A list of predictions.
-        annotation_type: type[BoundingBox] | type[Polygon] | type[Bitmask]
-            The type of annotation to compute IOUs for.
+        ious : NDArray[np.float64]
+            An array with shape (n_preds, n_gts) containing IOUs.
         """
 
-        pairs = list()
-        n_predictions = len(predictions)
-        n_groundtruths = len(groundtruths)
-
-        all_pairs = np.array(
-            [
-                np.array([gann, pann])
-                for _, _, _, pann in predictions
-                for _, _, gann in groundtruths
-            ]
-        )
-
-        match annotation_type:
-            case annotation.BoundingBox:
-                ious = compute_bbox_iou(all_pairs)
-            case annotation.Polygon:
-                ious = compute_polygon_iou(all_pairs)
-            case annotation.Bitmask:
-                ious = compute_bitmask_iou(all_pairs)
-            case _:
-                raise ValueError(
-                    f"Invalid annotation type `{annotation_type}`."
-                )
-
-        ious = ious.reshape(n_predictions, n_groundtruths)
         predictions_with_iou_of_zero = np.where((ious < 1e-9).all(axis=1))[0]
         groundtruths_with_iou_of_zero = np.where((ious < 1e-9).all(axis=0))[0]
 
-        pairs.extend(
-            [
-                np.array(
-                    [
-                        float(uid_index),
-                        float(gidx),
-                        float(pidx),
-                        ious[pidx, gidx],
-                        float(glabel),
-                        float(plabel),
-                        float(score),
-                    ]
-                )
-                for pidx, plabel, score, _ in predictions
-                for gidx, glabel, _ in groundtruths
-                if ious[pidx, gidx] >= 1e-9
-            ]
-        )
+        pairs = [
+            np.array(
+                [
+                    float(uid_index),
+                    float(gidx),
+                    float(pidx),
+                    ious[pidx, gidx],
+                    float(glabel),
+                    float(plabel),
+                    float(score),
+                ]
+            )
+            for pidx, plabel, score in predictions
+            for gidx, glabel in groundtruths
+            if ious[pidx, gidx] >= 1e-9
+        ]
         pairs.extend(
             [
                 np.array(
@@ -496,13 +466,12 @@ class DataLoader:
                 for index in groundtruths_with_iou_of_zero
             ]
         )
-
         self.pairs.append(np.array(pairs))
 
     def _add_data(
         self,
         detections: list[Detection],
-        annotation_type: type[Bitmask] | type[BoundingBox] | type[Polygon],
+        detection_ious: list[NDArray[np.float64]],
         show_progress: bool = False,
     ):
         """
@@ -512,13 +481,15 @@ class DataLoader:
         ----------
         detections : list[Detection]
             A list of Detection objects.
-        annotation_type : type[Bitmask] | type[BoundingBox] | type[Polygon]
-            The annotation type to process.
+        detection_ious : list[NDArray[np.float64]]
+            A list of arrays containing IOUs per detection.
         show_progress : bool, default=False
             Toggle for tqdm progress bar.
         """
         disable_tqdm = not show_progress
-        for detection in tqdm(detections, disable=disable_tqdm):
+        for detection, ious in tqdm(
+            zip(detections, detection_ious), disable=disable_tqdm
+        ):
 
             # update metadata
             self._evaluator.n_datums += 1
@@ -541,11 +512,6 @@ class DataLoader:
             predictions = list()
 
             for gidx, gann in enumerate(detection.groundtruths):
-                if not isinstance(gann, annotation_type):
-                    raise ValueError(
-                        f"Expected {annotation_type}, but annotation is of type {type(gann)}."
-                    )
-
                 self._evaluator.groundtruth_examples[uid_index][
                     gidx
                 ] = gann.extrema
@@ -556,16 +522,10 @@ class DataLoader:
                         (
                             gidx,
                             label_idx,
-                            gann.annotation,
                         )
                     )
 
             for pidx, pann in enumerate(detection.predictions):
-                if not isinstance(pann, annotation_type):
-                    raise ValueError(
-                        f"Expected {annotation_type}, but annotation is of type {type(pann)}."
-                    )
-
                 self._evaluator.prediction_examples[uid_index][
                     pidx
                 ] = pann.extrema
@@ -577,15 +537,14 @@ class DataLoader:
                             pidx,
                             label_idx,
                             pscore,
-                            pann.annotation,
                         )
                     )
 
-            self._compute_ious_and_cache_pairs(
+            self._cache_pairs(
                 uid_index=uid_index,
                 groundtruths=groundtruths,
                 predictions=predictions,
-                annotation_type=annotation_type,
+                ious=ious,
             )
 
     def add_bounding_boxes(
@@ -603,10 +562,22 @@ class DataLoader:
         show_progress : bool, default=False
             Toggle for tqdm progress bar.
         """
+        ious = [
+            compute_bbox_iou(
+                np.array(
+                    [
+                        [gt.extrema, pd.extrema]
+                        for pd in detection.predictions
+                        for gt in detection.groundtruths
+                    ]
+                )
+            ).reshape(len(detection.predictions), len(detection.groundtruths))
+            for detection in detections
+        ]
         return self._add_data(
             detections=detections,
+            detection_ious=ious,
             show_progress=show_progress,
-            annotation_type=BoundingBox,
         )
 
     def add_polygons(
@@ -624,10 +595,22 @@ class DataLoader:
         show_progress : bool, default=False
             Toggle for tqdm progress bar.
         """
+        ious = [
+            compute_polygon_iou(
+                np.array(
+                    [
+                        [gt.shape, pd.shape]  # type: ignore - using the AttributeError as a validator
+                        for pd in detection.predictions
+                        for gt in detection.groundtruths
+                    ]
+                )
+            ).reshape(len(detection.predictions), len(detection.groundtruths))
+            for detection in detections
+        ]
         return self._add_data(
             detections=detections,
+            detection_ious=ious,
             show_progress=show_progress,
-            annotation_type=Polygon,
         )
 
     def add_bitmasks(
@@ -645,10 +628,22 @@ class DataLoader:
         show_progress : bool, default=False
             Toggle for tqdm progress bar.
         """
+        ious = [
+            compute_bitmask_iou(
+                np.array(
+                    [
+                        [gt.mask, pd.mask]  # type: ignore - using the AttributeError as a validator
+                        for pd in detection.predictions
+                        for gt in detection.groundtruths
+                    ]
+                )
+            ).reshape(len(detection.predictions), len(detection.groundtruths))
+            for detection in detections
+        ]
         return self._add_data(
             detections=detections,
+            detection_ious=ious,
             show_progress=show_progress,
-            annotation_type=Bitmask,
         )
 
     def finalize(self) -> Evaluator:

valor_lite/profiling.py

@@ -0,0 +1,374 @@
+import json
+import math
+import multiprocessing as mp
+import resource
+import time
+from collections import deque
+from multiprocessing import Queue
+from typing import Any
+
+from tqdm import tqdm
+
+
+class BenchmarkError(Exception):
+    def __init__(
+        self, benchmark: str, error_type: str, error_message: str
+    ) -> None:
+        super().__init__(
+            f"'{benchmark}' raised '{error_type}' with the following message: {error_message}"
+        )
+
+
+def _timeit_subprocess(*args, __fn, __queue: Queue, **kwargs):
+    """
+    Multiprocessing subprocess that reports either runtime or errors.
+
+    This is handled within a subprocess to protect the benchmark against OOM errors.
+    """
+    try:
+        timer_start = time.perf_counter()
+        __fn(*args, **kwargs)
+        timer_end = time.perf_counter()
+        __queue.put(timer_end - timer_start)
+    except Exception as e:
+        __queue.put(e)
+
+
+def create_runtime_profiler(
+    time_limit: float | None,
+    repeat: int = 1,
+):
+    """
+    Creates a runtime profiler as a decorating function.
+
+    The profiler reports runtime of the wrapped function from a subprocess to protect against OOM errors.
+
+    Parameters
+    ----------
+    time_limit : float, optional
+        An optional time limit to constrain the benchmark.
+    repeat : int, default=1
+        The number of times to repeat the benchmark to produce an average runtime.
+    """
+    ctx = mp.get_context("spawn")
+
+    def decorator(fn):
+        def wrapper(*args, **kwargs):
+            # Record average runtime over repeated runs.
+            elapsed = 0
+            for _ in range(repeat):
+                q = ctx.Queue()
+                p = ctx.Process(
+                    target=_timeit_subprocess,
+                    args=args,
+                    kwargs={"__fn": fn, "__queue": q, **kwargs},
+                )
+                p.start()
+                p.join(timeout=time_limit)
+
+                # Check if computation finishes within the timeout
+                if p.is_alive():
+                    p.terminate()
+                    p.join()
+                    q.close()
+                    q.join_thread()
+                    raise TimeoutError(
+                        f"Function '{fn.__name__}' did not complete within {time_limit} seconds."
+                    )
+
+                # Retrieve the result
+                result = q.get(timeout=1)
+                if isinstance(result, Exception):
+                    raise result
+                elif isinstance(result, float):
+                    elapsed += result
+                else:
+                    raise TypeError(type(result).__name__)
+
+            return elapsed / repeat
+
+        return wrapper
+
+    return decorator
+
+
+def pretty_print_results(results: tuple):
+    valid, invalid, permutations = results
+
+    print(
+        "====================================================================="
+    )
+    print("Details")
+    print(json.dumps(permutations, indent=4))
+
+    if len(valid) > 0:
+        print()
+        print("Passed")
+        keys = ["complexity", "runtime", *valid[0]["details"].keys()]
+        header = " | ".join(f"{header:^15}" for header in keys)
+        print(header)
+        print("-" * len(header))
+        for entry in valid:
+            values = [
+                entry["complexity"],
+                round(entry["runtime"], 4),
+                *entry["details"].values(),
+            ]
+            row = " | ".join(f"{str(value):^15}" for value in values)
+            print(row)
+
+    if len(invalid) > 0:
+        print()
+        print("Failed")
+        keys = ["complexity", "error", *invalid[0]["details"].keys(), "msg"]
+        header = " | ".join(f"{header:^15}" for header in keys)
+        print(header)
+        print("-" * len(header))
+        for entry in invalid:
+            values = [
+                entry["complexity"],
+                entry["error"],
+                *entry["details"].values(),
+                entry["msg"],
+            ]
+            row = " | ".join(f"{str(value):^15}" for value in values)
+            print(row)
+
+
+def _calculate_complexity(params: list[int | tuple[int]]) -> int:
+    """
+    Basic metric of benchmark complexity.
+    """
+    flattened_params = [
+        math.prod(p) if isinstance(p, tuple) else p for p in params
+    ]
+    return math.prod(flattened_params)
+
+
+class Benchmark:
+    def __init__(
+        self,
+        time_limit: float | None,
+        memory_limit: int | None,
+        *_,
+        repeat: int | None = 1,
+        verbose: bool = False,
+    ):
+        self.time_limit = time_limit
+        self.memory_limit = memory_limit
+        self.repeat = repeat
+        self.verbose = verbose
+
+    def get_limits(
+        self,
+        *_,
+        readable: bool = True,
+        memory_unit: str = "GB",
+        time_unit: str = "seconds",
+    ) -> dict[str, str | int | float | None]:
+        """
+        Returns a dictionary of benchmark limits.
+
+        Parameters
+        ----------
+        readable : bool, default=True
+            Toggles whether the output should be human readable.
+        memory_unit : str, default="GB"
+            Toggles what unit to display the memory limit with when 'readable=True'.
+        time_unit : str, default="seconds"
+            Toggles what unit to display the time limit with when 'readable=True'.
+
+        Returns
+        -------
+        dict[str, str | int | float | None]
+            The benchmark limits.
+        """
+
+        memory_value = self.memory_limit
+        if readable and memory_value is not None:
+            match memory_unit:
+                case "TB":
+                    memory_value /= 1024**4
+                case "GB":
+                    memory_value /= 1024**3
+                case "MB":
+                    memory_value /= 1024**2
+                case "KB":
+                    memory_value /= 1024
+                case "B":
+                    pass
+                case _:
+                    valid_set = {"TB", "GB", "MB", "KB", "B"}
+                    raise ValueError(
+                        f"Expected memory unit to be in the set {valid_set}, received '{memory_unit}'."
+                    )
+            memory_value = f"{memory_value} {memory_unit}"
+
+        time_value = self.time_limit
+        if readable and time_value is not None:
+            match time_unit:
+                case "minutes":
+                    time_value /= 60
+                case "seconds":
+                    pass
+                case "milliseconds":
+                    time_value *= 1000
+                case _:
+                    valid_set = {"minutes", "seconds", "milliseconds"}
+                    raise ValueError(
+                        f"Expected time unit to be in the set {valid_set}, received '{time_unit}'."
+                    )
+            time_value = f"{time_value} {time_unit}"
+
+        return {
+            "memory_limit": memory_value,
+            "time_limit": time_value,
+            "repeat": self.repeat,
+        }
+
+    @property
+    def memory_limit(self) -> int | None:
+        """
+        The memory limit in bytes (B).
+        """
+        return self._memory_limit
+
+    @memory_limit.setter
+    def memory_limit(self, limit: int | None):
+        """
+        Stores the memory limit and restricts resources.
+        """
+        self._memory_limit = limit
+        if limit is not None:
+            _, hard = resource.getrlimit(resource.RLIMIT_AS)
+            resource.setrlimit(resource.RLIMIT_AS, (limit, hard))
+
+    def run(
+        self,
+        benchmark,
+        **kwargs: list[Any],
+    ):
+        """
+        Runs a benchmark with ranges of parameters.
+
+        Parameters
+        ----------
+        benchmark : Callable
+            The benchmark function.
+        **kwargs : list[Any]
+            Keyword arguments passing lists of parameters to benchmark. The values should be sorted in
+            decreasing complexity. For example, if the number of labels is a parameter then a higher
+            number of unique labels would be considered "more" complex.
+
+        Example
+        -------
+        >>> b = Benchmark(
+        ...     time_limit=10.0,
+        ...     memory_limit=8 * (1024**3),
+        ...     repeat=1,
+        ...     verbose=False,
+        ... )
+        >>> results = b.run(
+        ...     benchmark=semseg_add_data,
+        ...     n_labels=[
+        ...         100,
+        ...         10,
+        ...     ],
+        ...     shape=[
+        ...         (1000, 1000),
+        ...         (100, 100),
+        ...     ],
+        ... )
+        """
+
+        nvars = len(kwargs)
+        keys = tuple(kwargs.keys())
+        vars = tuple(kwargs[key] for key in keys)
+
+        initial_indices = tuple(0 for _ in range(nvars))
+        max_indices = tuple(len(v) for v in vars)
+        permutations = math.prod(max_indices)
+
+        # Initialize queue with the starting index (0, ...)
+        queue = deque()
+        queue.append(initial_indices)
+
+        # Keep track of explored combinations to avoid duplicates
+        explored = set()
+        explored.add(initial_indices)
+
+        # Store valid combinations that finish within the time limit
+        valid_combinations = []
+        invalid_combinations = []
+
+        pbar = tqdm(total=math.prod(max_indices), disable=(not self.verbose))
+        prev_count = 0
+        while queue:
+
+            current_indices = queue.popleft()
+            parameters = {
+                k: v[current_indices[idx]]
+                for idx, (k, v) in enumerate(zip(keys, vars))
+            }
+            complexity = _calculate_complexity(list(parameters.values()))
+
+            details: dict = {k: str(v) for k, v in parameters.items()}
+
+            # update terminal with status
+            count = len(valid_combinations) + len(invalid_combinations)
+            pbar.update(count - prev_count)
+            prev_count = count
+
+            try:
+                runtime = benchmark(
+                    time_limit=self.time_limit,
+                    repeat=self.repeat,
+                    **parameters,
+                )
+                valid_combinations.append(
+                    {
+                        "complexity": complexity,
+                        "runtime": runtime,
+                        "details": details,
+                    }
+                )
+                continue
+            except Exception as e:
+                invalid_combinations.append(
+                    {
+                        "complexity": complexity,
+                        "error": type(e).__name__,
+                        "msg": str(e),
+                        "details": details,
+                    }
+                )
+
+            for idx in range(nvars):
+                new_indices = list(current_indices)
+                if new_indices[idx] + 1 < max_indices[idx]:
+                    new_indices[idx] += 1
+                    new_indices_tuple = tuple(new_indices)
+                    if new_indices_tuple not in explored:
+                        queue.append(new_indices_tuple)
+                        explored.add(new_indices_tuple)
+
+        valid_combinations.sort(key=lambda x: -x["complexity"])
+        invalid_combinations.sort(key=lambda x: -x["complexity"])
+
+        # clear terminal and display results
+        results = (
+            valid_combinations,
+            invalid_combinations,
+            {
+                "benchmark": benchmark.__name__,
+                "limits": self.get_limits(readable=True),
+                "passed": permutations - len(invalid_combinations),
+                "failed": len(invalid_combinations),
+                "total": permutations,
+            },
+        )
+        pbar.close()
+        if self.verbose:
+            pretty_print_results(results)
+
+        return results

valor_lite/semantic_segmentation/__init__.py

@@ -1,4 +1,4 @@
-from .annotation import Bitmask, Segmentation
+from .annotation import Bitmask, Segmentation, generate_segmentation
 from .manager import DataLoader, Evaluator
 from .metric import Metric, MetricType
 
@@ -9,4 +9,5 @@ __all__ = [
     "Bitmask",
     "Metric",
     "MetricType",
+    "generate_segmentation",
 ]

valor_lite/semantic_segmentation/annotation.py

@@ -29,7 +29,7 @@ class Bitmask:
     def __post_init__(self):
         if self.mask.dtype != np.bool_:
             raise ValueError(
-                f"Bitmask recieved mask with dtype `{self.mask.dtype}`."
+                f"Bitmask recieved mask with dtype '{self.mask.dtype}'."
             )
 
 
@@ -94,3 +94,86 @@ class Segmentation:
 
         self.shape = groundtruth_shape.pop()
         self.size = int(np.prod(np.array(self.shape)))
+
+
+def generate_segmentation(
+    datum_uid: str,
+    number_of_unique_labels: int,
+    mask_height: int,
+    mask_width: int,
+) -> Segmentation:
+    """
+    Generates a semantic segmentation annotation.
+
+    Parameters
+    ----------
+    datum_uid : str
+        The datum UID for the generated segmentation.
+    number_of_unique_labels : int
+        The number of unique labels.
+    mask_height : int
+        The height of the mask in pixels.
+    mask_width : int
+        The width of the mask in pixels.
+
+    Returns
+    -------
+    Segmentation
+        A generated semantic segmenatation annotation.
+    """
+
+    if number_of_unique_labels > 1:
+        common_proba = 0.4 / (number_of_unique_labels - 1)
+        min_proba = min(common_proba, 0.1)
+        labels = [str(i) for i in range(number_of_unique_labels)] + [None]
+        proba = (
+            [0.5]
+            + [common_proba for _ in range(number_of_unique_labels - 1)]
+            + [0.1]
+        )
+    elif number_of_unique_labels == 1:
+        labels = ["0", None]
+        proba = [0.9, 0.1]
+        min_proba = 0.1
+    else:
+        raise ValueError(
+            "The number of unique labels should be greater than zero."
+        )
+
+    probabilities = np.array(proba, dtype=np.float64)
+    weights = (probabilities / min_proba).astype(np.int32)
+
+    indices = np.random.choice(
+        np.arange(len(weights)),
+        size=(mask_height * 2, mask_width),
+        p=probabilities,
+    )
+
+    N = len(labels)
+
+    masks = np.arange(N)[:, None, None] == indices
+
+    gts = []
+    pds = []
+    for lidx in range(N):
+        label = labels[lidx]
+        if label is None:
+            continue
+        gts.append(
+            Bitmask(
+                mask=masks[lidx, :mask_height, :],
+                label=label,
+            )
+        )
+        pds.append(
+            Bitmask(
+                mask=masks[lidx, mask_height:, :],
+                label=label,
+            )
+        )
+
+    return Segmentation(
+        uid=datum_uid,
+        groundtruths=gts,
+        predictions=pds,
+    )

valor_lite/semantic_segmentation/benchmark.py

@@ -0,0 +1,151 @@
+from valor_lite.profiling import create_runtime_profiler
+from valor_lite.semantic_segmentation import DataLoader, generate_segmentation
+
+
+def benchmark_add_data(
+    n_labels: int,
+    shape: tuple[int, int],
+    time_limit: float | None,
+    repeat: int = 1,
+) -> float:
+    """
+    Benchmarks 'Dataloader.add_data' for semantic segmentation.
+
+    Parameters
+    ----------
+    n_labels : int
+        The number of unique labels to generate.
+    shape : tuple[int, int]
+        The size (h,w) of the mask to generate.
+    time_limit : float, optional
+        An optional time limit to constrain the benchmark.
+    repeat : int
+        The number of times to run the benchmark to produce a runtime average.
+
+    Returns
+    -------
+    float
+        The average runtime.
+    """
+
+    profile = create_runtime_profiler(
+        time_limit=time_limit,
+        repeat=repeat,
+    )
+
+    elapsed = 0
+    for _ in range(repeat):
+        data = generate_segmentation(
+            datum_uid="uid",
+            number_of_unique_labels=n_labels,
+            mask_height=shape[0],
+            mask_width=shape[1],
+        )
+        loader = DataLoader()
+        elapsed += profile(loader.add_data)([data])
+    return elapsed / repeat
+
+
+def benchmark_finalize(
+    n_datums: int,
+    n_labels: int,
+    time_limit: float | None,
+    repeat: int = 1,
+):
+    """
+    Benchmarks 'Dataloader.finalize' for semantic segmentation.
+
+    Parameters
+    ----------
+    n_datums : int
+        The number of datums to generate.
+    n_labels : int
+        The number of unique labels to generate.
+    time_limit : float, optional
+        An optional time limit to constrain the benchmark.
+    repeat : int
+        The number of times to run the benchmark to produce a runtime average.
+
+    Returns
+    -------
+    float
+        The average runtime.
+    """
+
+    profile = create_runtime_profiler(
+        time_limit=time_limit,
+        repeat=repeat,
+    )
+
+    elapsed = 0
+    for _ in range(repeat):
+
+        data = [
+            generate_segmentation(
+                datum_uid=str(i),
+                number_of_unique_labels=n_labels,
+                mask_height=5,
+                mask_width=5,
+            )
+            for i in range(10)
+        ]
+        loader = DataLoader()
+        for datum_idx in range(n_datums):
+            segmentation = data[datum_idx % 10]
+            segmentation.uid = str(datum_idx)
+            loader.add_data([segmentation])
+        elapsed += profile(loader.finalize)()
+    return elapsed / repeat
+
+
+def benchmark_evaluate(
+    n_datums: int,
+    n_labels: int,
+    time_limit: float | None,
+    repeat: int = 1,
+):
+    """
+    Benchmarks 'Evaluator.evaluate' for semantic segmentation.
+
+    Parameters
+    ----------
+    n_datums : int
+        The number of datums to generate.
+    n_labels : int
+        The number of unique labels to generate.
+    time_limit : float, optional
+        An optional time limit to constrain the benchmark.
+    repeat : int
+        The number of times to run the benchmark to produce a runtime average.
+
+    Returns
+    -------
+    float
+        The average runtime.
+    """
+
+    profile = create_runtime_profiler(
+        time_limit=time_limit,
+        repeat=repeat,
+    )
+
+    elapsed = 0
+    for _ in range(repeat):
+
+        data = [
+            generate_segmentation(
+                datum_uid=str(i),
+                number_of_unique_labels=n_labels,
+                mask_height=5,
+                mask_width=5,
+            )
+            for i in range(10)
+        ]
+        loader = DataLoader()
+        for datum_idx in range(n_datums):
+            segmentation = data[datum_idx % 10]
+            segmentation.uid = str(datum_idx)
+            loader.add_data([segmentation])
+        evaluator = loader.finalize()
+        elapsed += profile(evaluator.evaluate)()
+    return elapsed / repeat

valor_lite/semantic_segmentation/computation.py

@@ -31,9 +31,6 @@ def compute_intermediate_confusion_matrices(
         A 2-D confusion matrix with shape (n_labels + 1, n_labels + 1).
     """
 
-    n_gt_labels = groundtruth_labels.size
-    n_pd_labels = prediction_labels.size
-
     groundtruth_counts = groundtruths.sum(axis=1)
     prediction_counts = predictions.sum(axis=1)
 
@@ -42,33 +39,23 @@ def compute_intermediate_confusion_matrices(
     ).sum()
 
     intersection_counts = np.logical_and(
-        groundtruths.reshape(n_gt_labels, 1, -1),
-        predictions.reshape(1, n_pd_labels, -1),
+        groundtruths[:, None, :],
+        predictions[None, :, :],
     ).sum(axis=2)
-
-    intersected_groundtruth_counts = intersection_counts.sum(axis=0)
-    intersected_prediction_counts = intersection_counts.sum(axis=1)
+    intersected_groundtruth_counts = intersection_counts.sum(axis=1)
+    intersected_prediction_counts = intersection_counts.sum(axis=0)
 
     confusion_matrix = np.zeros((n_labels + 1, n_labels + 1), dtype=np.int32)
     confusion_matrix[0, 0] = background_counts
-    for gidx in range(n_gt_labels):
-        gt_label_idx = groundtruth_labels[gidx]
-        for pidx in range(n_pd_labels):
-            pd_label_idx = prediction_labels[pidx]
-            confusion_matrix[
-                gt_label_idx + 1,
-                pd_label_idx + 1,
-            ] = intersection_counts[gidx, pidx]
-
-            if gidx == 0:
-                confusion_matrix[0, pd_label_idx + 1] = (
-                    prediction_counts[pidx]
-                    - intersected_prediction_counts[pidx]
-                )
-
-        confusion_matrix[gt_label_idx + 1, 0] = (
-            groundtruth_counts[gidx] - intersected_groundtruth_counts[gidx]
-        )
+    confusion_matrix[
+        np.ix_(groundtruth_labels + 1, prediction_labels + 1)
+    ] = intersection_counts
+    confusion_matrix[0, prediction_labels + 1] = (
+        prediction_counts - intersected_prediction_counts
+    )
+    confusion_matrix[groundtruth_labels + 1, 0] = (
+        groundtruth_counts - intersected_groundtruth_counts
+    )
 
     return confusion_matrix
 

valor_lite/semantic_segmentation/manager.py

@@ -243,6 +243,10 @@ class Evaluator:
         return self.compute_precision_recall_iou(filter_=filter_)
 
 
+def defaultdict_int():
+    return defaultdict(int)
+
+
 class DataLoader:
     """
     Segmentation DataLoader.
@@ -250,8 +254,8 @@ class DataLoader:
 
     def __init__(self):
         self._evaluator = Evaluator()
-        self.groundtruth_count = defaultdict(lambda: defaultdict(int))
-        self.prediction_count = defaultdict(lambda: defaultdict(int))
+        self.groundtruth_count = defaultdict(defaultdict_int)
+        self.prediction_count = defaultdict(defaultdict_int)
         self.matrices = list()
         self.pixel_count = list()
 

{valor_lite-0.33.16.dist-info → valor_lite-0.33.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.16
+Version: 0.33.18
 Summary: Compute valor metrics locally.
 License: MIT License
         

{valor_lite-0.33.16.dist-info → valor_lite-0.33.18.dist-info}/RECORD

@@ -1,5 +1,6 @@
 valor_lite/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
 valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite/profiling.py,sha256=TLIROA1qccFw9NoEkMeQcrvvGGO75c4K5yTIWoCUix8,11746
 valor_lite/schemas.py,sha256=pB0MrPx5qFLbwBWDiOUUm-vmXdWvbJLFCBmKgbcbI5g,198
 valor_lite/classification/__init__.py,sha256=8MI8bGwCxYGqRP7KxG7ezhYv4qQ5947XGvvlF8WPM5g,392
 valor_lite/classification/annotation.py,sha256=0aUOvcwBAZgiNOJuyh-pXyNTG7vP7r8CUfnU3OmpUwQ,1113
@@ -8,15 +9,16 @@ valor_lite/classification/manager.py,sha256=8GXZECSx4CBbG5NfPrA19BPENqmrjo-wZBma
 valor_lite/classification/metric.py,sha256=fkAo-_3s4EIRSkyn3owBSf4_Gp6lBK9xdToDYMWmT8A,12236
 valor_lite/classification/utilities.py,sha256=PmQar06Vt-ew4Jvnn0IM63mq730QVTsdRtFdVu1HMFU,6885
 valor_lite/object_detection/__init__.py,sha256=Ql8rju2q7y0Zd9zFvtBJDRhgQFDm1RSYkTsyH3ZE6pA,648
-valor_lite/object_detection/annotation.py,sha256=o6VfiRobiB0ljqsNBLAYMXgi32RSIR7uTA-dgxq6zBI,8248
+valor_lite/object_detection/annotation.py,sha256=x9bsl8b75yvkMByXXiIYI9d9T03olDqtykSvKJc3aFw,7729
 valor_lite/object_detection/computation.py,sha256=P5ijxEBuZ3mxYjBQy24TiQpGxRmPuS40Gwn44uv0J7M,28064
-valor_lite/object_detection/manager.py,sha256=rHY6-aiPVOXKQk7e_MmKpZxn6wdLAhdlj_njaNdYG7Q,23299
+valor_lite/object_detection/manager.py,sha256=utdILUUCx04EWC0_bHGpEPaxcCOhmsOx5lxT9qU1a9s,23033
 valor_lite/object_detection/metric.py,sha256=8QhdauuaRrzE39idetkFYTPxA12wrBalQDIR4IUzEbg,24794
 valor_lite/object_detection/utilities.py,sha256=98VSW-g8EYI8Cdd9KHLHdm6F4fI89jaX5I4z99zny4s,16271
-valor_lite/semantic_segmentation/__init__.py,sha256=HQQkr3iBPQfdUrsu0uvx-Uyv9SYmumU1B3slbWOnpNY,245
-valor_lite/semantic_segmentation/annotation.py,sha256=CujYFdHS3fgr4Y7mEDs_u1XBmbPJzNU2CdqvjCT_d_A,2938
-valor_lite/semantic_segmentation/computation.py,sha256=rrql3zmpqt4Zygc2BD4SyUfNW_NXC93_kHB-lGBzjXU,5122
-valor_lite/semantic_segmentation/manager.py,sha256=pMepH3zk_fApyFtC9tLrmEYuCbg1n5TLh1J8QRadE44,14287
+valor_lite/semantic_segmentation/__init__.py,sha256=BhTUbwbdJa1FdS4ZA3QSIZ8TuJmdGGLGCd5hX6SzKa4,297
+valor_lite/semantic_segmentation/annotation.py,sha256=xd2qJyIeTW8CT_Goyu3Kvl_51b9b6D3WvUfqwShR0Sk,4990
+valor_lite/semantic_segmentation/benchmark.py,sha256=iVdxUo9LgDbbXUa6eRhZ49LOYw-yyr2W4p9FP3KHg0k,3848
+valor_lite/semantic_segmentation/computation.py,sha256=myHjJZ70f2Xc-PGHx3DcLWvXXRu_H8w9z20n7qV-Abo,4687
+valor_lite/semantic_segmentation/manager.py,sha256=TtwJI7Bsn3zHL2ECOqCmymG-JqREo7I6qxYtycbz54Y,14322
 valor_lite/semantic_segmentation/metric.py,sha256=aJv3wPEl6USLhZ3c4yz6prnBU-EaG4Kz16f0BXcodd4,7046
 valor_lite/semantic_segmentation/utilities.py,sha256=vZM66YNMz9VJclhuKvcWp74nF65s6bscnnD5U9iDW7Q,2925
 valor_lite/text_generation/__init__.py,sha256=pGhpWCSZjLM0pPHCtPykAfos55B8ie3mi9EzbNxfj-U,356
@@ -31,8 +33,8 @@ valor_lite/text_generation/llm/instructions.py,sha256=fz2onBZZWcl5W8iy7zEWkPGU9N
 valor_lite/text_generation/llm/integrations.py,sha256=-rTfdAjq1zH-4ixwYuMQEOQ80pIFzMTe0BYfroVx3Pg,6974
 valor_lite/text_generation/llm/utilities.py,sha256=bjqatGgtVTcl1PrMwiDKTYPGJXKrBrx7PDtzIblGSys,1178
 valor_lite/text_generation/llm/validators.py,sha256=Wzr5RlfF58_2wOU-uTw7C8skan_fYdhy4Gfn0jSJ8HM,2700
-valor_lite-0.33.16.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
-valor_lite-0.33.16.dist-info/METADATA,sha256=mpXXDWKiCL8OsCLqRevVH6AkWMsYBT4Qjqdum3ZYFos,5888
-valor_lite-0.33.16.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-valor_lite-0.33.16.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
-valor_lite-0.33.16.dist-info/RECORD,,
+valor_lite-0.33.18.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite-0.33.18.dist-info/METADATA,sha256=oo3sEQQvJJvAIelgFRB1Me2Jmkk-nb_dkphL2k4wo7Y,5888
+valor_lite-0.33.18.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+valor_lite-0.33.18.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.33.18.dist-info/RECORD,,

{valor_lite-0.33.16.dist-info → valor_lite-0.33.18.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.5.0)
 Root-Is-Purelib: true
 Tag: py3-none-any